From 8b0e2287a5202816c7a8805e45aacb39d2283d48 Mon Sep 17 00:00:00 2001 From: Eric Meehan Date: Wed, 21 Oct 2026 02:39:02 -0400 Subject: [PATCH] Paired-backup verifier now checks more than the manifest hash --- compose.custody-backup.yaml | 1 + docs/operations-recovery.md | 6 +- docs/postgres-financial-storage.md | 2 +- scripts/custody-pair-backup.ts | 6 +- server/coordinated-backup.ts | 92 +++++++++++++++++++++++++++++- tests/postgres/custody.test.ts | 53 +++++++++++++++-- 6 files changed, 144 insertions(+), 16 deletions(-) diff --git a/compose.custody-backup.yaml b/compose.custody-backup.yaml index 9edb56d..982de0d 100755 --- a/compose.custody-backup.yaml +++ b/compose.custody-backup.yaml @@ -1,5 +1,6 @@ # Explicit local sandbox backup helper. Start the persistent stack first, then stop only wikideck. # Requires WIKIDECK_BACKUP_DIR (existing restricted host directory) and WIKIDECK_BACKUP_NAME (new leaf name). +# For verification, override the service command with: verify --source /backups/. services: backup: image: wikideck:custody-backup diff --git a/docs/operations-recovery.md b/docs/operations-recovery.md index 7e09e0e..cca99ee 100755 --- a/docs/operations-recovery.md +++ b/docs/operations-recovery.md @@ -72,10 +72,10 @@ This procedure is **not yet a production backup runbook**. Use only after review 1. Stop only the app service and verify no other node/application process writes the same data directory. Leave the PostgreSQL service running. Do not stop/restart or remove any user deployment as part of development tests. 2. With the matching stopped-node directory, run the one-shot provision and migration services under an approved maintenance window so older v1 local secret volumes receive the additive read-only backup credential and database role. Verify existing passwords, CA and custodian ID were preserved; do not regenerate volumes. 3. Set `WIKIDECK_BACKUP_DIR` to an **existing, restricted** host directory on encrypted storage and `WIKIDECK_BACKUP_NAME` to a new leaf name. Run `docker compose -p wikideck-custody-local -f compose.custody-backup.yaml run --rm backup`. The helper requires an existing destination parent and refuses overwrite. The node lock blocks startup during the entire paired snapshot. A PostgreSQL session lock fences all compliant Wikideck custody writers; privileged/manual DB writers must also be operationally fenced. -4. Run the helper with `verify --source /backups/` (overriding the service command) to check the strict pair manifest, standalone node databases, hashes, and exact package contents. Verify pg_dump compatibility/restorability in a separately isolated PostgreSQL database; verification here does not by itself parse/restore the archive. +4. Run the helper with `verify --source /backups/` (overriding the service command) to check the strict pair manifest, exact node/root directory contents, hashes, recorded server/client versions, and that a compatible `pg_restore --list` parses the custom archive and includes required custody accounting tables. This is structural/content-presence validation, not a full restore or authenticity proof. Separately rehearse restoration into a new isolated PostgreSQL database; never point `pg_restore` at the authoritative database. Use matching or newer PostgreSQL client tools; this helper refuses a restore client older than the archived dump utility. 5. Store an encrypted off-host copy with separately protected authenticity metadata. This package is **not encrypted** by the utility, and colocating it with the source host does not satisfy disaster recovery. -Creation holds locks and runs a full financial audit; a large journal may exceed configured query/audit budgets and fail closed, so this needs measured long-history tests before production use. A database superuser/schema owner can bypass application advisory-lock discipline or rewrite records; external writer fencing remains mandatory. The output hash/manifest detects accidental damage, not a malicious coherent replacement. The PostgreSQL dump is not atomic with arbitrary unmanaged ledger or schema-owner activity. There is no WAL/PITR, remote scheduler, retention, restore orchestration, RPO/RTO claim, encryption, signature or production failover process in this increment. The local Compose backup service mounts the external host destination read-write into a container with a stopped-node source lock; review Docker volume access and the helper image before using it around any valuable data. +Creation holds locks and runs a full financial audit; a large journal may exceed configured query/audit budgets and fail closed, so this needs measured long-history tests before production use. A database superuser/schema owner can bypass application advisory-lock discipline or rewrite records; external writer fencing remains mandatory. Hashes and archive parsing detect accidental damage, not a malicious coherent replacement; the manifest is unsigned. The `pg_dump` subprocess uses only a whitelisted connection environment, so inherited `PGHOSTADDR` or TLS overrides cannot redirect it away from the audited/writer-fenced endpoint. The PostgreSQL dump is not atomic with arbitrary unmanaged ledger or schema-owner activity. There is no WAL/PITR, remote scheduler, retention, restore orchestration, RPO/RTO claim, encryption, signature or production failover process in this increment. The local Compose backup service mounts the external host destination read-write into a container with a stopped-node source lock; review Docker volume access and the helper image before using it around any valuable data. ## Evidence and remaining hold points @@ -83,7 +83,7 @@ Automated tests cover default proxy distrust/spoofed-header rejection, explicit The native HTTP integration now cleanly stops its disposable nodes, backs up their **real three-block RandomX chain and market journal**, restores to a fresh disposable directory, opens/revalidates copied proofs with the actual reference engine, and checks canonical ownership, market instance, order buyer and nonce receipt history. It then restarts the original isolated fixture. This is a local automated drill, not an off-host operational recovery or crash/power-loss certification. -**Coordinated custody recovery evidence (2026-10-20):** the latest build passed 172 offline regressions and 31 isolated PostgreSQL integration tests. The new paired-backup test mines a valid native-RandomX source block, writes fictitious PostgreSQL custody state, freezes/binds the simulated SQLite cutover source, creates and verifies the pair, restores PostgreSQL into a scratch database, and restores/replays the node database into a new directory with the reference RandomX engine. It also verifies backup-login restrictions, mismatched identity/cutover detection, stale node-lock refusal, failed `pg_dump` cleanup/writer-lock release, output overwrite refusal, incomplete package handling and dump tamper detection. Tests use tmpfs/internal trust-auth databases and synthetic local files; this does not demonstrate protected persistent-volume provisioning, host-backed destination ACLs, off-host encryption or production operational recovery. The persistent sandbox was not restarted or migrated for this work. +**Coordinated custody recovery evidence (2026-10-20):** the latest build passed 172 offline regressions and 31 isolated PostgreSQL integration tests. The paired-backup test mines a valid native-RandomX source block, writes fictitious PostgreSQL custody state, freezes/binds the simulated SQLite cutover source, creates and verifies the pair, restores PostgreSQL into a scratch database, and restores/replays the node database into a new directory with the reference RandomX engine. It verifies backup-login restrictions, mismatched identity/cutover detection, stale node-lock refusal, failed `pg_dump` cleanup/writer-lock release, output overwrite refusal, incomplete package handling, exact directory contents, rejection of a valid unrelated empty-database archive and a corrupt dump even when its unsigned manifest hash is recomputed. A hostile inherited `PGHOSTADDR`/TLS environment also fails to redirect the dump child. Tests use tmpfs/internal trust-auth databases and synthetic local files; this does not demonstrate protected persistent-volume provisioning, host-backed destination ACLs, off-host encryption or production operational recovery. The persistent sandbox was not restarted or migrated for this work. Final Node 24 TypeScript/Vite and production runtime builds passed. The suite passed **98 tests, 0 failed, 0 skipped**. A separate packaged-runtime smoke drill ran with no external network, read-only root, dropped capabilities and a disposable 64 MiB tmpfs: monitoring correctly returned nonzero for isolation, empty chain, not-yet-reconciled market and low disk; clean stopped-node backup, verification and new-directory restore each exited zero. That empty-chain CLI drill reported proof revalidation as false, as designed; nonempty real-proof replay is covered by the integration drill above. No host volume or production wallet was used. diff --git a/docs/postgres-financial-storage.md b/docs/postgres-financial-storage.md index 6136fea..d4a4654 100755 --- a/docs/postgres-financial-storage.md +++ b/docs/postgres-financial-storage.md @@ -89,7 +89,7 @@ Recommended operations before any production release: - Reconcile Monero external history before resuming real spending once that integration exists. No local backup can reverse an external payment. Database timeouts/connection loss leave requests failed closed; retry the original signed command, not a new withdrawal. - Monitor application and database errors without logging statements/parameters containing private customer data in production. Fixture logs deliberately show failed constraint probes on fictitious data; do not adopt their logging/auth configuration. -The integration suite performs both a PostgreSQL `pg_dump`/isolated `pg_restore` rehearsal preserving ID, exact liabilities and receipt replay, and a paired node/PostgreSQL archive-creation/verification/restore test using the actual runtime adapter plus a read-only backup role. This does **not** certify managed backup schedules, PITR, encrypted off-host custody, signature/authenticity, power-loss/fsync behavior, long-history timing or production failover. +The integration suite performs both a PostgreSQL `pg_dump`/isolated `pg_restore` rehearsal preserving ID, exact liabilities and receipt replay, and a paired node/PostgreSQL archive-creation/verification/restore test using the actual runtime adapter plus a read-only backup role. Pair verification records PostgreSQL server/dump/restore versions, checks exact package entries and checksums, and parses the custom archive using `pg_restore --list` while requiring core custody tables and a compatible restore client. The dump subprocess is isolated from inherited libpq endpoint/TLS overrides. This does **not** certify managed backup schedules, PITR, encrypted off-host custody, signature/authenticity, power-loss/fsync behavior, long-history timing or production failover. ## Evidence and remaining gates diff --git a/scripts/custody-pair-backup.ts b/scripts/custody-pair-backup.ts index 3a4d6dd..67f6207 100755 --- a/scripts/custody-pair-backup.ts +++ b/scripts/custody-pair-backup.ts @@ -2,7 +2,7 @@ import { createCoordinatedCustodyBackup, verifyCoordinatedCustodyBackup } from ' const [action, ...args] = process.argv.slice(2); if (action !== 'create' && action !== 'verify') throw new Error('Expected create or verify'); -const allowed = new Set(['--source', '--destination', '--consent']); +const allowed = new Set(['--source', '--destination', '--consent', '--pg-restore-binary']); for (let index = 0; index < args.length; index++) { if (!allowed.has(args[index]) || (args[index] !== '--consent' && (!args[index + 1] || args[index + 1].startsWith('--')))) { throw new Error('Invalid coordinated backup arguments'); @@ -19,8 +19,8 @@ try { if (!destination || !args.includes('--consent')) throw new Error('Create requires --destination and explicit --consent'); result = await createCoordinatedCustodyBackup(source, destination); } else { - if (value('--destination') || args.includes('--consent')) throw new Error('Verify accepts only --source'); - result = await verifyCoordinatedCustodyBackup(source); + if (value('--destination') || args.includes('--consent')) throw new Error('Verify accepts --source and optional --pg-restore-binary'); + result = await verifyCoordinatedCustodyBackup(source, value('--pg-restore-binary')); } console.log(JSON.stringify(result, null, 2)); } catch { diff --git a/server/coordinated-backup.ts b/server/coordinated-backup.ts index 7bdf60e..9da7069 100755 --- a/server/coordinated-backup.ts +++ b/server/coordinated-backup.ts @@ -17,6 +17,8 @@ import { auditPostgres } from './market/postgres-custody.ts'; const pairManifestSchema = z.object({ format: z.literal('wikideck-coordinated-backup'), version: z.literal(1), network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID), profile: z.literal(PROFILE), createdAt: z.string().datetime(), custodianId: z.string().uuid(), database: z.string().regex(/^[a-zA-Z0-9_-]+$/), + postgresServerVersion: z.string().min(1).max(64), postgresServerVersionNum: z.string().regex(/^\d{5,6}$/), + pgDumpVersion: z.string().min(1).max(128), pgRestoreVersion: z.string().min(1).max(128), sqliteCutoverSourceDigest: z.string().regex(/^[a-f0-9]{64}$/).nullable(), nodeManifestSha256: z.string().regex(/^[a-f0-9]{64}$/), postgresDump: z.object({ name: z.literal('custody.dump'), bytes: z.number().int().positive().max(256 * 1024 ** 3), @@ -49,7 +51,11 @@ function childEnvironment(config: Awaited>, ca const tls = ssl === false ? 'disable' : 'verify-full'; const ca = typeof ssl === 'object' && ssl && 'ca' in ssl ? ssl.ca : undefined; if (ca && !caFile) throw new Error('Private PostgreSQL CA must be staged for the backup client'); - const env: NodeJS.ProcessEnv = { ...process.env, PGHOST: config.host, PGPORT: String(config.port), PGDATABASE: config.database, + // Whitelist child environment. Inherited PGHOSTADDR/SSL* variables could redirect libpq + // to a different endpoint than the node-pg advisory-lock connection we just audited. + const env: NodeJS.ProcessEnv = { PATH: process.env.PATH ?? '', + ...(process.platform === 'win32' ? { SYSTEMROOT: process.env.SYSTEMROOT ?? '', WINDIR: process.env.WINDIR ?? '' } : {}), + PGHOST: config.host, PGPORT: String(config.port), PGDATABASE: config.database, PGUSER: config.user, PGPASSWORD: String(config.password ?? ''), PGPASSFILE: passFile, PGCONNECT_TIMEOUT: '10', PGSERVICEFILE: serviceFile, PGSSLMODE: tls, ...(caFile ? { PGSSLROOTCERT: caFile } : {}) }; delete env.PGSERVICE; delete env.PGSSLCERT; delete env.PGSSLKEY; delete env.PGOPTIONS; @@ -88,6 +94,32 @@ async function dumpDatabase(config: Awaited>, }); } finally { await rm(temp, { recursive: true, force: true }); } } +async function toolVersion(binary: string): Promise { + return new Promise((resolveVersion, reject) => { + let stdout = ''; + const child: ChildProcess = spawn(binary, ['--version'], { shell: false, windowsHide: true, + stdio: ['ignore', 'pipe', 'ignore'], env: { PATH: process.env.PATH ?? '', ...(process.platform === 'win32' + ? { SYSTEMROOT: process.env.SYSTEMROOT ?? '', WINDIR: process.env.WINDIR ?? '' } : {}) } }); + child.stdout?.on('data', chunk => { + if (stdout.length + chunk.length > 1024) { child.kill(); return; } + stdout += chunk.toString('utf8'); + }); + const timeout = setTimeout(() => child.kill(), 10_000); timeout.unref(); + child.once('error', () => { clearTimeout(timeout); reject(new Error('PostgreSQL client utility failed to start')); }); + child.once('exit', code => { + clearTimeout(timeout); + const version = stdout.trim(); + if (code !== 0 || !/^pg_(?:dump|restore) \(PostgreSQL\) \d+(?:\.\d+){1,2}$/.test(version)) { + reject(new Error('PostgreSQL client utility returned an invalid version response')); + } else resolveVersion(version); + }); + }); +} +function pgMajor(version: string): number { + const match = /\(PostgreSQL\) (\d+)/.exec(version); + if (!match) throw new Error('Invalid PostgreSQL client version'); + return Number(match[1]); +} /** Stop-node + PostgreSQL writer-fenced snapshot. This sandbox tool never restores or resets a database. */ export async function createCoordinatedCustodyBackup(dataDirectory: string, destination: string, @@ -98,7 +130,11 @@ export async function createCoordinatedCustodyBackup(dataDirectory: string, dest const runtimeConfig = await postgresConfig(env); const backupConfig = await postgresConfig({ ...env, WIKIDECK_CUSTODY_PG_URL_FILE: backupUrlFile }); const dumpBinary = env.WIKIDECK_PG_DUMP_BINARY ?? 'pg_dump'; + const restoreBinary = env.WIKIDECK_PG_RESTORE_BINARY ?? 'pg_restore'; if (dumpBinary !== 'pg_dump' && !isAbsolute(dumpBinary)) throw new Error('Custom PostgreSQL dump binary must use an absolute path'); + if (restoreBinary !== 'pg_restore' && !isAbsolute(restoreBinary)) throw new Error('Custom PostgreSQL restore binary must use an absolute path'); + const [pgDumpVersion, pgRestoreVersion] = await Promise.all([toolVersion(dumpBinary), toolVersion(restoreBinary)]); + if (pgMajor(pgRestoreVersion) < pgMajor(pgDumpVersion)) throw new Error('PostgreSQL restore utility is older than the dump utility'); if (backupConfig.host !== runtimeConfig.host || backupConfig.port !== runtimeConfig.port || backupConfig.database !== runtimeConfig.database || backupConfig.user === runtimeConfig.user) { throw new Error('Backup database target must match custody, using a separate backup role'); @@ -126,6 +162,8 @@ export async function createCoordinatedCustodyBackup(dataDirectory: string, dest client = await pool.connect(); const actualDb = await client.query('SELECT current_database() AS name'); if (actualDb.rows[0].name !== backupConfig.database) throw new Error('Backup connected to an unexpected database'); + const serverVersion = await client.query(`SELECT current_setting('server_version') AS version, + current_setting('server_version_num')::text AS version_num`); await client.query('SELECT pg_advisory_lock($1::bigint)', [CUSTODY_WRITE_LOCK]); const identity = await client.query("SELECT value FROM wk_custody.custody_meta WHERE key='identity'"); if (!identity.rowCount) throw new Error('Custody PostgreSQL identity missing'); @@ -159,7 +197,9 @@ export async function createCoordinatedCustodyBackup(dataDirectory: string, dest await flush(dumpPath); const manifest = pairManifestSchema.parse({ format: 'wikideck-coordinated-backup', version: 1, network: NETWORK, chainId: CHAIN_ID, profile: PROFILE, createdAt: new Date().toISOString(), custodianId: binding.custodianId, - database: backupConfig.database, sqliteCutoverSourceDigest: imported.rows[0]?.value ?? null, + database: backupConfig.database, postgresServerVersion: serverVersion.rows[0].version, + postgresServerVersionNum: serverVersion.rows[0].version_num, pgDumpVersion, pgRestoreVersion, + sqliteCutoverSourceDigest: imported.rows[0]?.value ?? null, nodeManifestSha256: digest(JSON.stringify(nodeManifest)), postgresDump: { name: 'custody.dump', bytes: dumpInfo.size, sha256: await fileDigest(dumpPath) } }); await writeFile(join(target, 'manifest.json'), JSON.stringify(manifest, null, 2), { flag: 'wx', mode: 0o600 }); @@ -176,20 +216,66 @@ export async function createCoordinatedCustodyBackup(dataDirectory: string, dest }); } -export async function verifyCoordinatedCustodyBackup(directory: string): Promise { +async function verifyDumpArchive(path: string, binary: string): Promise { + if (binary !== 'pg_restore' && !isAbsolute(binary)) throw new Error('Custom PostgreSQL restore binary must use an absolute path'); + await new Promise((resolveArchive, reject) => { + let diagnostic = '', listing = '', listingTooLarge = false; + const child: ChildProcess = spawn(binary, ['--list', path], { + shell: false, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'], + env: { PATH: process.env.PATH ?? '', SYSTEMROOT: process.env.SYSTEMROOT ?? '', WINDIR: process.env.WINDIR ?? '' }, + }); + child.stdout?.on('data', chunk => { + if (listing.length + chunk.length > 2 * 1024 * 1024) { + listingTooLarge = true; child.kill(); return; + } + listing += chunk.toString('utf8'); + }); + child.stderr?.on('data', chunk => { diagnostic = (diagnostic + chunk.toString('utf8')).slice(-1024); }); + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; child.kill(); + const forceKill = setTimeout(() => child.kill('SIGKILL'), 5000); forceKill.unref(); + child.once('exit', () => clearTimeout(forceKill)); + }, 60_000); + timeout.unref(); + child.once('error', () => { clearTimeout(timeout); reject(new Error('PostgreSQL restore utility failed to start')); }); + child.once('exit', code => { + clearTimeout(timeout); + if (timedOut) reject(new Error('PostgreSQL archive verification exceeded its time budget')); + else if (listingTooLarge) reject(new Error('PostgreSQL dump archive listing exceeds verification budget')); + else if (code !== 0) reject(new Error(`PostgreSQL dump archive is invalid (${code ?? 'signal'}): ${diagnostic.replace(/[\r\n\x00-\x1f]/g, ' ').slice(0, 500)}`)); + else if (!['schema_versions', 'custody_meta', 'accounts', 'journals', 'entries'].every(name => + new RegExp(`\\bTABLE\\s+wk_custody\\s+${name}(?:\\s|$)`).test(listing))) { + reject(new Error('PostgreSQL dump archive is missing required custody accounting tables')); + } + else resolveArchive(); + }); + }); +} + +export async function verifyCoordinatedCustodyBackup(directory: string, + restoreBinary = process.env.WIKIDECK_PG_RESTORE_BINARY ?? 'pg_restore'): Promise { + if (restoreBinary !== 'pg_restore' && !isAbsolute(restoreBinary)) throw new Error('Custom PostgreSQL restore binary must use an absolute path'); const root = await realpath(directory); const manifestPath = join(root, 'manifest.json'), info = await regularFile(manifestPath); if (info.size > 16 * 1024) throw new Error('Coordinated backup manifest exceeds limit'); const manifest = pairManifestSchema.parse(JSON.parse(await readFile(manifestPath, 'utf8'))); + const restoreVersion = await toolVersion(restoreBinary); + if (pgMajor(restoreVersion) < pgMajor(manifest.pgDumpVersion)) throw new Error('Installed pg_restore is older than archive pg_dump; use a compatible maintenance image'); const nodePath = join(root, 'node'), nodeInfo = await lstat(nodePath); if (!nodeInfo.isDirectory() || nodeInfo.isSymbolicLink()) throw new Error('Node backup package must be a regular directory'); const nodeManifest = await verifyBackup(nodePath); if (digest(JSON.stringify(nodeManifest)) !== manifest.nodeManifestSha256 || nodeManifest.network !== manifest.network || nodeManifest.chainId !== manifest.chainId || nodeManifest.profile !== manifest.profile) throw new Error('Node snapshot does not match pair manifest'); + const expectedNodeEntries = [...nodeManifest.files.map(file => file.name), 'manifest.json'].sort(); + if ((await readdir(nodePath)).sort().join('|') !== expectedNodeEntries.join('|')) { + throw new Error('Unexpected coordinated node backup contents'); + } const dumpPath = join(root, manifest.postgresDump.name), dumpInfo = await regularFile(dumpPath); if (dumpInfo.size !== manifest.postgresDump.bytes || await fileDigest(dumpPath) !== manifest.postgresDump.sha256) { throw new Error('PostgreSQL dump hash/size mismatch'); } + await verifyDumpArchive(dumpPath, restoreBinary); const entries = (await readdir(root)).sort(); if (entries.join('|') !== ['custody.dump', 'manifest.json', 'node'].sort().join('|')) throw new Error('Unexpected coordinated backup contents'); return manifest; diff --git a/tests/postgres/custody.test.ts b/tests/postgres/custody.test.ts index 4b89f9e..2288539 100755 --- a/tests/postgres/custody.test.ts +++ b/tests/postgres/custody.test.ts @@ -1,8 +1,8 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { Pool, type PoolConfig } from 'pg'; -import { generateKeyPairSync, randomUUID, sign } from 'node:crypto'; -import { access, lstat, mkdtemp, rm, writeFile, mkdir, readFile } from 'node:fs/promises'; +import { createHash, generateKeyPairSync, randomUUID, sign } from 'node:crypto'; +import { access, chmod, lstat, mkdtemp, rm, writeFile, mkdir, readFile } from 'node:fs/promises'; import { constants } from 'node:fs'; import { join, resolve } from 'node:path'; import { tmpdir } from 'node:os'; @@ -414,10 +414,33 @@ test('coordinated node/PostgreSQL backup holds both writer fences and verifies m const env = { ...process.env, WIKIDECK_CUSTODY_PG_URL_FILE: runtimeUrl, WIKIDECK_CUSTODY_PG_BACKUP_URL_FILE: backupUrl, WIKIDECK_CUSTODY_PG_LOCAL_TEST: '1' }; const destination = join(dir, 'paired-backup'); - const manifest = await createCoordinatedCustodyBackup(data, destination, env); + const inheritedLibpq = { hostAddr: process.env.PGHOSTADDR, sslMode: process.env.PGSSLMODE, sslRoot: process.env.PGSSLROOTCERT }; + let manifest: Awaited>; + try { + process.env.PGHOSTADDR = '192.0.2.123'; process.env.PGSSLMODE = 'disable'; process.env.PGSSLROOTCERT = 'invalid-inherited-root.pem'; + manifest = await createCoordinatedCustodyBackup(data, destination, env); + } finally { + if (inheritedLibpq.hostAddr === undefined) delete process.env.PGHOSTADDR; else process.env.PGHOSTADDR = inheritedLibpq.hostAddr; + if (inheritedLibpq.sslMode === undefined) delete process.env.PGSSLMODE; else process.env.PGSSLMODE = inheritedLibpq.sslMode; + if (inheritedLibpq.sslRoot === undefined) delete process.env.PGSSLROOTCERT; else process.env.PGSSLROOTCERT = inheritedLibpq.sslRoot; + } assert.equal(manifest.custodianId, f.market.id); assert.equal(manifest.database, 'custody_test'); + assert.match(manifest.postgresServerVersion, /^\d+\.\d+/); + assert.match(manifest.postgresServerVersionNum, /^\d{5,6}$/); + assert.match(manifest.pgDumpVersion, /^pg_dump \(PostgreSQL\) /); + assert.match(manifest.pgRestoreVersion, /^pg_restore \(PostgreSQL\) /); assert.equal(manifest.sqliteCutoverSourceDigest, cutoverDigest); assert.equal((await verifyCoordinatedCustodyBackup(destination)).nodeManifestSha256, manifest.nodeManifestSha256); + if (process.platform !== 'win32') { + const oldRestore = join(dir, 'old-pg-restore'); + await writeFile(oldRestore, '#!/bin/sh\nprintf "pg_restore (PostgreSQL) 9.6\\n"\n', { mode: 0o700 }); + await chmod(oldRestore, 0o700); + await assert.rejects(verifyCoordinatedCustodyBackup(destination, oldRestore), /older than archive pg_dump/); + } + const extraNodeFile = join(destination, 'node', 'unmanifested.txt'); + await writeFile(extraNodeFile, 'not part of backup manifest'); + await assert.rejects(verifyCoordinatedCustodyBackup(destination), /Unexpected coordinated node backup contents/); + await rm(extraNodeFile); await assert.rejects(lstat(join(destination, 'node', 'custody-sandbox.sqlite')), { code: 'ENOENT' }); const restoredNodeDir = join(dir, 'restored-node'); await restoreBackup(join(destination, 'node'), restoredNodeDir); @@ -454,8 +477,8 @@ test('coordinated node/PostgreSQL backup holds both writer fences and verifies m await assert.rejects(lstat(lockedDestination)); await rm(staleLock); const failedDump = join(dir, 'failed-dump'), brokenEnv = { ...env, WIKIDECK_PG_DUMP_BINARY: join(dir, 'missing-pg-dump') }; - await assert.rejects(createCoordinatedCustodyBackup(data, failedDump, brokenEnv), /dump utility failed to start/); - await assert.rejects(verifyCoordinatedCustodyBackup(failedDump), /manifest/); + await assert.rejects(createCoordinatedCustodyBackup(data, failedDump, brokenEnv), /client utility failed to start/); + await assert.rejects(verifyCoordinatedCustodyBackup(failedDump), /ENOENT/); // A failed external dump releases PostgreSQL's shared-writer fence; normal sandbox commands still work. const recoveredWrite = await f.execute(other, { type: 'mock-credit' }) as CustodyActivity; assert.equal(recoveredWrite.availableAtomicXmr, '1000000000000'); @@ -465,11 +488,29 @@ test('coordinated node/PostgreSQL backup holds both writer fences and verifies m await writeFile(bindingPath, JSON.stringify({ version: 1, custodianId: randomUUID() }), { mode: 0o600 }); await assert.rejects(createCoordinatedCustodyBackup(data, mismatchedBinding, env), /identities differ/); await writeFile(bindingPath, originalBinding, { mode: 0o600 }); - await assert.rejects(verifyCoordinatedCustodyBackup(mismatchedBinding), /manifest/); + await assert.rejects(verifyCoordinatedCustodyBackup(mismatchedBinding), /ENOENT/); const corruptDump = join(destination, 'custody.dump'); await writeFile(corruptDump, 'corrupt'); await assert.rejects(verifyCoordinatedCustodyBackup(destination), /hash\/size mismatch/); + // Even if an operator or attacker rewrites the unsigned manifest hashes, pg_restore parsing rejects junk bytes. + const manifestPath = join(destination, 'manifest.json'), altered = JSON.parse(await readFile(manifestPath, 'utf8')); + altered.postgresDump.bytes = Buffer.byteLength('corrupt'); + altered.postgresDump.sha256 = createHash('sha256').update('corrupt').digest('hex'); + await writeFile(manifestPath, JSON.stringify(altered)); + await assert.rejects(verifyCoordinatedCustodyBackup(destination), /dump archive is invalid/); + const unrelatedDatabase = `empty_${randomUUID().replaceAll('-', '')}`, unrelatedDump = join(dir, 'unrelated.dump'); + await f.admin.query(`CREATE DATABASE "${unrelatedDatabase}"`); + try { + await run('pg_dump', ['-h', '127.0.0.1', '-U', 'postgres', '-d', unrelatedDatabase, '-Fc', '-f', unrelatedDump]); + await writeFile(corruptDump, await readFile(unrelatedDump)); + const validButWrong = JSON.parse(await readFile(manifestPath, 'utf8')); + const unrelated = await readFile(unrelatedDump); + validButWrong.postgresDump.bytes = unrelated.length; + validButWrong.postgresDump.sha256 = createHash('sha256').update(unrelated).digest('hex'); + await writeFile(manifestPath, JSON.stringify(validButWrong)); + await assert.rejects(verifyCoordinatedCustodyBackup(destination), /missing required custody accounting tables/); + } finally { await f.admin.query(`DROP DATABASE IF EXISTS "${unrelatedDatabase}" WITH (FORCE)`); } }); test('failed cutover freezes the source, rolls back the entire target, and resumes without duplicate balances', async t => {