283 lines
19 KiB
TypeScript
Executable File
283 lines
19 KiB
TypeScript
Executable File
import { createHash } from 'node:crypto';
|
|
import { spawn, type ChildProcess } from 'node:child_process';
|
|
import { createReadStream } from 'node:fs';
|
|
import { chmod, mkdir, lstat, readFile, readdir, writeFile, rm, open, realpath } from 'node:fs/promises';
|
|
import { dirname, join, resolve, relative, isAbsolute } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
import { Pool } from 'pg';
|
|
import { DatabaseSync } from 'node:sqlite';
|
|
import { z } from 'zod';
|
|
import { IS_MAINNET, NETWORK, PROFILE } from '../shared/network.ts';
|
|
import { CHAIN_ID } from './testnet/consensus.ts';
|
|
import { CUSTODY_WRITE_LOCK } from './market/postgres-schema.ts';
|
|
import { postgresConfig } from './market/postgres-config.ts';
|
|
import { withStoppedNodeLock, createNodeSnapshotUnlocked, verifyBackup } from './recovery.ts';
|
|
import { auditPostgres } from './market/postgres-custody.ts';
|
|
|
|
const pairManifestSchema = z.object({ format: z.literal('wikideck-coordinated-backup'), version: z.literal(1),
|
|
network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID), profile: z.literal(PROFILE),
|
|
createdAt: z.string().datetime(), custodianId: z.string().uuid(), database: z.string().regex(/^[a-zA-Z0-9_-]+$/),
|
|
postgresServerVersion: z.string().min(1).max(64), postgresServerVersionNum: z.string().regex(/^\d{5,6}$/),
|
|
pgDumpVersion: z.string().min(1).max(128), pgRestoreVersion: z.string().min(1).max(128),
|
|
sqliteCutoverSourceDigest: z.string().regex(/^[a-f0-9]{64}$/).nullable(),
|
|
nodeManifestSha256: z.string().regex(/^[a-f0-9]{64}$/),
|
|
postgresDump: z.object({ name: z.literal('custody.dump'), bytes: z.number().int().positive().max(256 * 1024 ** 3),
|
|
sha256: z.string().regex(/^[a-f0-9]{64}$/) }).strict() }).strict();
|
|
export type CoordinatedBackupManifest = z.infer<typeof pairManifestSchema>;
|
|
const bindingSchema = z.object({ version: z.literal(1), custodianId: z.string().uuid() }).strict();
|
|
function digest(data: Buffer | string): string { return createHash('sha256').update(data).digest('hex'); }
|
|
async function fileDigest(path: string): Promise<string> {
|
|
const hash = createHash('sha256');
|
|
for await (const chunk of createReadStream(path)) hash.update(chunk);
|
|
return hash.digest('hex');
|
|
}
|
|
async function flush(path: string): Promise<void> {
|
|
const handle = await open(path, 'r+'); try { await handle.sync(); } finally { await handle.close(); }
|
|
}
|
|
async function flushDirectory(path: string): Promise<void> {
|
|
if (process.platform === 'win32') return;
|
|
const handle = await open(path, 'r'); try { await handle.sync(); } finally { await handle.close(); }
|
|
}
|
|
async function regularFile(path: string): Promise<{ size: number }> {
|
|
const info = await lstat(path);
|
|
if (!info.isFile() || info.isSymbolicLink()) throw new Error('Coordinated backup file is not a regular non-symlink file');
|
|
return info;
|
|
}
|
|
function childEnvironment(config: Awaited<ReturnType<typeof postgresConfig>>, caFile: string | undefined,
|
|
serviceFile: string, passFile: string): NodeJS.ProcessEnv {
|
|
if (typeof config.host !== 'string' || !config.host || typeof config.database !== 'string' || !config.database ||
|
|
typeof config.user !== 'string' || !config.user) throw new Error('Incomplete PostgreSQL backup connection settings');
|
|
const ssl = config.ssl;
|
|
const tls = ssl === false ? 'disable' : 'verify-full';
|
|
const ca = typeof ssl === 'object' && ssl && 'ca' in ssl ? ssl.ca : undefined;
|
|
if (ca && !caFile) throw new Error('Private PostgreSQL CA must be staged for the backup client');
|
|
// Whitelist child environment. Inherited PGHOSTADDR/SSL* variables could redirect libpq
|
|
// to a different endpoint than the node-pg advisory-lock connection we just audited.
|
|
const env: NodeJS.ProcessEnv = { PATH: process.env.PATH ?? '',
|
|
...(process.platform === 'win32' ? { SYSTEMROOT: process.env.SYSTEMROOT ?? '', WINDIR: process.env.WINDIR ?? '' } : {}),
|
|
PGHOST: config.host, PGPORT: String(config.port), PGDATABASE: config.database,
|
|
PGUSER: config.user, PGPASSWORD: String(config.password ?? ''), PGPASSFILE: passFile, PGCONNECT_TIMEOUT: '10',
|
|
PGSERVICEFILE: serviceFile, PGSSLMODE: tls, ...(caFile ? { PGSSLROOTCERT: caFile } : {}) };
|
|
delete env.PGSERVICE; delete env.PGSSLCERT; delete env.PGSSLKEY; delete env.PGOPTIONS;
|
|
return env;
|
|
}
|
|
async function dumpDatabase(config: Awaited<ReturnType<typeof postgresConfig>>, output: string, binary: string): Promise<void> {
|
|
const temp = await import('node:fs/promises').then(m => m.mkdtemp(join(tmpdir(), 'wikideck-pg-backup-')));
|
|
let caFile: string | undefined;
|
|
try {
|
|
const ssl = config.ssl;
|
|
const ca = typeof ssl === 'object' && ssl && 'ca' in ssl ? ssl.ca : undefined;
|
|
if (typeof ca === 'string') { caFile = join(temp, 'root-ca.pem'); await writeFile(caFile, ca, { flag: 'wx', mode: 0o600 }); }
|
|
const serviceFile = join(temp, 'pg_service.conf');
|
|
await writeFile(serviceFile, '', { flag: 'wx', mode: 0o600 });
|
|
const passFile = join(temp, 'pgpass');
|
|
await writeFile(passFile, '', { flag: 'wx', mode: 0o600 });
|
|
const env = childEnvironment(config, caFile, serviceFile, passFile);
|
|
await new Promise<void>((resolveDump, reject) => {
|
|
let diagnostic = '';
|
|
const child: ChildProcess = spawn(binary, ['--no-password', '--format=custom', '--compress=6',
|
|
'--file', output, '--dbname', config.database!], { shell: false, windowsHide: true, stdio: ['ignore', 'ignore', 'pipe'], env });
|
|
child.stderr?.on('data', chunk => { diagnostic = (diagnostic + chunk.toString('utf8')).slice(-1024); });
|
|
let timedOut = false;
|
|
const timeout = setTimeout(() => {
|
|
timedOut = true; child.kill();
|
|
const forceKill = setTimeout(() => child.kill('SIGKILL'), 5000); forceKill.unref();
|
|
child.once('exit', () => clearTimeout(forceKill));
|
|
}, 30 * 60_000);
|
|
timeout.unref();
|
|
child.once('error', () => { clearTimeout(timeout); reject(new Error('PostgreSQL dump utility failed to start')); });
|
|
child.once('exit', code => {
|
|
clearTimeout(timeout);
|
|
if (timedOut) reject(new Error('PostgreSQL dump exceeded its time budget'));
|
|
else code === 0 ? resolveDump() : reject(new Error(`PostgreSQL dump failed (${code ?? 'signal'}): ${diagnostic.replace(/[\r\n\x00-\x1f]/g, ' ').slice(0, 500)}`));
|
|
});
|
|
});
|
|
} finally { await rm(temp, { recursive: true, force: true }); }
|
|
}
|
|
async function toolVersion(binary: string): Promise<string> {
|
|
return new Promise<string>((resolveVersion, reject) => {
|
|
let stdout = '';
|
|
const child: ChildProcess = spawn(binary, ['--version'], { shell: false, windowsHide: true,
|
|
stdio: ['ignore', 'pipe', 'ignore'], env: { PATH: process.env.PATH ?? '', ...(process.platform === 'win32'
|
|
? { SYSTEMROOT: process.env.SYSTEMROOT ?? '', WINDIR: process.env.WINDIR ?? '' } : {}) } });
|
|
child.stdout?.on('data', chunk => {
|
|
if (stdout.length + chunk.length > 1024) { child.kill(); return; }
|
|
stdout += chunk.toString('utf8');
|
|
});
|
|
const timeout = setTimeout(() => child.kill(), 10_000); timeout.unref();
|
|
child.once('error', () => { clearTimeout(timeout); reject(new Error('PostgreSQL client utility failed to start')); });
|
|
child.once('exit', code => {
|
|
clearTimeout(timeout);
|
|
const version = stdout.trim();
|
|
if (code !== 0 || !/^pg_(?:dump|restore) \(PostgreSQL\) \d+(?:\.\d+){1,2}$/.test(version)) {
|
|
reject(new Error('PostgreSQL client utility returned an invalid version response'));
|
|
} else resolveVersion(version);
|
|
});
|
|
});
|
|
}
|
|
function pgMajor(version: string): number {
|
|
const match = /\(PostgreSQL\) (\d+)/.exec(version);
|
|
if (!match) throw new Error('Invalid PostgreSQL client version');
|
|
return Number(match[1]);
|
|
}
|
|
|
|
/** Stop-node + PostgreSQL writer-fenced snapshot. This sandbox tool never restores or resets a database. */
|
|
export async function createCoordinatedCustodyBackup(dataDirectory: string, destination: string,
|
|
env: NodeJS.ProcessEnv = process.env): Promise<CoordinatedBackupManifest> {
|
|
if (IS_MAINNET) throw new Error('Coordinated custody backup utility is not approved for mainnet');
|
|
const backupUrlFile = env.WIKIDECK_CUSTODY_PG_BACKUP_URL_FILE;
|
|
if (!backupUrlFile) throw new Error('A separate PostgreSQL backup-role URL file is required');
|
|
const runtimeConfig = await postgresConfig(env);
|
|
const backupConfig = await postgresConfig({ ...env, WIKIDECK_CUSTODY_PG_URL_FILE: backupUrlFile });
|
|
const dumpBinary = env.WIKIDECK_PG_DUMP_BINARY ?? 'pg_dump';
|
|
const restoreBinary = env.WIKIDECK_PG_RESTORE_BINARY ?? 'pg_restore';
|
|
if (dumpBinary !== 'pg_dump' && !isAbsolute(dumpBinary)) throw new Error('Custom PostgreSQL dump binary must use an absolute path');
|
|
if (restoreBinary !== 'pg_restore' && !isAbsolute(restoreBinary)) throw new Error('Custom PostgreSQL restore binary must use an absolute path');
|
|
const [pgDumpVersion, pgRestoreVersion] = await Promise.all([toolVersion(dumpBinary), toolVersion(restoreBinary)]);
|
|
if (pgMajor(pgRestoreVersion) < pgMajor(pgDumpVersion)) throw new Error('PostgreSQL restore utility is older than the dump utility');
|
|
if (backupConfig.host !== runtimeConfig.host || backupConfig.port !== runtimeConfig.port ||
|
|
backupConfig.database !== runtimeConfig.database || backupConfig.user === runtimeConfig.user) {
|
|
throw new Error('Backup database target must match custody, using a separate backup role');
|
|
}
|
|
const source = await realpath(dataDirectory);
|
|
const target = resolve(destination), rel = relative(source, target);
|
|
if (!rel || (!rel.startsWith('..') && !isAbsolute(rel))) throw new Error('Backup destination must be outside node data directory');
|
|
const parentPath = dirname(target), parent = await realpath(parentPath);
|
|
if (process.platform === 'win32' ? parent.toLowerCase() !== parentPath.toLowerCase() : parent !== parentPath) {
|
|
throw new Error('Backup destination parent must not be symlinked');
|
|
}
|
|
try { await lstat(target); throw new Error('Backup destination already exists'); }
|
|
catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
|
|
|
|
return withStoppedNodeLock(source, async lockedSource => {
|
|
// Reserve the final name exclusively. The pair manifest is written last; partial folders never verify.
|
|
await mkdir(target, { mode: 0o700 });
|
|
const bindingPath = join(lockedSource, PROFILE, 'custody-postgres.json');
|
|
const bindingInfo = await regularFile(bindingPath);
|
|
if (bindingInfo.size > 4096) throw new Error('PostgreSQL data-directory binding exceeds limit');
|
|
const binding = bindingSchema.parse(JSON.parse(await readFile(bindingPath, 'utf8')));
|
|
const pool = new Pool({ ...backupConfig, max: 1, application_name: 'wikideck-coordinated-backup' });
|
|
let client;
|
|
try {
|
|
client = await pool.connect();
|
|
const actualDb = await client.query('SELECT current_database() AS name');
|
|
if (actualDb.rows[0].name !== backupConfig.database) throw new Error('Backup connected to an unexpected database');
|
|
const serverVersion = await client.query(`SELECT current_setting('server_version') AS version,
|
|
current_setting('server_version_num')::text AS version_num`);
|
|
await client.query('SELECT pg_advisory_lock($1::bigint)', [CUSTODY_WRITE_LOCK]);
|
|
const identity = await client.query("SELECT value FROM wk_custody.custody_meta WHERE key='identity'");
|
|
if (!identity.rowCount) throw new Error('Custody PostgreSQL identity missing');
|
|
const record = JSON.parse(identity.rows[0].value);
|
|
if (record.id !== binding.custodianId || record.version !== 1 || record.network !== NETWORK ||
|
|
record.chainId !== CHAIN_ID || record.mode !== 'sandbox') throw new Error('PostgreSQL and node custody identities differ');
|
|
const imported = await client.query("SELECT value FROM wk_custody.custody_meta WHERE key='import_source'");
|
|
const frozenSource = join(lockedSource, PROFILE, 'custody-sandbox.sqlite');
|
|
try {
|
|
const sourceInfo = await lstat(frozenSource);
|
|
if (!sourceInfo.isFile() || sourceInfo.isSymbolicLink() || !imported.rowCount) throw new Error('Frozen SQLite cutover source is not bound to this PostgreSQL database');
|
|
const sourceDb = new DatabaseSync(frozenSource, { readOnly: true });
|
|
try {
|
|
const marker = sourceDb.prepare("SELECT value FROM custody_meta WHERE key='postgres_cutover'").get() as { value: string } | undefined;
|
|
if (marker?.value !== imported.rows[0].value) throw new Error('Frozen SQLite source digest differs from PostgreSQL import binding');
|
|
} finally { sourceDb.close(); }
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
|
if (imported.rowCount) throw new Error('Imported PostgreSQL custody is missing its frozen SQLite source; operator review required');
|
|
}
|
|
await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY');
|
|
try { await auditPostgres(client); await client.query('COMMIT'); }
|
|
catch (error) { await client.query('ROLLBACK').catch(() => {}); throw error; }
|
|
|
|
const nodeManifest = await createNodeSnapshotUnlocked(lockedSource, join(target, 'node'), true);
|
|
const dumpPath = join(target, 'custody.dump');
|
|
await dumpDatabase(backupConfig, dumpPath, dumpBinary);
|
|
const dumpInfo = await regularFile(dumpPath);
|
|
if (dumpInfo.size < 1 || dumpInfo.size > 256 * 1024 ** 3) throw new Error('PostgreSQL dump size outside supported bounds');
|
|
if (process.platform !== 'win32') await chmod(dumpPath, 0o600);
|
|
await flush(dumpPath);
|
|
const manifest = pairManifestSchema.parse({ format: 'wikideck-coordinated-backup', version: 1, network: NETWORK,
|
|
chainId: CHAIN_ID, profile: PROFILE, createdAt: new Date().toISOString(), custodianId: binding.custodianId,
|
|
database: backupConfig.database, postgresServerVersion: serverVersion.rows[0].version,
|
|
postgresServerVersionNum: serverVersion.rows[0].version_num, pgDumpVersion, pgRestoreVersion,
|
|
sqliteCutoverSourceDigest: imported.rows[0]?.value ?? null,
|
|
nodeManifestSha256: digest(JSON.stringify(nodeManifest)),
|
|
postgresDump: { name: 'custody.dump', bytes: dumpInfo.size, sha256: await fileDigest(dumpPath) } });
|
|
await writeFile(join(target, 'manifest.json'), JSON.stringify(manifest, null, 2), { flag: 'wx', mode: 0o600 });
|
|
if (process.platform !== 'win32') await chmod(join(target, 'manifest.json'), 0o600);
|
|
await flush(join(target, 'manifest.json')); await flushDirectory(target); await flushDirectory(parent);
|
|
return manifest;
|
|
} finally {
|
|
if (client) {
|
|
await client.query('SELECT pg_advisory_unlock($1::bigint)', [CUSTODY_WRITE_LOCK]).catch(() => {});
|
|
client.release();
|
|
}
|
|
await pool.end();
|
|
}
|
|
});
|
|
}
|
|
|
|
async function verifyDumpArchive(path: string, binary: string): Promise<void> {
|
|
if (binary !== 'pg_restore' && !isAbsolute(binary)) throw new Error('Custom PostgreSQL restore binary must use an absolute path');
|
|
await new Promise<void>((resolveArchive, reject) => {
|
|
let diagnostic = '', listing = '', listingTooLarge = false;
|
|
const child: ChildProcess = spawn(binary, ['--list', path], {
|
|
shell: false, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'],
|
|
env: { PATH: process.env.PATH ?? '', SYSTEMROOT: process.env.SYSTEMROOT ?? '', WINDIR: process.env.WINDIR ?? '' },
|
|
});
|
|
child.stdout?.on('data', chunk => {
|
|
if (listing.length + chunk.length > 2 * 1024 * 1024) {
|
|
listingTooLarge = true; child.kill(); return;
|
|
}
|
|
listing += chunk.toString('utf8');
|
|
});
|
|
child.stderr?.on('data', chunk => { diagnostic = (diagnostic + chunk.toString('utf8')).slice(-1024); });
|
|
let timedOut = false;
|
|
const timeout = setTimeout(() => {
|
|
timedOut = true; child.kill();
|
|
const forceKill = setTimeout(() => child.kill('SIGKILL'), 5000); forceKill.unref();
|
|
child.once('exit', () => clearTimeout(forceKill));
|
|
}, 60_000);
|
|
timeout.unref();
|
|
child.once('error', () => { clearTimeout(timeout); reject(new Error('PostgreSQL restore utility failed to start')); });
|
|
child.once('exit', code => {
|
|
clearTimeout(timeout);
|
|
if (timedOut) reject(new Error('PostgreSQL archive verification exceeded its time budget'));
|
|
else if (listingTooLarge) reject(new Error('PostgreSQL dump archive listing exceeds verification budget'));
|
|
else if (code !== 0) reject(new Error(`PostgreSQL dump archive is invalid (${code ?? 'signal'}): ${diagnostic.replace(/[\r\n\x00-\x1f]/g, ' ').slice(0, 500)}`));
|
|
else if (!['schema_versions', 'custody_meta', 'accounts', 'journals', 'entries'].every(name =>
|
|
new RegExp(`\\bTABLE\\s+wk_custody\\s+${name}(?:\\s|$)`).test(listing))) {
|
|
reject(new Error('PostgreSQL dump archive is missing required custody accounting tables'));
|
|
}
|
|
else resolveArchive();
|
|
});
|
|
});
|
|
}
|
|
|
|
export async function verifyCoordinatedCustodyBackup(directory: string,
|
|
restoreBinary = process.env.WIKIDECK_PG_RESTORE_BINARY ?? 'pg_restore'): Promise<CoordinatedBackupManifest> {
|
|
if (restoreBinary !== 'pg_restore' && !isAbsolute(restoreBinary)) throw new Error('Custom PostgreSQL restore binary must use an absolute path');
|
|
const root = await realpath(directory);
|
|
const manifestPath = join(root, 'manifest.json'), info = await regularFile(manifestPath);
|
|
if (info.size > 16 * 1024) throw new Error('Coordinated backup manifest exceeds limit');
|
|
const manifest = pairManifestSchema.parse(JSON.parse(await readFile(manifestPath, 'utf8')));
|
|
const restoreVersion = await toolVersion(restoreBinary);
|
|
if (pgMajor(restoreVersion) < pgMajor(manifest.pgDumpVersion)) throw new Error('Installed pg_restore is older than archive pg_dump; use a compatible maintenance image');
|
|
const nodePath = join(root, 'node'), nodeInfo = await lstat(nodePath);
|
|
if (!nodeInfo.isDirectory() || nodeInfo.isSymbolicLink()) throw new Error('Node backup package must be a regular directory');
|
|
const nodeManifest = await verifyBackup(nodePath);
|
|
if (digest(JSON.stringify(nodeManifest)) !== manifest.nodeManifestSha256 || nodeManifest.network !== manifest.network ||
|
|
nodeManifest.chainId !== manifest.chainId || nodeManifest.profile !== manifest.profile) throw new Error('Node snapshot does not match pair manifest');
|
|
const expectedNodeEntries = [...nodeManifest.files.map(file => file.name), 'manifest.json'].sort();
|
|
if ((await readdir(nodePath)).sort().join('|') !== expectedNodeEntries.join('|')) {
|
|
throw new Error('Unexpected coordinated node backup contents');
|
|
}
|
|
const dumpPath = join(root, manifest.postgresDump.name), dumpInfo = await regularFile(dumpPath);
|
|
if (dumpInfo.size !== manifest.postgresDump.bytes || await fileDigest(dumpPath) !== manifest.postgresDump.sha256) {
|
|
throw new Error('PostgreSQL dump hash/size mismatch');
|
|
}
|
|
await verifyDumpArchive(dumpPath, restoreBinary);
|
|
const entries = (await readdir(root)).sort();
|
|
if (entries.join('|') !== ['custody.dump', 'manifest.json', 'node'].sort().join('|')) throw new Error('Unexpected coordinated backup contents');
|
|
return manifest;
|
|
}
|