197 lines
13 KiB
TypeScript
Executable File
197 lines
13 KiB
TypeScript
Executable File
import { createHash } from 'node:crypto';
|
|
import { spawn, type ChildProcess } from 'node:child_process';
|
|
import { createReadStream } from 'node:fs';
|
|
import { chmod, mkdir, lstat, readFile, readdir, writeFile, rm, open, realpath } from 'node:fs/promises';
|
|
import { dirname, join, resolve, relative, isAbsolute } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
import { Pool } from 'pg';
|
|
import { DatabaseSync } from 'node:sqlite';
|
|
import { z } from 'zod';
|
|
import { IS_MAINNET, NETWORK, PROFILE } from '../shared/network.ts';
|
|
import { CHAIN_ID } from './testnet/consensus.ts';
|
|
import { CUSTODY_WRITE_LOCK } from './market/postgres-schema.ts';
|
|
import { postgresConfig } from './market/postgres-config.ts';
|
|
import { withStoppedNodeLock, createNodeSnapshotUnlocked, verifyBackup } from './recovery.ts';
|
|
import { auditPostgres } from './market/postgres-custody.ts';
|
|
|
|
const pairManifestSchema = z.object({ format: z.literal('wikideck-coordinated-backup'), version: z.literal(1),
|
|
network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID), profile: z.literal(PROFILE),
|
|
createdAt: z.string().datetime(), custodianId: z.string().uuid(), database: z.string().regex(/^[a-zA-Z0-9_-]+$/),
|
|
sqliteCutoverSourceDigest: z.string().regex(/^[a-f0-9]{64}$/).nullable(),
|
|
nodeManifestSha256: z.string().regex(/^[a-f0-9]{64}$/),
|
|
postgresDump: z.object({ name: z.literal('custody.dump'), bytes: z.number().int().positive().max(256 * 1024 ** 3),
|
|
sha256: z.string().regex(/^[a-f0-9]{64}$/) }).strict() }).strict();
|
|
export type CoordinatedBackupManifest = z.infer<typeof pairManifestSchema>;
|
|
const bindingSchema = z.object({ version: z.literal(1), custodianId: z.string().uuid() }).strict();
|
|
function digest(data: Buffer | string): string { return createHash('sha256').update(data).digest('hex'); }
|
|
async function fileDigest(path: string): Promise<string> {
|
|
const hash = createHash('sha256');
|
|
for await (const chunk of createReadStream(path)) hash.update(chunk);
|
|
return hash.digest('hex');
|
|
}
|
|
async function flush(path: string): Promise<void> {
|
|
const handle = await open(path, 'r+'); try { await handle.sync(); } finally { await handle.close(); }
|
|
}
|
|
async function flushDirectory(path: string): Promise<void> {
|
|
if (process.platform === 'win32') return;
|
|
const handle = await open(path, 'r'); try { await handle.sync(); } finally { await handle.close(); }
|
|
}
|
|
async function regularFile(path: string): Promise<{ size: number }> {
|
|
const info = await lstat(path);
|
|
if (!info.isFile() || info.isSymbolicLink()) throw new Error('Coordinated backup file is not a regular non-symlink file');
|
|
return info;
|
|
}
|
|
function childEnvironment(config: Awaited<ReturnType<typeof postgresConfig>>, caFile: string | undefined,
|
|
serviceFile: string, passFile: string): NodeJS.ProcessEnv {
|
|
if (typeof config.host !== 'string' || !config.host || typeof config.database !== 'string' || !config.database ||
|
|
typeof config.user !== 'string' || !config.user) throw new Error('Incomplete PostgreSQL backup connection settings');
|
|
const ssl = config.ssl;
|
|
const tls = ssl === false ? 'disable' : 'verify-full';
|
|
const ca = typeof ssl === 'object' && ssl && 'ca' in ssl ? ssl.ca : undefined;
|
|
if (ca && !caFile) throw new Error('Private PostgreSQL CA must be staged for the backup client');
|
|
const env: NodeJS.ProcessEnv = { ...process.env, PGHOST: config.host, PGPORT: String(config.port), PGDATABASE: config.database,
|
|
PGUSER: config.user, PGPASSWORD: String(config.password ?? ''), PGPASSFILE: passFile, PGCONNECT_TIMEOUT: '10',
|
|
PGSERVICEFILE: serviceFile, PGSSLMODE: tls, ...(caFile ? { PGSSLROOTCERT: caFile } : {}) };
|
|
delete env.PGSERVICE; delete env.PGSSLCERT; delete env.PGSSLKEY; delete env.PGOPTIONS;
|
|
return env;
|
|
}
|
|
async function dumpDatabase(config: Awaited<ReturnType<typeof postgresConfig>>, output: string, binary: string): Promise<void> {
|
|
const temp = await import('node:fs/promises').then(m => m.mkdtemp(join(tmpdir(), 'wikideck-pg-backup-')));
|
|
let caFile: string | undefined;
|
|
try {
|
|
const ssl = config.ssl;
|
|
const ca = typeof ssl === 'object' && ssl && 'ca' in ssl ? ssl.ca : undefined;
|
|
if (typeof ca === 'string') { caFile = join(temp, 'root-ca.pem'); await writeFile(caFile, ca, { flag: 'wx', mode: 0o600 }); }
|
|
const serviceFile = join(temp, 'pg_service.conf');
|
|
await writeFile(serviceFile, '', { flag: 'wx', mode: 0o600 });
|
|
const passFile = join(temp, 'pgpass');
|
|
await writeFile(passFile, '', { flag: 'wx', mode: 0o600 });
|
|
const env = childEnvironment(config, caFile, serviceFile, passFile);
|
|
await new Promise<void>((resolveDump, reject) => {
|
|
let diagnostic = '';
|
|
const child: ChildProcess = spawn(binary, ['--no-password', '--format=custom', '--compress=6',
|
|
'--file', output, '--dbname', config.database!], { shell: false, windowsHide: true, stdio: ['ignore', 'ignore', 'pipe'], env });
|
|
child.stderr?.on('data', chunk => { diagnostic = (diagnostic + chunk.toString('utf8')).slice(-1024); });
|
|
let timedOut = false;
|
|
const timeout = setTimeout(() => {
|
|
timedOut = true; child.kill();
|
|
const forceKill = setTimeout(() => child.kill('SIGKILL'), 5000); forceKill.unref();
|
|
child.once('exit', () => clearTimeout(forceKill));
|
|
}, 30 * 60_000);
|
|
timeout.unref();
|
|
child.once('error', () => { clearTimeout(timeout); reject(new Error('PostgreSQL dump utility failed to start')); });
|
|
child.once('exit', code => {
|
|
clearTimeout(timeout);
|
|
if (timedOut) reject(new Error('PostgreSQL dump exceeded its time budget'));
|
|
else code === 0 ? resolveDump() : reject(new Error(`PostgreSQL dump failed (${code ?? 'signal'}): ${diagnostic.replace(/[\r\n\x00-\x1f]/g, ' ').slice(0, 500)}`));
|
|
});
|
|
});
|
|
} finally { await rm(temp, { recursive: true, force: true }); }
|
|
}
|
|
|
|
/** Stop-node + PostgreSQL writer-fenced snapshot. This sandbox tool never restores or resets a database. */
|
|
export async function createCoordinatedCustodyBackup(dataDirectory: string, destination: string,
|
|
env: NodeJS.ProcessEnv = process.env): Promise<CoordinatedBackupManifest> {
|
|
if (IS_MAINNET) throw new Error('Coordinated custody backup utility is not approved for mainnet');
|
|
const backupUrlFile = env.WIKIDECK_CUSTODY_PG_BACKUP_URL_FILE;
|
|
if (!backupUrlFile) throw new Error('A separate PostgreSQL backup-role URL file is required');
|
|
const runtimeConfig = await postgresConfig(env);
|
|
const backupConfig = await postgresConfig({ ...env, WIKIDECK_CUSTODY_PG_URL_FILE: backupUrlFile });
|
|
const dumpBinary = env.WIKIDECK_PG_DUMP_BINARY ?? 'pg_dump';
|
|
if (dumpBinary !== 'pg_dump' && !isAbsolute(dumpBinary)) throw new Error('Custom PostgreSQL dump binary must use an absolute path');
|
|
if (backupConfig.host !== runtimeConfig.host || backupConfig.port !== runtimeConfig.port ||
|
|
backupConfig.database !== runtimeConfig.database || backupConfig.user === runtimeConfig.user) {
|
|
throw new Error('Backup database target must match custody, using a separate backup role');
|
|
}
|
|
const source = await realpath(dataDirectory);
|
|
const target = resolve(destination), rel = relative(source, target);
|
|
if (!rel || (!rel.startsWith('..') && !isAbsolute(rel))) throw new Error('Backup destination must be outside node data directory');
|
|
const parentPath = dirname(target), parent = await realpath(parentPath);
|
|
if (process.platform === 'win32' ? parent.toLowerCase() !== parentPath.toLowerCase() : parent !== parentPath) {
|
|
throw new Error('Backup destination parent must not be symlinked');
|
|
}
|
|
try { await lstat(target); throw new Error('Backup destination already exists'); }
|
|
catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
|
|
|
|
return withStoppedNodeLock(source, async lockedSource => {
|
|
// Reserve the final name exclusively. The pair manifest is written last; partial folders never verify.
|
|
await mkdir(target, { mode: 0o700 });
|
|
const bindingPath = join(lockedSource, PROFILE, 'custody-postgres.json');
|
|
const bindingInfo = await regularFile(bindingPath);
|
|
if (bindingInfo.size > 4096) throw new Error('PostgreSQL data-directory binding exceeds limit');
|
|
const binding = bindingSchema.parse(JSON.parse(await readFile(bindingPath, 'utf8')));
|
|
const pool = new Pool({ ...backupConfig, max: 1, application_name: 'wikideck-coordinated-backup' });
|
|
let client;
|
|
try {
|
|
client = await pool.connect();
|
|
const actualDb = await client.query('SELECT current_database() AS name');
|
|
if (actualDb.rows[0].name !== backupConfig.database) throw new Error('Backup connected to an unexpected database');
|
|
await client.query('SELECT pg_advisory_lock($1::bigint)', [CUSTODY_WRITE_LOCK]);
|
|
const identity = await client.query("SELECT value FROM wk_custody.custody_meta WHERE key='identity'");
|
|
if (!identity.rowCount) throw new Error('Custody PostgreSQL identity missing');
|
|
const record = JSON.parse(identity.rows[0].value);
|
|
if (record.id !== binding.custodianId || record.version !== 1 || record.network !== NETWORK ||
|
|
record.chainId !== CHAIN_ID || record.mode !== 'sandbox') throw new Error('PostgreSQL and node custody identities differ');
|
|
const imported = await client.query("SELECT value FROM wk_custody.custody_meta WHERE key='import_source'");
|
|
const frozenSource = join(lockedSource, PROFILE, 'custody-sandbox.sqlite');
|
|
try {
|
|
const sourceInfo = await lstat(frozenSource);
|
|
if (!sourceInfo.isFile() || sourceInfo.isSymbolicLink() || !imported.rowCount) throw new Error('Frozen SQLite cutover source is not bound to this PostgreSQL database');
|
|
const sourceDb = new DatabaseSync(frozenSource, { readOnly: true });
|
|
try {
|
|
const marker = sourceDb.prepare("SELECT value FROM custody_meta WHERE key='postgres_cutover'").get() as { value: string } | undefined;
|
|
if (marker?.value !== imported.rows[0].value) throw new Error('Frozen SQLite source digest differs from PostgreSQL import binding');
|
|
} finally { sourceDb.close(); }
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
|
if (imported.rowCount) throw new Error('Imported PostgreSQL custody is missing its frozen SQLite source; operator review required');
|
|
}
|
|
await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY');
|
|
try { await auditPostgres(client); await client.query('COMMIT'); }
|
|
catch (error) { await client.query('ROLLBACK').catch(() => {}); throw error; }
|
|
|
|
const nodeManifest = await createNodeSnapshotUnlocked(lockedSource, join(target, 'node'), true);
|
|
const dumpPath = join(target, 'custody.dump');
|
|
await dumpDatabase(backupConfig, dumpPath, dumpBinary);
|
|
const dumpInfo = await regularFile(dumpPath);
|
|
if (dumpInfo.size < 1 || dumpInfo.size > 256 * 1024 ** 3) throw new Error('PostgreSQL dump size outside supported bounds');
|
|
if (process.platform !== 'win32') await chmod(dumpPath, 0o600);
|
|
await flush(dumpPath);
|
|
const manifest = pairManifestSchema.parse({ format: 'wikideck-coordinated-backup', version: 1, network: NETWORK,
|
|
chainId: CHAIN_ID, profile: PROFILE, createdAt: new Date().toISOString(), custodianId: binding.custodianId,
|
|
database: backupConfig.database, sqliteCutoverSourceDigest: imported.rows[0]?.value ?? null,
|
|
nodeManifestSha256: digest(JSON.stringify(nodeManifest)),
|
|
postgresDump: { name: 'custody.dump', bytes: dumpInfo.size, sha256: await fileDigest(dumpPath) } });
|
|
await writeFile(join(target, 'manifest.json'), JSON.stringify(manifest, null, 2), { flag: 'wx', mode: 0o600 });
|
|
if (process.platform !== 'win32') await chmod(join(target, 'manifest.json'), 0o600);
|
|
await flush(join(target, 'manifest.json')); await flushDirectory(target); await flushDirectory(parent);
|
|
return manifest;
|
|
} finally {
|
|
if (client) {
|
|
await client.query('SELECT pg_advisory_unlock($1::bigint)', [CUSTODY_WRITE_LOCK]).catch(() => {});
|
|
client.release();
|
|
}
|
|
await pool.end();
|
|
}
|
|
});
|
|
}
|
|
|
|
export async function verifyCoordinatedCustodyBackup(directory: string): Promise<CoordinatedBackupManifest> {
|
|
const root = await realpath(directory);
|
|
const manifestPath = join(root, 'manifest.json'), info = await regularFile(manifestPath);
|
|
if (info.size > 16 * 1024) throw new Error('Coordinated backup manifest exceeds limit');
|
|
const manifest = pairManifestSchema.parse(JSON.parse(await readFile(manifestPath, 'utf8')));
|
|
const nodePath = join(root, 'node'), nodeInfo = await lstat(nodePath);
|
|
if (!nodeInfo.isDirectory() || nodeInfo.isSymbolicLink()) throw new Error('Node backup package must be a regular directory');
|
|
const nodeManifest = await verifyBackup(nodePath);
|
|
if (digest(JSON.stringify(nodeManifest)) !== manifest.nodeManifestSha256 || nodeManifest.network !== manifest.network ||
|
|
nodeManifest.chainId !== manifest.chainId || nodeManifest.profile !== manifest.profile) throw new Error('Node snapshot does not match pair manifest');
|
|
const dumpPath = join(root, manifest.postgresDump.name), dumpInfo = await regularFile(dumpPath);
|
|
if (dumpInfo.size !== manifest.postgresDump.bytes || await fileDigest(dumpPath) !== manifest.postgresDump.sha256) {
|
|
throw new Error('PostgreSQL dump hash/size mismatch');
|
|
}
|
|
const entries = (await readdir(root)).sort();
|
|
if (entries.join('|') !== ['custody.dump', 'manifest.json', 'node'].sort().join('|')) throw new Error('Unexpected coordinated backup contents');
|
|
return manifest;
|
|
}
|