diff --git a/Dockerfile b/Dockerfile index 05ff6c3..bd8cd21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,3 +53,11 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ CMD ["node", "-e", "fetch('http://127.0.0.1:3001/healthz', {signal: AbortSignal.timeout(4000)}).then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))"] # Node 24 strips the server's erasable TypeScript; no tsx/dev dependencies at runtime. CMD ["node", "--experimental-strip-types", "server/index.ts"] + +# Explicit one-shot maintenance image, derived from runtime so it has no development dependencies. +FROM runtime AS custody-backup +USER root +RUN apk add --no-cache postgresql-client +COPY scripts/custody-pair-backup.ts ./scripts/custody-pair-backup.ts +USER node +ENTRYPOINT ["node", "--experimental-strip-types", "scripts/custody-pair-backup.ts"] diff --git a/compose.custody-backup.yaml b/compose.custody-backup.yaml new file mode 100755 index 0000000..9edb56d --- /dev/null +++ b/compose.custody-backup.yaml @@ -0,0 +1,47 @@ +# Explicit local sandbox backup helper. Start the persistent stack first, then stop only wikideck. +# Requires WIKIDECK_BACKUP_DIR (existing restricted host directory) and WIKIDECK_BACKUP_NAME (new leaf name). +services: + backup: + image: wikideck:custody-backup + build: + context: . + dockerfile: Dockerfile + target: custody-backup + environment: + WIKIDECK_NETWORK: testnet + WIKIDECK_CUSTODY_STORAGE: postgres + WIKIDECK_CUSTODY_PG_URL_FILE: /secrets/runtime/url + WIKIDECK_CUSTODY_PG_BACKUP_URL_FILE: /secrets/backup/url + WIKIDECK_CUSTODY_PG_CA_FILE: /secrets/backup/ca.crt + command: ["create", "--source", "/app/data", "--destination", "/backups/${WIKIDECK_BACKUP_NAME:?Set a new backup leaf name}", "--consent"] + volumes: + - custody-node:/app/data + - pg-runtime:/secrets/runtime:ro + - pg-backup:/secrets/backup:ro + - type: bind + source: ${WIKIDECK_BACKUP_DIR:?Set an existing restricted host backup directory} + target: /backups + bind: + create_host_path: false + networks: [financial] + read_only: true + tmpfs: ["/tmp:mode=1777,size=32m,noexec,nosuid"] + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + pids_limit: 32 + mem_limit: 512m + restart: "no" +networks: + financial: + external: true + name: wikideck-custody-local_financial +volumes: + custody-node: + external: true + name: wikideck-custody-local_custody-node + pg-runtime: + external: true + name: wikideck-custody-local_pg-runtime + pg-backup: + external: true + name: wikideck-custody-local_pg-backup diff --git a/compose.custody.yaml b/compose.custody.yaml index 7de33be..6cac6c0 100755 --- a/compose.custody.yaml +++ b/compose.custody.yaml @@ -9,6 +9,7 @@ services: - pg-admin:/secrets/admin - pg-owner:/secrets/owner - pg-runtime:/secrets/runtime + - pg-backup:/secrets/backup - pg-tls:/secrets/tls network_mode: none read_only: true @@ -58,6 +59,7 @@ services: - pg-admin:/secrets/admin:ro - pg-owner:/secrets/owner:ro - pg-runtime:/secrets/runtime:ro + - pg-backup:/secrets/backup:ro networks: [financial] read_only: true cap_drop: [ALL] @@ -102,5 +104,6 @@ volumes: pg-admin: pg-owner: pg-runtime: + pg-backup: pg-tls: custody-node: diff --git a/deploy/postgres/pg_hba.conf b/deploy/postgres/pg_hba.conf index 4ada7e9..44575d9 100755 --- a/deploy/postgres/pg_hba.conf +++ b/deploy/postgres/pg_hba.conf @@ -4,4 +4,5 @@ hostnossl all all all reject hostssl wikideck_custody postgres all scram-sha-256 hostssl wikideck_custody custody_owner all scram-sha-256 hostssl wikideck_custody custody_runtime all scram-sha-256 +hostssl wikideck_custody custody_backup all scram-sha-256 host all all all reject diff --git a/deploy/postgres/setup.mjs b/deploy/postgres/setup.mjs index 2af9141..78d5da4 100755 --- a/deploy/postgres/setup.mjs +++ b/deploy/postgres/setup.mjs @@ -1,32 +1,49 @@ import { randomBytes } from 'node:crypto'; import { execFileSync } from 'node:child_process'; -import { mkdirSync, writeFileSync, readFileSync, existsSync, chmodSync, chownSync, openSync, fsyncSync, closeSync } from 'node:fs'; +import { mkdirSync, writeFileSync, readFileSync, existsSync, chmodSync, chownSync, openSync, fsyncSync, closeSync, renameSync } from 'node:fs'; // Local SANDBOX provisioning only. No secrets pass through logs, command arguments or source control. -const roots = ['/secrets/admin', '/secrets/owner', '/secrets/runtime', '/secrets/tls']; +const roots = ['/secrets/admin', '/secrets/owner', '/secrets/runtime', '/secrets/backup', '/secrets/tls']; for (const root of roots) mkdirSync(root, { recursive: true, mode: 0o755 }); const files = ['/secrets/admin/password', '/secrets/admin/url', '/secrets/owner/url', '/secrets/runtime/url', '/secrets/tls/server.key', '/secrets/tls/server.crt', '/secrets/owner/ca.crt', '/secrets/runtime/ca.crt', '/secrets/admin/ca.crt']; +const backupFiles = ['/secrets/backup/url', '/secrets/backup/ca.crt']; const marker = '/secrets/admin/complete'; +const put = (file, text, mode = 0o600) => { + writeFileSync(file, text, { flag: 'wx', mode }); chownSync(file, 1000, 1000); + const fd = openSync(file, 'r'); try { fsyncSync(fd); } finally { closeSync(fd); } +}; +const provisionBackupRole = () => { + if (backupFiles.some(existsSync)) throw new Error('Partial backup-role secret initialization; preserve volumes for inspection'); + const backup = randomBytes(32).toString('hex'); + put('/secrets/backup/url', `postgresql://custody_backup:${backup}@database:5432/wikideck_custody`); + put('/secrets/backup/ca.crt', readFileSync('/secrets/admin/ca.crt'), 0o644); +}; if (existsSync(marker)) { - if (readFileSync(marker, 'utf8') !== 'wikideck-local-pg-v1' || files.some(f => !existsSync(f))) { + const version = readFileSync(marker, 'utf8'); + if (!['wikideck-local-pg-v1', 'wikideck-local-pg-v2'].includes(version) || files.some(f => !existsSync(f))) { throw new Error('Incomplete secret volumes; restore the matching volumes, never regenerate over an existing database'); } execFileSync('openssl', ['x509', '-checkend', '86400', '-noout', '-in', '/secrets/tls/server.crt'], { stdio: 'ignore' }); - console.log('Existing local sandbox credentials/certificate retained.'); + if (version === 'wikideck-local-pg-v1') { + provisionBackupRole(); + const tempMarker = `${marker}.upgrade`; + writeFileSync(tempMarker, 'wikideck-local-pg-v2', { flag: 'wx', mode: 0o644 }); + renameSync(tempMarker, marker); + } else if (backupFiles.some(f => !existsSync(f))) { + throw new Error('Backup-role secret volumes are incomplete; preserve existing database and inspect volumes'); + } + console.log('Existing local sandbox credentials/certificate retained; read-only backup role available.'); } else { - if (files.some(existsSync)) throw new Error('Partial secret initialization; inspect preserved volumes before retrying'); - const admin = randomBytes(32).toString('hex'), owner = randomBytes(32).toString('hex'), runtime = randomBytes(32).toString('hex'); - const put = (file, text, mode = 0o600) => { - writeFileSync(file, text, { flag: 'wx', mode }); chownSync(file, 1000, 1000); - const fd = openSync(file, 'r'); try { fsyncSync(fd); } finally { closeSync(fd); } - }; + if (files.some(existsSync) || backupFiles.some(existsSync)) throw new Error('Partial secret initialization; inspect preserved volumes before retrying'); + const admin = randomBytes(32).toString('hex'), owner = randomBytes(32).toString('hex'), runtime = randomBytes(32).toString('hex'), backup = randomBytes(32).toString('hex'); // PostgreSQL's unprivileged entrypoint also reads POSTGRES_PASSWORD_FILE. This separate scoped // admin volume is never mounted into the application; the password file must be readable there. put('/secrets/admin/password', admin, 0o644); put('/secrets/admin/url', `postgresql://postgres:${admin}@database:5432/wikideck_custody`); put('/secrets/owner/url', `postgresql://custody_owner:${owner}@database:5432/wikideck_custody`); put('/secrets/runtime/url', `postgresql://custody_runtime:${runtime}@database:5432/wikideck_custody`); + put('/secrets/backup/url', `postgresql://custody_backup:${backup}@database:5432/wikideck_custody`); execFileSync('openssl', ['req', '-x509', '-newkey', 'rsa:3072', '-sha256', '-nodes', '-days', '30', '-subj', '/CN=Wikideck local sandbox CA', '-keyout', '/tmp/ca.key', '-out', '/tmp/ca.crt', '-addext', 'basicConstraints=critical,CA:TRUE', '-addext', 'keyUsage=critical,keyCertSign,cRLSign'], { stdio: 'ignore' }); @@ -35,10 +52,10 @@ if (existsSync(marker)) { writeFileSync('/tmp/extensions', 'subjectAltName=DNS:database\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\n'); execFileSync('openssl', ['x509', '-req', '-in', '/tmp/server.csr', '-CA', '/tmp/ca.crt', '-CAkey', '/tmp/ca.key', '-CAcreateserial', '-days', '30', '-sha256', '-extfile', '/tmp/extensions', '-out', '/secrets/tls/server.crt'], { stdio: 'ignore' }); - for (const root of ['admin', 'owner', 'runtime']) put(`/secrets/${root}/ca.crt`, readFileSync('/tmp/ca.crt'), 0o644); + for (const root of ['admin', 'owner', 'runtime', 'backup']) put(`/secrets/${root}/ca.crt`, readFileSync('/tmp/ca.crt'), 0o644); // postgres:17-alpine runs as UID/GID 70. Private key never mounted into Wikideck or migrator. chownSync('/secrets/tls/server.key', 70, 70); chmodSync('/secrets/tls/server.key', 0o600); chmodSync('/secrets/tls/server.crt', 0o644); - put(marker, 'wikideck-local-pg-v1', 0o644); + put(marker, 'wikideck-local-pg-v2', 0o644); console.log('Provisioned fresh local sandbox credentials and verified-TLS certificate; no secrets logged.'); } diff --git a/docs/custody-sale-rehearsal.md b/docs/custody-sale-rehearsal.md index 726a46e..18ef56d 100755 --- a/docs/custody-sale-rehearsal.md +++ b/docs/custody-sale-rehearsal.md @@ -45,7 +45,7 @@ The local-ledger adapter is **not** linked to these terminal candidate IDs: acti ## Validation and remaining gates -TypeScript/Vite Docker build passed; **172 offline regressions and 30 PostgreSQL integration tests passed, zero failures/skips**. [Candidate tests](../tests/sale-candidate.test.ts) cover signature compatibility/recipient tampering and candidate transition rules. [PostgreSQL tests](../tests/postgres/custody.test.ts) add: +TypeScript/Vite Docker build passed; **172 offline regressions and 31 PostgreSQL integration tests passed, zero failures/skips**. [Candidate tests](../tests/sale-candidate.test.ts) cover signature compatibility/recipient tampering and candidate transition rules. [PostgreSQL tests](../tests/postgres/custody.test.ts) add: - explicit instance/database consent and exact signed reservation matching; - seed-binding changes, malformed/cyclic/oversized collections, and denial of ordinary cancellation after binding; diff --git a/docs/market-host-configuration.md b/docs/market-host-configuration.md index 902feff..93b8eda 100755 --- a/docs/market-host-configuration.md +++ b/docs/market-host-configuration.md @@ -32,4 +32,4 @@ Existing signature bytes remain valid: omission retains the original message; ex The custody SQLite fixture records newly settled simulated host revenue under its snapshotted recipient. PostgreSQL production settlement/host withdrawals are still unimplemented: configuring an address does not create automatic transfers. Historical unallocated host revenue remains historical; it is not silently reassigned. The offline sale-lock candidate now preserves the hosting recipient in a tagged signed-terms extension. A [disposable PostgreSQL accounting rehearsal](custody-sale-rehearsal.md) verifies that the signed snapshot receives simulated revenue even after current host policy changes; this is not active-chain delivery or a live host withdrawal service. -Validation: current TypeScript/Vite build, 172 offline regressions and 28 PostgreSQL integration tests passed. Tests cover strict flag/address validation, paused/disabled behavior, continuing existing-order delivery, preserved balances/cancellations, signed-recipient tampering and recipient snapshots across config changes, including the isolated accounting rehearsal. Real XMR remains disabled. +Validation: current TypeScript/Vite build, 172 offline regressions and 31 PostgreSQL integration tests passed. Tests cover strict flag/address validation, paused/disabled behavior, continuing existing-order delivery, preserved balances/cancellations, signed-recipient tampering and recipient snapshots across config changes, including the isolated accounting rehearsal. Real XMR remains disabled. diff --git a/docs/operations-recovery.md b/docs/operations-recovery.md index 0d30247..e51c210 100755 --- a/docs/operations-recovery.md +++ b/docs/operations-recovery.md @@ -59,6 +59,24 @@ Action `verify` needs `--source ` only. It reads a bounded str An operator must choose recovery point/retention/RPO/RTO, practice off-host recovery and approve any production restore explicitly. Never remove a lock or replace a running volume just to make a recovery test pass. +## Coordinated PostgreSQL + node custody backup rehearsal + +The prior file-only `node:backup` still refuses PostgreSQL-bound node directories. A separate explicit [paired backup helper](../compose.custody-backup.yaml) and [utility](../server/coordinated-backup.ts) now create a small-sandbox rehearsal package containing the SQLite ledger/market pair and a custom-format PostgreSQL database dump. Creation holds the exact node-directory lock, then a dedicated PostgreSQL session advisory writer lock, checks the local `custody-postgres.json` binding against the database custodian identity, verifies any retained SQLite cutover source is frozen and its digest matches PostgreSQL's `import_source`, audits PostgreSQL liabilities under the lock, snapshots the active node files and runs `pg_dump`. A frozen SQLite source is deliberately omitted from the active backup pair; PostgreSQL is authoritative after cutover. The pair manifest is written last and binds network/chain/profile/custodian ID, the node manifest digest, PostgreSQL database name and dump size/hash. No restore/reset operation is included. + +The PostgreSQL dump uses a **separate read-only `custody_backup` login**, not the app runtime or schema owner. The local Compose provisioner now creates its credential volume, the one-shot bootstrap grants only database connect, schema usage, table SELECT and sequence SELECT, and the database HBA permits only TLS/SCRAM for this role. Existing local v1 secret volumes get an additive backup-role secret upgrade without rotating database credentials. This changes the next explicit persistent-stack provision/migrate cycle; no running stack was restarted here. The helper image has `pg_dump` and is a one-shot maintenance container; those tools and backup credentials are not added to the long-running app image. The disposable integration test checks that the role can dump but cannot update balances, delete entries or advance journal sequences. + +### Local sandbox invocation + +This procedure is **not yet a production backup runbook**. Use only after reviewing the exact Compose project/volume names and destination permissions: + +1. Stop only the app service and verify no other node/application process writes the same data directory. Leave the PostgreSQL service running. Do not stop/restart or remove any user deployment as part of development tests. +2. With the matching stopped-node directory, run the one-shot provision and migration services under an approved maintenance window so older v1 local secret volumes receive the additive read-only backup credential and database role. Verify existing passwords, CA and custodian ID were preserved; do not regenerate volumes. +3. Set `WIKIDECK_BACKUP_DIR` to an **existing, restricted** host directory on encrypted storage and `WIKIDECK_BACKUP_NAME` to a new leaf name. Run `docker compose -p wikideck-custody-local -f compose.custody-backup.yaml run --rm backup`. The helper requires an existing destination parent and refuses overwrite. The node lock blocks startup during the entire paired snapshot. A PostgreSQL session lock fences all compliant Wikideck custody writers; privileged/manual DB writers must also be operationally fenced. +4. Run the helper with `verify --source /backups/` (overriding the service command) to check the strict pair manifest, standalone node databases, hashes, and exact package contents. Verify pg_dump compatibility/restorability in a separately isolated PostgreSQL database; verification here does not by itself parse/restore the archive. +5. Store an encrypted off-host copy with separately protected authenticity metadata. This package is **not encrypted** by the utility, and colocating it with the source host does not satisfy disaster recovery. + +Creation holds locks and runs a full financial audit; a large journal may exceed configured query/audit budgets and fail closed, so this needs measured long-history tests before production use. A database superuser/schema owner can bypass application advisory-lock discipline or rewrite records; external writer fencing remains mandatory. The output hash/manifest detects accidental damage, not a malicious coherent replacement. The PostgreSQL dump is not atomic with arbitrary unmanaged ledger or schema-owner activity. There is no WAL/PITR, remote scheduler, retention, restore orchestration, RPO/RTO claim, encryption, signature or production failover process in this increment. The local Compose backup service mounts the external host destination read-write into a container with a stopped-node source lock; review Docker volume access and the helper image before using it around any valuable data. + ## Evidence and remaining hold points Automated tests cover default proxy distrust/spoofed-header rejection, explicit loopback proxy recognition, invalid trust/port settings, private listener and threshold checks, paired backup/restore identity preservation, refusal to overwrite/ignore locks/back up within data, and rejection of tampering, wrong network, incomplete packages and unexpected sidecars. diff --git a/docs/persistent-custody-sandbox.md b/docs/persistent-custody-sandbox.md index 84b5534..c349df4 100755 --- a/docs/persistent-custody-sandbox.md +++ b/docs/persistent-custody-sandbox.md @@ -21,7 +21,7 @@ Completed one-shot services are expected to show **Exited (0)**. PostgreSQL and ## Persistent volumes and isolation -Named volumes separately retain PostgreSQL data, node data, admin credentials, schema-owner credentials, runtime credentials and database TLS material. Generated secrets are not written into source, command arguments, Compose environment or browser variables. The app receives no admin/owner credential or server private key. Generated files are scoped by volume mounts; the admin password file is readable inside the trusted database/provisioning containers for the PostgreSQL entrypoint, not mounted in the app. +Named volumes separately retain PostgreSQL data, node data, admin credentials, schema-owner credentials, runtime credentials, a read-only `custody_backup` login, and database TLS material. Generated secrets are not written into source, command arguments, Compose environment or browser variables. The app receives no admin/owner/backup credential or server private key. The separate one-shot migration provisions backup role grants limited to SELECT on custody tables/sequences; the role cannot advance sequence values or update/delete financial records. The admin password file is readable inside the trusted database/provisioning containers for the PostgreSQL entrypoint, not mounted in the app. This is a **new independent market**, not a migration of any existing node or wallet balance. Do not attach an existing SQLite custody volume and expect automatic import. Follow the stopped-node, backup and source-freezing procedure in [ADR 003](postgres-financial-storage.md) for that case. Do not point multiple independent market identities at this one custody schema. @@ -43,7 +43,7 @@ The optional **Check persistent custody TLS and credit** task adds a single 1-XM The **Stop persistent custody without deleting data** task uses Compose `down` without volume removal. Restarting with the same project/file recreates containers and reuses database, credentials, TLS and node state. **Never add `--volumes`/`-v` to normal shutdown:** it destroys custody history and credentials for this stack. Unclean node shutdown may leave the existing exclusive node lock; verify the old process is gone and follow the stale-lock review procedure—do not clear it automatically. -Existing SQLite-only node backup intentionally refuses PostgreSQL-bound nodes. This stack does not yet supply a coordinated backup job: full PostgreSQL backups, protected role/secret material and matching node state must be covered together. Named-volume persistence is **not a backup**, and Docker Desktop resets, disk loss or volume removal can still lose data. WAL/PITR, off-host encrypted copies, restore/failover exercises and RPO/RTO remain production gates. +Existing SQLite-only node backup intentionally refuses PostgreSQL-bound nodes. An explicit [coordinated backup rehearsal](operations-recovery.md#coordinated-postgresql--node-custody-backup-rehearsal) is now available as a separate one-shot compose file; it does not run continuously and is not invoked on ordinary app startup. After stopping the app, it takes a node+PostgreSQL snapshot under both writer fences and a full custody audit. This small-sandbox helper is not an encrypted/off-host backup system, WAL/PITR, restore orchestrator or RPO/RTO claim. Named-volume persistence is **not a backup**, and Docker Desktop resets, disk loss or volume removal can still lose data. A v1 secret-volume upgrade adds the backup-only credential without rotating existing secrets, but should first be rehearsed on a copy of the intended deployment volumes. ## Evidence and limits diff --git a/docs/postgres-financial-storage.md b/docs/postgres-financial-storage.md index 088cdca..6136fea 100755 --- a/docs/postgres-financial-storage.md +++ b/docs/postgres-financial-storage.md @@ -79,7 +79,7 @@ There is no live dual-write, automatic migration or automatic rollback. An old p ## Backup, restore and failure handling -Existing node backup packages cover local SQLite databases only. They now **refuse** a PostgreSQL-bound data directory or frozen cutover source instead of claiming to capture all customer liabilities. No automatic cross-database backup coordinator is implemented. +Existing node backup packages cover local SQLite databases only. They now **refuse** a PostgreSQL-bound data directory or frozen cutover source instead of claiming to capture all customer liabilities. A separate [coordinated stopped-node rehearsal](operations-recovery.md#coordinated-postgresql--node-custody-backup-rehearsal) can now package node SQLite plus a PostgreSQL custom dump while holding the node lock, the database advisory writer lock and an audit snapshot. It uses a separate read-only backup role in the local persistent sandbox. This is not an automatic production coordinator, and uncooperative privileged writers still require external fencing. Recommended operations before any production release: @@ -89,7 +89,7 @@ Recommended operations before any production release: - Reconcile Monero external history before resuming real spending once that integration exists. No local backup can reverse an external payment. Database timeouts/connection loss leave requests failed closed; retry the original signed command, not a new withdrawal. - Monitor application and database errors without logging statements/parameters containing private customer data in production. Fixture logs deliberately show failed constraint probes on fictitious data; do not adopt their logging/auth configuration. -The integration suite performs a real full `pg_dump`/isolated `pg_restore` rehearsal preserving ID, exact liabilities and receipt replay. It does **not** certify managed backup schedules, PITR, encrypted off-host custody, power-loss/fsync behavior or production failover. +The integration suite performs both a PostgreSQL `pg_dump`/isolated `pg_restore` rehearsal preserving ID, exact liabilities and receipt replay, and a paired node/PostgreSQL archive-creation/verification/restore test using the actual runtime adapter plus a read-only backup role. This does **not** certify managed backup schedules, PITR, encrypted off-host custody, signature/authenticity, power-loss/fsync behavior, long-history timing or production failover. ## Evidence and remaining gates diff --git a/docs/production-plan.md b/docs/production-plan.md index 2adc503..33dba7c 100755 --- a/docs/production-plan.md +++ b/docs/production-plan.md @@ -90,6 +90,10 @@ Blocked on P0 ADR; use P1 amount types and P2 durable events. Consensus work is **Canonical/per-order reorg rehearsal increment, 2026-10-20:** the gated disposable PostgreSQL observer stores bounded pages from an in-process `Ledger` adapter, checks stream/anchor/hash/tip/ancestor continuity, supports retries, and audits retained history. Tests cover a block mined and independently verified with native RandomX flowing through the local ledger into PostgreSQL, plus a fake-PoW equal-work branch/reorg. A separate synthetic-event test links the deterministic candidate terminal transaction ID to its posted order; disconnecting that ID (even followed by same-page re-inclusion) commits the event history and durable order review hold atomically, without undoing or repeating the payout. Generic incident resolution cannot clear a sale-reorg record. The server now has an opt-in polling lifecycle, restricted to PostgreSQL `custody_test`, both disposable flags and non-production mode; it drains on shutdown, resumes the persistent cursor after restart and stops on review/failure. Final build, 172 offline regressions and 30 PostgreSQL tests pass. The worker remains disabled in all regular deployments, active v6 cannot emit candidate IDs, and synthetic linkage is not canonical sale evidence. Stream hashes are not tamper-proof against the schema owner; 4,096-event cap/full scan are non-production. M3 remains blocked on reviewed candidate activation, actual candidate IDs from an activated local ledger, production observer/checkpoint design and production readiness. +**Coordinated backup rehearsal increment, 2026-10-20:** a separate stopped-node maintenance utility now holds the node lock and custody advisory writer lock, checks the local/PostgreSQL custodian ID and runs a complete accounting audit before pairing SQLite ledger/market snapshots with a custom-format `pg_dump`. It uses a dedicated SELECT-only backup login (the local persistent stack provisions it additively, without rotating the existing credentials); the helper is separate from the normal app image. A strict pair manifest is written last. Disposable integration tests restore the dump into an isolated database and verify identity/balances; lock, mismatch, failed dump, incomplete output, overwrite and tamper paths fail closed. Final build, 172 offline regressions and 31 PostgreSQL tests pass. This package is not encrypted or off-host, has no restore orchestrator, PITR/retention/scheduler/RPO/RTO, and has not been run against the persistent stack. Existing services/data were not restarted or changed. Production coordinated recovery remains open. + +**Coordinated-backup rehearsal increment, 2026-10-20:** a stopped-node maintenance utility now holds the node startup lock and PostgreSQL custody advisory writer lock, confirms local/database custodian identity, performs a liability audit, snapshots node SQLite databases, writes a full custom-format PostgreSQL dump through a dedicated read-only backup login, and writes a pair manifest last. A disposable integration verifies package hashes and restores the dump into an isolated database with balances/identity preserved; failure/lock/tamper behavior is tested. The persistent local sandbox provisioner adds the backup secret/role without rotating existing credentials. This is not encrypted, off-host, signed, scheduled, PITR, long-history performance or production recovery evidence. No persistent deployment was restarted. Production backups and measurable RPO/RTO remain open. + - If a reviewed trustless design is selected: implement its actual cross-chain validity/evidence rules, compatible cryptographic primitives, both-party failure/refund paths and commission semantics. Run maintained reference vectors, negative tests, fuzz/property tests and specialist audit before claiming atomicity. - If a custodial/attested fallback is explicitly approved: define chain-enforced NFT lock, order identity, authorized releaser, cancellation/refund/expiry and stale-signature rules. Separate host control keys from seller wallet and processor payout keys. Reserve/refund procedures must not permit both buyer refund and later seller delivery/release. Describe who can steal, censor or lock assets and how recovery is authorized. - Completed-sale inclusion must require the approved evidence that payment is sufficiently confirmed. A peer must not insert a completed sale merely because its local database says paid. Define threshold/policy version, proof/attestation authorization and deterministic validation; no external HTTP calls in consensus validation. diff --git a/docs/sale-lock-candidate.md b/docs/sale-lock-candidate.md index 2d5a1c2..76f9853 100755 --- a/docs/sale-lock-candidate.md +++ b/docs/sale-lock-candidate.md @@ -66,7 +66,7 @@ An already mature terminal record remains usable evidence if the NFT is later gi ## Evidence and next release gates -Initial candidate evidence: 162 offline tests passed, including 13 candidate tests. Current TypeScript/Vite Docker build and **172 offline tests plus 28 PostgreSQL integration tests passed, zero failures/skips**, including 14 candidate tests in [tests/sale-candidate.test.ts](../tests/sale-candidate.test.ts). Coverage includes current-consensus rejection, field/signature and hosting-recipient tampering, exact height boundaries, sequence exhaustion, native transfer/trade lock bypass, competing-market/reused-order locks, atomic conflicting-outcome rejection, reorg financial decisions, deterministic snapshot restore, state commitments, actual custody-order drafting and enumerated adversarial histories through expiry. The [accounting rehearsal](custody-sale-rehearsal.md) records actual PostgreSQL/fictitious-funds postings, a local-ledger event adapter test with fake PoW, and synthetic candidate-terminal disconnect linkage. No protocol guarantee is inferred from a test count. +Initial candidate evidence: 162 offline tests passed, including 13 candidate tests. Current TypeScript/Vite Docker build and **172 offline tests plus 31 PostgreSQL integration tests passed, zero failures/skips**, including 14 candidate tests in [tests/sale-candidate.test.ts](../tests/sale-candidate.test.ts). Coverage includes current-consensus rejection, field/signature and hosting-recipient tampering, exact height boundaries, sequence exhaustion, native transfer/trade lock bypass, competing-market/reused-order locks, atomic conflicting-outcome rejection, reorg financial decisions, deterministic snapshot restore, state commitments, actual custody-order drafting and enumerated adversarial histories through expiry. The [accounting rehearsal](custody-sale-rehearsal.md) records actual PostgreSQL/fictitious-funds postings, a local-ledger event adapter with native RandomX validation, fake-PoW branch/reorg fixtures, and synthetic candidate-terminal disconnect linkage. Coordinated backup tests use a separate read-only fixture role and isolated restore. No protocol or production-recovery guarantee is inferred from a test count. **This does not close M3.** Required next steps: diff --git a/package.json b/package.json index ea6237c..4178ce0 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "benchmark:randomx": "node --experimental-strip-types scripts/benchmark-randomx.ts", "benchmark:index": "node --experimental-strip-types scripts/benchmark-canonical-index.ts", "custody:db": "node --experimental-strip-types scripts/custody-db.ts", + "custody:backup": "node --experimental-strip-types scripts/custody-pair-backup.ts", "node:backup": "node --experimental-strip-types scripts/node-backup.ts", "ops:check": "node --experimental-strip-types scripts/ops-check.ts", "miner": "tsx server/testnet/miner.ts", diff --git a/scripts/custody-deployment-init.ts b/scripts/custody-deployment-init.ts index fda6106..0338b2f 100755 --- a/scripts/custody-deployment-init.ts +++ b/scripts/custody-deployment-init.ts @@ -11,17 +11,26 @@ try { const admin = new Pool(await config('admin')); try { const ownerConfig = await config('owner'), runtimeConfig = await config('runtime'); - for (const [role, password] of [['custody_owner', ownerConfig.password], ['custody_runtime', runtimeConfig.password]] as const) { + const backupConfig = await postgresConfig({ WIKIDECK_CUSTODY_PG_URL_FILE: '/secrets/backup/url', + WIKIDECK_CUSTODY_PG_CA_FILE: '/secrets/backup/ca.crt' }); + for (const [role, password] of [['custody_owner', ownerConfig.password], ['custody_runtime', runtimeConfig.password], + ['custody_backup', backupConfig.password]] as const) { // Static role allowlist; password is a generated hex token. Values are never printed, even on error. if (typeof password !== 'string' || !/^[a-f0-9]{64}$/.test(password)) throw new Error('Invalid generated credential'); const exists = await admin.query('SELECT 1 FROM pg_roles WHERE rolname=$1', [role]); if (!exists.rowCount) await admin.query(`CREATE ROLE ${role} LOGIN PASSWORD '${password}' NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS`); } await admin.query('REVOKE CREATE ON SCHEMA public FROM PUBLIC; REVOKE ALL ON DATABASE wikideck_custody FROM PUBLIC'); - await admin.query('GRANT CONNECT,CREATE ON DATABASE wikideck_custody TO custody_owner; GRANT CONNECT ON DATABASE wikideck_custody TO custody_runtime'); + await admin.query(`GRANT CONNECT,CREATE ON DATABASE wikideck_custody TO custody_owner; + GRANT CONNECT ON DATABASE wikideck_custody TO custody_runtime,custody_backup`); } finally { await admin.end(); } const owner = new Pool(await config('owner')); - try { await migrateCustody(owner); await grantCustodyRuntime(owner, 'custody_runtime'); } + try { + await migrateCustody(owner); await grantCustodyRuntime(owner, 'custody_runtime'); + await owner.query(`GRANT USAGE ON SCHEMA wk_custody TO custody_backup; + GRANT SELECT ON ALL TABLES IN SCHEMA wk_custody TO custody_backup; + GRANT SELECT ON ALL SEQUENCES IN SCHEMA wk_custody TO custody_backup`); + } finally { await owner.end(); } console.log('Local PostgreSQL schema and restricted runtime grants ready. Sandbox payments only.'); } catch { diff --git a/scripts/custody-pair-backup.ts b/scripts/custody-pair-backup.ts new file mode 100755 index 0000000..3a4d6dd --- /dev/null +++ b/scripts/custody-pair-backup.ts @@ -0,0 +1,29 @@ +import { createCoordinatedCustodyBackup, verifyCoordinatedCustodyBackup } from '../server/coordinated-backup.ts'; + +const [action, ...args] = process.argv.slice(2); +if (action !== 'create' && action !== 'verify') throw new Error('Expected create or verify'); +const allowed = new Set(['--source', '--destination', '--consent']); +for (let index = 0; index < args.length; index++) { + if (!allowed.has(args[index]) || (args[index] !== '--consent' && (!args[index + 1] || args[index + 1].startsWith('--')))) { + throw new Error('Invalid coordinated backup arguments'); + } + if (args[index] !== '--consent') index++; +} +const value = (name: string) => { const index = args.indexOf(name); return index < 0 ? undefined : args[index + 1]; }; +const source = value('--source'); +if (!source) throw new Error('Explicit --source is required'); +let result: unknown; +try { + if (action === 'create') { + const destination = value('--destination'); + if (!destination || !args.includes('--consent')) throw new Error('Create requires --destination and explicit --consent'); + result = await createCoordinatedCustodyBackup(source, destination); + } else { + if (value('--destination') || args.includes('--consent')) throw new Error('Verify accepts only --source'); + result = await verifyCoordinatedCustodyBackup(source); + } + console.log(JSON.stringify(result, null, 2)); +} catch { + console.error('Coordinated backup failed; check stopped-node state, separate backup credentials, PostgreSQL tools, destination permissions and partial output privately. No source database was reset.'); + process.exitCode = 1; +} diff --git a/server/coordinated-backup.ts b/server/coordinated-backup.ts new file mode 100755 index 0000000..7bdf60e --- /dev/null +++ b/server/coordinated-backup.ts @@ -0,0 +1,196 @@ +import { createHash } from 'node:crypto'; +import { spawn, type ChildProcess } from 'node:child_process'; +import { createReadStream } from 'node:fs'; +import { chmod, mkdir, lstat, readFile, readdir, writeFile, rm, open, realpath } from 'node:fs/promises'; +import { dirname, join, resolve, relative, isAbsolute } from 'node:path'; +import { tmpdir } from 'node:os'; +import { Pool } from 'pg'; +import { DatabaseSync } from 'node:sqlite'; +import { z } from 'zod'; +import { IS_MAINNET, NETWORK, PROFILE } from '../shared/network.ts'; +import { CHAIN_ID } from './testnet/consensus.ts'; +import { CUSTODY_WRITE_LOCK } from './market/postgres-schema.ts'; +import { postgresConfig } from './market/postgres-config.ts'; +import { withStoppedNodeLock, createNodeSnapshotUnlocked, verifyBackup } from './recovery.ts'; +import { auditPostgres } from './market/postgres-custody.ts'; + +const pairManifestSchema = z.object({ format: z.literal('wikideck-coordinated-backup'), version: z.literal(1), + network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID), profile: z.literal(PROFILE), + createdAt: z.string().datetime(), custodianId: z.string().uuid(), database: z.string().regex(/^[a-zA-Z0-9_-]+$/), + sqliteCutoverSourceDigest: z.string().regex(/^[a-f0-9]{64}$/).nullable(), + nodeManifestSha256: z.string().regex(/^[a-f0-9]{64}$/), + postgresDump: z.object({ name: z.literal('custody.dump'), bytes: z.number().int().positive().max(256 * 1024 ** 3), + sha256: z.string().regex(/^[a-f0-9]{64}$/) }).strict() }).strict(); +export type CoordinatedBackupManifest = z.infer; +const bindingSchema = z.object({ version: z.literal(1), custodianId: z.string().uuid() }).strict(); +function digest(data: Buffer | string): string { return createHash('sha256').update(data).digest('hex'); } +async function fileDigest(path: string): Promise { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest('hex'); +} +async function flush(path: string): Promise { + const handle = await open(path, 'r+'); try { await handle.sync(); } finally { await handle.close(); } +} +async function flushDirectory(path: string): Promise { + if (process.platform === 'win32') return; + const handle = await open(path, 'r'); try { await handle.sync(); } finally { await handle.close(); } +} +async function regularFile(path: string): Promise<{ size: number }> { + const info = await lstat(path); + if (!info.isFile() || info.isSymbolicLink()) throw new Error('Coordinated backup file is not a regular non-symlink file'); + return info; +} +function childEnvironment(config: Awaited>, caFile: string | undefined, + serviceFile: string, passFile: string): NodeJS.ProcessEnv { + if (typeof config.host !== 'string' || !config.host || typeof config.database !== 'string' || !config.database || + typeof config.user !== 'string' || !config.user) throw new Error('Incomplete PostgreSQL backup connection settings'); + const ssl = config.ssl; + const tls = ssl === false ? 'disable' : 'verify-full'; + const ca = typeof ssl === 'object' && ssl && 'ca' in ssl ? ssl.ca : undefined; + if (ca && !caFile) throw new Error('Private PostgreSQL CA must be staged for the backup client'); + const env: NodeJS.ProcessEnv = { ...process.env, PGHOST: config.host, PGPORT: String(config.port), PGDATABASE: config.database, + PGUSER: config.user, PGPASSWORD: String(config.password ?? ''), PGPASSFILE: passFile, PGCONNECT_TIMEOUT: '10', + PGSERVICEFILE: serviceFile, PGSSLMODE: tls, ...(caFile ? { PGSSLROOTCERT: caFile } : {}) }; + delete env.PGSERVICE; delete env.PGSSLCERT; delete env.PGSSLKEY; delete env.PGOPTIONS; + return env; +} +async function dumpDatabase(config: Awaited>, output: string, binary: string): Promise { + const temp = await import('node:fs/promises').then(m => m.mkdtemp(join(tmpdir(), 'wikideck-pg-backup-'))); + let caFile: string | undefined; + try { + const ssl = config.ssl; + const ca = typeof ssl === 'object' && ssl && 'ca' in ssl ? ssl.ca : undefined; + if (typeof ca === 'string') { caFile = join(temp, 'root-ca.pem'); await writeFile(caFile, ca, { flag: 'wx', mode: 0o600 }); } + const serviceFile = join(temp, 'pg_service.conf'); + await writeFile(serviceFile, '', { flag: 'wx', mode: 0o600 }); + const passFile = join(temp, 'pgpass'); + await writeFile(passFile, '', { flag: 'wx', mode: 0o600 }); + const env = childEnvironment(config, caFile, serviceFile, passFile); + await new Promise((resolveDump, reject) => { + let diagnostic = ''; + const child: ChildProcess = spawn(binary, ['--no-password', '--format=custom', '--compress=6', + '--file', output, '--dbname', config.database!], { shell: false, windowsHide: true, stdio: ['ignore', 'ignore', 'pipe'], env }); + child.stderr?.on('data', chunk => { diagnostic = (diagnostic + chunk.toString('utf8')).slice(-1024); }); + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; child.kill(); + const forceKill = setTimeout(() => child.kill('SIGKILL'), 5000); forceKill.unref(); + child.once('exit', () => clearTimeout(forceKill)); + }, 30 * 60_000); + timeout.unref(); + child.once('error', () => { clearTimeout(timeout); reject(new Error('PostgreSQL dump utility failed to start')); }); + child.once('exit', code => { + clearTimeout(timeout); + if (timedOut) reject(new Error('PostgreSQL dump exceeded its time budget')); + else code === 0 ? resolveDump() : reject(new Error(`PostgreSQL dump failed (${code ?? 'signal'}): ${diagnostic.replace(/[\r\n\x00-\x1f]/g, ' ').slice(0, 500)}`)); + }); + }); + } finally { await rm(temp, { recursive: true, force: true }); } +} + +/** Stop-node + PostgreSQL writer-fenced snapshot. This sandbox tool never restores or resets a database. */ +export async function createCoordinatedCustodyBackup(dataDirectory: string, destination: string, + env: NodeJS.ProcessEnv = process.env): Promise { + if (IS_MAINNET) throw new Error('Coordinated custody backup utility is not approved for mainnet'); + const backupUrlFile = env.WIKIDECK_CUSTODY_PG_BACKUP_URL_FILE; + if (!backupUrlFile) throw new Error('A separate PostgreSQL backup-role URL file is required'); + const runtimeConfig = await postgresConfig(env); + const backupConfig = await postgresConfig({ ...env, WIKIDECK_CUSTODY_PG_URL_FILE: backupUrlFile }); + const dumpBinary = env.WIKIDECK_PG_DUMP_BINARY ?? 'pg_dump'; + if (dumpBinary !== 'pg_dump' && !isAbsolute(dumpBinary)) throw new Error('Custom PostgreSQL dump binary must use an absolute path'); + if (backupConfig.host !== runtimeConfig.host || backupConfig.port !== runtimeConfig.port || + backupConfig.database !== runtimeConfig.database || backupConfig.user === runtimeConfig.user) { + throw new Error('Backup database target must match custody, using a separate backup role'); + } + const source = await realpath(dataDirectory); + const target = resolve(destination), rel = relative(source, target); + if (!rel || (!rel.startsWith('..') && !isAbsolute(rel))) throw new Error('Backup destination must be outside node data directory'); + const parentPath = dirname(target), parent = await realpath(parentPath); + if (process.platform === 'win32' ? parent.toLowerCase() !== parentPath.toLowerCase() : parent !== parentPath) { + throw new Error('Backup destination parent must not be symlinked'); + } + try { await lstat(target); throw new Error('Backup destination already exists'); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } + + return withStoppedNodeLock(source, async lockedSource => { + // Reserve the final name exclusively. The pair manifest is written last; partial folders never verify. + await mkdir(target, { mode: 0o700 }); + const bindingPath = join(lockedSource, PROFILE, 'custody-postgres.json'); + const bindingInfo = await regularFile(bindingPath); + if (bindingInfo.size > 4096) throw new Error('PostgreSQL data-directory binding exceeds limit'); + const binding = bindingSchema.parse(JSON.parse(await readFile(bindingPath, 'utf8'))); + const pool = new Pool({ ...backupConfig, max: 1, application_name: 'wikideck-coordinated-backup' }); + let client; + try { + client = await pool.connect(); + const actualDb = await client.query('SELECT current_database() AS name'); + if (actualDb.rows[0].name !== backupConfig.database) throw new Error('Backup connected to an unexpected database'); + await client.query('SELECT pg_advisory_lock($1::bigint)', [CUSTODY_WRITE_LOCK]); + const identity = await client.query("SELECT value FROM wk_custody.custody_meta WHERE key='identity'"); + if (!identity.rowCount) throw new Error('Custody PostgreSQL identity missing'); + const record = JSON.parse(identity.rows[0].value); + if (record.id !== binding.custodianId || record.version !== 1 || record.network !== NETWORK || + record.chainId !== CHAIN_ID || record.mode !== 'sandbox') throw new Error('PostgreSQL and node custody identities differ'); + const imported = await client.query("SELECT value FROM wk_custody.custody_meta WHERE key='import_source'"); + const frozenSource = join(lockedSource, PROFILE, 'custody-sandbox.sqlite'); + try { + const sourceInfo = await lstat(frozenSource); + if (!sourceInfo.isFile() || sourceInfo.isSymbolicLink() || !imported.rowCount) throw new Error('Frozen SQLite cutover source is not bound to this PostgreSQL database'); + const sourceDb = new DatabaseSync(frozenSource, { readOnly: true }); + try { + const marker = sourceDb.prepare("SELECT value FROM custody_meta WHERE key='postgres_cutover'").get() as { value: string } | undefined; + if (marker?.value !== imported.rows[0].value) throw new Error('Frozen SQLite source digest differs from PostgreSQL import binding'); + } finally { sourceDb.close(); } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + if (imported.rowCount) throw new Error('Imported PostgreSQL custody is missing its frozen SQLite source; operator review required'); + } + await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY'); + try { await auditPostgres(client); await client.query('COMMIT'); } + catch (error) { await client.query('ROLLBACK').catch(() => {}); throw error; } + + const nodeManifest = await createNodeSnapshotUnlocked(lockedSource, join(target, 'node'), true); + const dumpPath = join(target, 'custody.dump'); + await dumpDatabase(backupConfig, dumpPath, dumpBinary); + const dumpInfo = await regularFile(dumpPath); + if (dumpInfo.size < 1 || dumpInfo.size > 256 * 1024 ** 3) throw new Error('PostgreSQL dump size outside supported bounds'); + if (process.platform !== 'win32') await chmod(dumpPath, 0o600); + await flush(dumpPath); + const manifest = pairManifestSchema.parse({ format: 'wikideck-coordinated-backup', version: 1, network: NETWORK, + chainId: CHAIN_ID, profile: PROFILE, createdAt: new Date().toISOString(), custodianId: binding.custodianId, + database: backupConfig.database, sqliteCutoverSourceDigest: imported.rows[0]?.value ?? null, + nodeManifestSha256: digest(JSON.stringify(nodeManifest)), + postgresDump: { name: 'custody.dump', bytes: dumpInfo.size, sha256: await fileDigest(dumpPath) } }); + await writeFile(join(target, 'manifest.json'), JSON.stringify(manifest, null, 2), { flag: 'wx', mode: 0o600 }); + if (process.platform !== 'win32') await chmod(join(target, 'manifest.json'), 0o600); + await flush(join(target, 'manifest.json')); await flushDirectory(target); await flushDirectory(parent); + return manifest; + } finally { + if (client) { + await client.query('SELECT pg_advisory_unlock($1::bigint)', [CUSTODY_WRITE_LOCK]).catch(() => {}); + client.release(); + } + await pool.end(); + } + }); +} + +export async function verifyCoordinatedCustodyBackup(directory: string): Promise { + const root = await realpath(directory); + const manifestPath = join(root, 'manifest.json'), info = await regularFile(manifestPath); + if (info.size > 16 * 1024) throw new Error('Coordinated backup manifest exceeds limit'); + const manifest = pairManifestSchema.parse(JSON.parse(await readFile(manifestPath, 'utf8'))); + const nodePath = join(root, 'node'), nodeInfo = await lstat(nodePath); + if (!nodeInfo.isDirectory() || nodeInfo.isSymbolicLink()) throw new Error('Node backup package must be a regular directory'); + const nodeManifest = await verifyBackup(nodePath); + if (digest(JSON.stringify(nodeManifest)) !== manifest.nodeManifestSha256 || nodeManifest.network !== manifest.network || + nodeManifest.chainId !== manifest.chainId || nodeManifest.profile !== manifest.profile) throw new Error('Node snapshot does not match pair manifest'); + const dumpPath = join(root, manifest.postgresDump.name), dumpInfo = await regularFile(dumpPath); + if (dumpInfo.size !== manifest.postgresDump.bytes || await fileDigest(dumpPath) !== manifest.postgresDump.sha256) { + throw new Error('PostgreSQL dump hash/size mismatch'); + } + const entries = (await readdir(root)).sort(); + if (entries.join('|') !== ['custody.dump', 'manifest.json', 'node'].sort().join('|')) throw new Error('Unexpected coordinated backup contents'); + return manifest; +} diff --git a/server/recovery.ts b/server/recovery.ts index 934a084..c69e147 100755 --- a/server/recovery.ts +++ b/server/recovery.ts @@ -51,19 +51,28 @@ async function offlineLock(dataDir:string,fn:()=>Promise):Promise{ try{await handle.writeFile(`maintenance:${process.pid}`);return await fn();} finally{await handle.close();await unlink(lockPath);} } -export async function createBackup(dataDir:string,destination:string){ - const source=await realpath(dataDir),dest=resolve(destination); +/** Run a pair-backup operation while holding the exact exclusive lock used by node startup. */ +export async function withStoppedNodeLock(dataDir:string,operation:(source:string)=>Promise):Promise{ + const source=await realpath(dataDir); + return offlineLock(source,()=>operation(source)); +} +/** Snapshot the local SQLite half while the caller holds withStoppedNodeLock(). */ +export async function createNodeSnapshotUnlocked(source:string,destination:string,allowPostgresBinding=false){ + const dest=resolve(destination); const rel=relative(source,dest); if(!rel||(!rel.startsWith('..')&&!isAbsolute(rel)))throw new Error('Backup destination must be outside the node data directory'); - return offlineLock(source,async()=>{ - // File-only backups cannot represent an external financial database. Fail rather than omit liabilities. - try { await lstat(join(source,PROFILE,'custody-postgres.json')); throw new Error('PostgreSQL custody requires coordinated database and node backup; SQLite-only backup refused'); } - catch(e) { if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; } + // File-only backups cannot represent an external financial database. Fail rather than omit liabilities. + try { + await lstat(join(source,PROFILE,'custody-postgres.json')); + if(!allowPostgresBinding)throw new Error('PostgreSQL custody requires coordinated database and node backup; SQLite-only backup refused'); + } catch(e) { if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; } if(await realpath(join(source,PROFILE))!==join(source,PROFILE))throw new Error('Network data directory must not be symlinked'); await mkdir(dest,{mode:0o700}); // Exclusive: never overwrite or recursively create a surprising destination. const entries=[]; for(const name of files){ const src=join(source,PROFILE,name),target=join(dest,name); + // PostgreSQL is authoritative after cutover. Do not reinterpret or copy the frozen SQLite source as current custody. + if(name==='custody-sandbox.sqlite'&&allowPostgresBinding)continue; if(name==='custody-sandbox.sqlite'){ try{await lstat(src);}catch(e){if((e as NodeJS.ErrnoException).code==='ENOENT')continue;throw e;} } @@ -82,7 +91,9 @@ export async function createBackup(dataDir:string,destination:string){ await flush(join(dest,'manifest.json')); await flushDirectory(dest); return manifest; - }); +} +export async function createBackup(dataDir:string,destination:string){ + return withStoppedNodeLock(dataDir,source=>createNodeSnapshotUnlocked(source,destination)); } export async function verifyBackup(directory:string){ const root=await realpath(directory),manifestFile=join(root,'manifest.json'); diff --git a/tests/postgres/custody.test.ts b/tests/postgres/custody.test.ts index 3e32ca8..8381c68 100755 --- a/tests/postgres/custody.test.ts +++ b/tests/postgres/custody.test.ts @@ -2,7 +2,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { Pool, type PoolConfig } from 'pg'; import { generateKeyPairSync, randomUUID, sign } from 'node:crypto'; -import { access, mkdtemp, rm, writeFile, mkdir, readFile } from 'node:fs/promises'; +import { access, lstat, mkdtemp, rm, writeFile, mkdir, readFile } from 'node:fs/promises'; import { constants } from 'node:fs'; import { join, resolve } from 'node:path'; import { tmpdir } from 'node:os'; @@ -19,6 +19,7 @@ import { importSqliteCustody } from '../../server/market/custody-import.ts'; import { postgresConfig, custodyStorage } from '../../server/market/postgres-config.ts'; import { openCustodyStorage } from '../../server/market/custody-storage.ts'; import { createBackup } from '../../server/recovery.ts'; +import { createCoordinatedCustodyBackup, verifyCoordinatedCustodyBackup } from '../../server/coordinated-backup.ts'; import { CUSTODY_POLICY, custodyMessage, type CustodyAction, type CustodyCommand, type CustodyActivity, type CustodyListing, type CustodyOrder, type CustodyWithdrawal } from '../../shared/custody.ts'; import { NETWORK } from '../../shared/network.ts'; @@ -367,6 +368,93 @@ test('full PostgreSQL dump/isolated restore preserves custody identity, receipts } }); +test('coordinated node/PostgreSQL backup holds both writer fences and verifies matching custodian identity', async t => { + const f = await fixture(t); await f.execute(buyer, { type: 'mock-credit' }); + const dir = await mkdtemp(join(tmpdir(), 'wikideck-coordinated-backup-')); + t.after(() => rm(dir, { recursive: true, force: true })); + const data = join(dir, 'node-data'), networkDir = join(data, 'testnet'); + await mkdir(networkDir, { recursive: true }); + const ledgerDb = new DatabaseSync(join(networkDir, 'ledger.sqlite')); + try { + ledgerDb.exec(`CREATE TABLE metadata(key TEXT PRIMARY KEY,value TEXT NOT NULL); + INSERT INTO metadata VALUES('schema_version','1'),('network','${NETWORK}'),('chain_id','${CHAIN_ID}'); + CREATE TABLE blocks(sequence INTEGER PRIMARY KEY AUTOINCREMENT,hash TEXT NOT NULL,previous_hash TEXT NOT NULL,block_json TEXT NOT NULL); + CREATE TABLE pending_transfers(position INTEGER PRIMARY KEY,id TEXT NOT NULL,transaction_json TEXT NOT NULL);`); + } finally { ledgerDb.close(); } + const marketDb = new DatabaseSync(join(networkDir, 'market.sqlite')); + try { + marketDb.exec('CREATE TABLE market_meta(key TEXT PRIMARY KEY,value TEXT NOT NULL)'); + marketDb.prepare('INSERT INTO market_meta VALUES(?,?)').run('identity', JSON.stringify({ version: 1, + network: NETWORK, chainId: CHAIN_ID, id: randomUUID() })); + } finally { marketDb.close(); } + await writeFile(join(networkDir, 'custody-postgres.json'), JSON.stringify({ version: 1, custodianId: f.market.id }), { mode: 0o600 }); + // Model a legitimate post-cutover node: its frozen SQLite source is historical and PostgreSQL is authoritative. + const cutoverDigest = 'a'.repeat(64), legacyDb = new DatabaseSync(join(networkDir, 'custody-sandbox.sqlite')); + try { + legacyDb.exec('CREATE TABLE custody_meta(key TEXT PRIMARY KEY,value TEXT NOT NULL)'); + legacyDb.prepare('INSERT INTO custody_meta VALUES(?,?)').run('postgres_cutover', cutoverDigest); + } finally { legacyDb.close(); } + await f.admin.query('INSERT INTO wk_custody.custody_meta(key,value) VALUES($1,$2)', ['import_source', cutoverDigest]); + const runtimeUrl = join(dir, 'runtime-url'), backupUrl = join(dir, 'backup-url'); + await writeFile(runtimeUrl, 'postgresql://custody_runtime@127.0.0.1/custody_test', { mode: 0o600 }); + const backupRole = `custody_backup_${randomUUID().replaceAll('-', '')}`; + await f.admin.query(`CREATE ROLE ${backupRole} LOGIN; GRANT CONNECT ON DATABASE custody_test TO ${backupRole}; + GRANT USAGE ON SCHEMA wk_custody TO ${backupRole}; GRANT SELECT ON ALL TABLES IN SCHEMA wk_custody TO ${backupRole}; + GRANT SELECT ON ALL SEQUENCES IN SCHEMA wk_custody TO ${backupRole}`); + await writeFile(backupUrl, `postgresql://${backupRole}@127.0.0.1/custody_test`, { mode: 0o600 }); + const env = { ...process.env, WIKIDECK_CUSTODY_PG_URL_FILE: runtimeUrl, + WIKIDECK_CUSTODY_PG_BACKUP_URL_FILE: backupUrl, WIKIDECK_CUSTODY_PG_LOCAL_TEST: '1' }; + const destination = join(dir, 'paired-backup'); + const manifest = await createCoordinatedCustodyBackup(data, destination, env); + assert.equal(manifest.custodianId, f.market.id); assert.equal(manifest.database, 'custody_test'); + assert.equal(manifest.sqliteCutoverSourceDigest, cutoverDigest); + assert.equal((await verifyCoordinatedCustodyBackup(destination)).nodeManifestSha256, manifest.nodeManifestSha256); + await assert.rejects(lstat(join(destination, 'node', 'custody-sandbox.sqlite')), { code: 'ENOENT' }); + const restoreDatabase = `pair_restore_${randomUUID().replaceAll('-', '')}`, run = promisify(execFile); + await f.admin.query(`CREATE DATABASE "${restoreDatabase}"`); + try { + await run('pg_restore', ['-h', '127.0.0.1', '-U', 'postgres', '-d', restoreDatabase, '--exit-on-error', join(destination, 'custody.dump')]); + const restored = await PostgresCustody.open({ ...f.runtime, database: restoreDatabase }, options); + try { + assert.equal(restored.id, f.market.id); + assert.deepEqual(await restored.audit(), await f.market.audit()); + assert.equal(await restored.importSource(), cutoverDigest); + const activity = await restored.execute(envelope(restored.id, buyer, { type: 'activity', offset: 0 })) as CustodyActivity; + assert.equal(activity.availableAtomicXmr, '1000000000000'); + } finally { await restored.close(); } + } finally { await f.admin.query(`DROP DATABASE IF EXISTS "${restoreDatabase}" WITH (FORCE)`); } + const backupRolePool = new Pool({ ...config, user: backupRole }); + try { + await assert.rejects(backupRolePool.query("UPDATE wk_custody.accounts SET balance=0")); + await assert.rejects(backupRolePool.query("DELETE FROM wk_custody.entries")); + await assert.rejects(backupRolePool.query("SELECT nextval('wk_custody.journals_sequence_seq')")); + } finally { await backupRolePool.end(); } + assert.equal((await f.market.audit()).assetsAtomicXmr, '1000000000000'); + await assert.rejects(createCoordinatedCustodyBackup(data, destination, env), /already exists/); + const staleLock = join(data, 'testnet-node.lock'), lockedDestination = join(dir, 'locked-backup'); + await writeFile(staleLock, 'operator-lock'); + await assert.rejects(createCoordinatedCustodyBackup(data, lockedDestination, env), /Node directory is locked/); + await assert.rejects(lstat(lockedDestination)); await rm(staleLock); + + const failedDump = join(dir, 'failed-dump'), brokenEnv = { ...env, WIKIDECK_PG_DUMP_BINARY: join(dir, 'missing-pg-dump') }; + await assert.rejects(createCoordinatedCustodyBackup(data, failedDump, brokenEnv), /dump utility failed to start/); + await assert.rejects(verifyCoordinatedCustodyBackup(failedDump), /manifest/); + // A failed external dump releases PostgreSQL's shared-writer fence; normal sandbox commands still work. + const recoveredWrite = await f.execute(other, { type: 'mock-credit' }) as CustodyActivity; + assert.equal(recoveredWrite.availableAtomicXmr, '1000000000000'); + + const mismatchedBinding = join(dir, 'mismatch-binding'), bindingPath = join(networkDir, 'custody-postgres.json'); + const originalBinding = await readFile(bindingPath, 'utf8'); + await writeFile(bindingPath, JSON.stringify({ version: 1, custodianId: randomUUID() }), { mode: 0o600 }); + await assert.rejects(createCoordinatedCustodyBackup(data, mismatchedBinding, env), /identities differ/); + await writeFile(bindingPath, originalBinding, { mode: 0o600 }); + await assert.rejects(verifyCoordinatedCustodyBackup(mismatchedBinding), /manifest/); + + const corruptDump = join(destination, 'custody.dump'); + await writeFile(corruptDump, 'corrupt'); + await assert.rejects(verifyCoordinatedCustodyBackup(destination), /hash\/size mismatch/); +}); + test('failed cutover freezes the source, rolls back the entire target, and resumes without duplicate balances', async t => { const f = await fixture(t); await f.admin.query('DELETE FROM wk_custody.custody_meta'); const dir = await mkdtemp(join(tmpdir(), 'custody-pg-failed-import-')); t.after(() => rm(dir, { recursive: true, force: true }));