From 66582331130c2f8cf743f3b36f0d9a6476262bb1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 00:21:18 -0500 Subject: [PATCH] deploy/night: rolling node restarts; defer auto-deploys while matches are live QA (night-review, #4): a simultaneous restart of all 3 nodes killed a live hosted match. Deploys now wait while any liveMatches/openLobbies > 0 (forced after FORCE_AFTER=900 s), and restart node-a, b, c one at a time, each passing /healthz before the next. Co-Authored-By: Claude Opus 5.5 --- deploy/night/autodeploy.sh | 16 ++++++++++++++-- deploy/night/testnet.sh | 19 ++++++++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/deploy/night/autodeploy.sh b/deploy/night/autodeploy.sh index 59c11a3..3d994ea 100755 --- a/deploy/night/autodeploy.sh +++ b/deploy/night/autodeploy.sh @@ -1,13 +1,17 @@ #!/bin/sh # sim-testnet: keep the wikideck-night testnet on the integration branch head. # Run from the root of a *dedicated* checkout (e.g. a detached git worktree): it checks out -# each new head of $BRANCH and runs `deploy/night/testnet.sh up` (ledgers persist in the volumes). +# each new head of $BRANCH and runs `deploy/night/testnet.sh rolling` (ledgers persist in the volumes): +# nodes restart one at a time, and deploys wait (up to FORCE_AFTER) while matches or lobbies are live, +# because lobbies are in memory and a restart ends games in progress. # Head detection uses the public Gitea API, which also works where SSH auth isn't available. set -u BRANCH=${BRANCH:-night/2026-11-16} API=${API:-https://gitea.eom.dev/api/v1/repos/claude/wikideck} INTERVAL=${INTERVAL:-180} LOG=${LOG:-/tmp/wikideck-night-autodeploy.log} +FORCE_AFTER=${FORCE_AFTER:-900} # defer while matches/lobbies are live, but at most this many seconds +deferred_since= enc=$(printf %s "$BRANCH" | sed 's#/#%2F#g') while :; do new=$(node -e "fetch('$API/branches/$enc',{signal:AbortSignal.timeout(15000)}).then(r=>r.json()).then(b=>console.log(b.commit.id)).catch(()=>{})") @@ -17,9 +21,17 @@ while :; do if ! git fetch -q origin "+refs/heads/$BRANCH:refs/remotes/origin/$BRANCH" 2>"$LOG"; then echo "WARN fetch failed: $(tail -1 "$LOG")" else + busy=$(deploy/night/testnet.sh busy 2>/dev/null || echo 0) + now=$(date +%s) + if [ "${busy:-0}" -gt 0 ] && { [ -z "$deferred_since" ] || [ $((now - deferred_since)) -lt "$FORCE_AFTER" ]; }; then + [ -z "$deferred_since" ] && { deferred_since=$now; echo "DEFER ${new%"${new#???????}"}: $busy live matches/lobbies"; } + sleep 60; continue + fi + [ -n "$deferred_since" ] && [ "${busy:-0}" -gt 0 ] && echo "FORCE deploy after ${FORCE_AFTER}s ($busy still live)" + deferred_since= git checkout -q --detach "$new" echo "DEPLOY start $(git log --oneline -1 | cut -c1-90)" - if deploy/night/testnet.sh up >"$LOG" 2>&1; then + if deploy/night/testnet.sh rolling >"$LOG" 2>&1; then sleep 20; echo "DEPLOY ok $(git rev-parse --short HEAD): $(deploy/night/testnet.sh status 2>&1 | tr '\n' ' ')" else echo "DEPLOY FAILED $(git rev-parse --short HEAD): $(tail -3 "$LOG" | tr '\n' ' ')"; fi fi diff --git a/deploy/night/testnet.sh b/deploy/night/testnet.sh index e392f4e..4d43f0b 100755 --- a/deploy/night/testnet.sh +++ b/deploy/night/testnet.sh @@ -2,6 +2,8 @@ # sim-testnet: private 3-node testnet fixture for WikiDeck Night 1 (poweredge-r350, LAN only). # Usage (from the repo root): # deploy/night/testnet.sh up build the image from this checkout and (re)start the 3 nodes +# deploy/night/testnet.sh rolling build, then restart node-a, b, c one at a time (waits for /healthz) +# deploy/night/testnet.sh busy print live matches + open lobbies across the 3 nodes # deploy/night/testnet.sh mine N [W] mine N real RandomX blocks with W workers (default 4, max 8) # deploy/night/testnet.sh status tip/height of each node # deploy/night/testnet.sh down stop the nodes (volumes, i.e. the ledgers, are kept) @@ -15,6 +17,21 @@ DC="docker compose -p $P -f compose.testnet.yaml -f deploy/night/compose.lan.yam case "${1:-}" in up) $DC build node-a && $DC up -d --no-build ;; down) $DC down ;; + rolling) + # Never take the whole testnet dark: one node at a time, each must pass /healthz before the next. + $DC build node-a + for n in a b c; do + port=$(( 3100 + $(printf %d "'$n") - 96 )) + $DC up -d --no-build --no-deps node-$n + i=0 + until node -e "fetch('http://$LAN_IP:$port/healthz',{signal:AbortSignal.timeout(3000)}).then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"; do + i=$((i+1)); [ $i -ge 60 ] && { echo "node-$n not healthy after 120 s"; exit 1; }; sleep 2 + done + echo "node-$n healthy" + done ;; + busy) + # Sum of liveMatches + openLobbies over all hosted modes on all nodes (unreachable node counts 0). + node -e "Promise.all([3101,3102,3103].map(p=>fetch('http://$LAN_IP:'+p+'/api/game/modes',{signal:AbortSignal.timeout(5000)}).then(r=>r.json()).then(m=>m.modes.reduce((t,x)=>t+(x.stats?.liveMatches??0)+(x.stats?.openLobbies??0),0)).catch(()=>0))).then(v=>console.log(v.reduce((a,b)=>a+b,0)))" ;; mine) mkdir -p "$STATE_DIR" W=${3:-4} @@ -28,5 +45,5 @@ case "${1:-}" in for port in 3101 3102 3103; do docker run --rm --network host wikideck:testnet node -e "fetch('http://$LAN_IP:$port/p2p/status').then(r=>r.json()).then(s=>console.log($port,'blocks',s.size,'tip',s.tip.slice(0,16))).catch(e=>console.log($port,'DOWN',e.message))" done ;; - *) sed -n 2,9p "$0"; exit 1 ;; + *) sed -n 2,11p "$0"; exit 1 ;; esac -- 2.54.0