Adds docs/ci-cd/soak/soak.yaml (20 soak jobs plus a control failure, for a cicd-test repo) and docs/ci-cd/soak-test.md with preconditions, pass criteria and the first evidence lining up silent job failures with runner evictions. The runbook now says to cancel orphaned in-progress jobs before rerunning. Refs DevOps/software-infrastructure#41 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
108 lines
3.4 KiB
YAML
108 lines
3.4 KiB
YAML
# S8 soak test for the shared Gitea Actions runner.
|
|
#
|
|
# Copy this file to `.gitea/workflows/soak.yaml` in a throwaway repo owned by
|
|
# `claude` (named `cicd-test-*`), then start it from the Actions tab. It is
|
|
# kept here, not in this repo's `.gitea/workflows/`, so it never runs on a push
|
|
# to software-infrastructure. Procedure: docs/ci-cd/soak-test.md.
|
|
#
|
|
# 20 soak jobs plus one control job. Each soak job pulls large images, compiles
|
|
# Go's standard library, and churns some disk, so it exercises the conditions
|
|
# under which jobs used to die without a step error. Every job prints the time
|
|
# and disk usage at the start and end of each step, so a job that stops
|
|
# reporting can be lined up against runner and kubelet events.
|
|
|
|
name: soak
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
rounds:
|
|
description: "Number of soak jobs (the acceptance test needs 20)"
|
|
default: "20"
|
|
|
|
concurrency:
|
|
group: soak-${{ github.repository }}
|
|
cancel-in-progress: false
|
|
|
|
env:
|
|
# Pinned tags; keep in step with what the runner's job images allow.
|
|
PULL_IMAGES: >-
|
|
golang:1.25.4-bookworm
|
|
node:22.21.1-bookworm
|
|
postgres:17.7-bookworm
|
|
GO_IMAGE: golang:1.25.4-bookworm
|
|
|
|
jobs:
|
|
plan:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
outputs:
|
|
rounds: ${{ steps.r.outputs.rounds }}
|
|
steps:
|
|
- id: r
|
|
run: |
|
|
n="${{ inputs.rounds }}"
|
|
case "$n" in ''|*[!0-9]*) n=20 ;; esac
|
|
[ "$n" -gt 40 ] && n=40
|
|
printf 'rounds=[%s]\n' "$(seq -s, 1 "$n")" >> "$GITHUB_OUTPUT"
|
|
|
|
soak:
|
|
needs: plan
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 45
|
|
strategy:
|
|
fail-fast: false
|
|
# Two at a time, so the runner sees concurrent jobs when its capacity
|
|
# allows it, without crowding out other repos' CI.
|
|
max-parallel: 2
|
|
matrix:
|
|
round: ${{ fromJSON(needs.plan.outputs.rounds) }}
|
|
steps:
|
|
- name: Record start
|
|
run: |
|
|
echo "round ${{ matrix.round }} start $(date -u +%FT%TZ)"
|
|
df -h / /var/lib/docker 2>/dev/null || df -h /
|
|
docker system df || true
|
|
|
|
- name: Pull large images
|
|
run: |
|
|
for img in $PULL_IMAGES; do
|
|
echo "$(date -u +%T) pull $img"
|
|
docker pull -q "$img"
|
|
done
|
|
docker system df
|
|
|
|
- name: Heavy compile
|
|
run: |
|
|
echo "$(date -u +%T) compile start"
|
|
# The daemon can't see the job's files, so nothing is bind-mounted.
|
|
docker run --rm "$GO_IMAGE" sh -c 'go build -a std && echo compiled'
|
|
echo "$(date -u +%T) compile end"
|
|
|
|
- name: Disk churn
|
|
run: |
|
|
vol="soak-${{ github.run_id }}-${{ matrix.round }}"
|
|
docker volume create "$vol" >/dev/null
|
|
docker run --rm -v "$vol:/data" "$GO_IMAGE" \
|
|
sh -c 'dd if=/dev/zero of=/data/blob bs=1M count=2048 status=none && sync && ls -lh /data'
|
|
docker volume rm "$vol" >/dev/null
|
|
df -h / /var/lib/docker 2>/dev/null || df -h /
|
|
|
|
- name: Record end
|
|
if: always()
|
|
run: |
|
|
echo "round ${{ matrix.round }} end $(date -u +%FT%TZ)"
|
|
docker system df || true
|
|
|
|
# A job that must fail at a named step. If it ever shows success, or fails
|
|
# with no step marked failed, status reporting is not accurate.
|
|
control-failure:
|
|
needs: plan
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
steps:
|
|
- name: Expected failure
|
|
run: |
|
|
echo "this step fails on purpose"
|
|
exit 3
|