Runner: prune Docker data in DinD on a schedule
Demo / hello (push) Successful in 25s

Add a docker-prune native sidecar (docker:29.5.2-cli, matching the
deployed DinD) that runs files/docker-prune.sh against DinD's socket
every 15 minutes. It removes stopped containers, unused volumes,
dangling images and build cache older than 6 hours, and once Docker
uses more than 30 GB it removes every unused image except the pinned
job images. Running containers and anything a container uses are
never touched. Settings are gitea_actions_prune_* defaults.

Refs DevOps/software-infrastructure#38

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-11 12:24:55 -05:00
co-authored by Claude Opus 5.5
parent 5fb2d9c7cb
commit 4d7b0b9e01
3 changed files with 145 additions and 0 deletions
+16
View File
@@ -59,6 +59,22 @@ gitea_actions_docker_data_size: 50Gi
gitea_actions_dind_ephemeral_request: 20Gi
gitea_actions_dind_ephemeral_limit: 55Gi
# Docker cleanup (S5). A sidecar runs files/docker-prune.sh against DinD every
# interval. It removes stopped containers, unused volumes, dangling images and
# build cache older than max_age, and when Docker uses more than
# images_above_gb it also removes every unused image except the job images
# below. Nothing a container uses, running or stopped, is ever removed.
# images_above_gb is decimal GB as `docker system df` reports it; 30 GB keeps
# well clear of the 50Gi docker-data cap with room for one large job.
gitea_actions_prune_image: docker.io/library/docker:29.5.2-cli
gitea_actions_prune_interval_seconds: 900
gitea_actions_prune_max_age_minutes: 360
gitea_actions_prune_images_above_gb: 30
gitea_actions_prune_keep_images: >-
{{ gitea_actions_runner_labels
| map('regex_replace', '^[^:]+:docker://', '')
| unique | list }}
# Runner and DinD CPU and memory. Sized for a 2-CPU, 16 GB worker (S3's
# recommended node) running one job at a time. The runner process itself is
# light; DinD holds the job containers, so it gets nearly all of the budget.
+101
View File
@@ -0,0 +1,101 @@
#!/bin/sh
# Prunes Docker data in the runner's Docker-in-Docker so the docker-data
# emptyDir stays below its sizeLimit. Runs as a sidecar next to DinD.
#
# Never touches anything a job is using:
# - only stopped containers are pruned (Docker never prunes running ones);
# - volumes are removed only when no container references them, and Docker
# refuses to remove a volume that is in use;
# - images are removed with a plain `docker image rm`, which fails for any
# image a container (running or stopped) uses, and images in KEEP are skipped.
#
# Settings (environment):
# PRUNE_INTERVAL_SECONDS seconds between passes [900]
# PRUNE_MAX_AGE_MINUTES age before stopped containers, unused
# volumes, dangling images and build
# cache are removed [360]
# PRUNE_IMAGES_ABOVE_GB when Docker uses more than this, remove
# every unused image not in KEEP [30]
# PRUNE_KEEP_IMAGES space-separated image refs never removed,
# normally the runner's job images []
# PRUNE_ONCE set to 1 to run one pass and exit (tests)
set -u
interval=${PRUNE_INTERVAL_SECONDS:-900}
max_age=${PRUNE_MAX_AGE_MINUTES:-360}
images_above_gb=${PRUNE_IMAGES_ABOVE_GB:-30}
keep=${PRUNE_KEEP_IMAGES:-}
log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) docker-prune: $*"; }
# Bytes Docker uses for images, containers, volumes and build cache.
docker_bytes() {
docker system df --format '{{.Size}}' | awk '
function bytes(s, n, u) {
n = s + 0; u = s; sub(/^[0-9.]+/, "", u)
if (u == "kB" || u == "KB") return n * 1000
if (u == "MB") return n * 1000 ^ 2
if (u == "GB") return n * 1000 ^ 3
if (u == "TB") return n * 1000 ^ 4
return n
}
{ total += bytes($1) }
END { printf "%.0f\n", total }'
}
gb() { awk -v b="$1" 'BEGIN { printf "%.1f", b / 1000 ^ 3 }'; }
is_kept() {
for k in $keep; do
[ "$1" = "$k" ] && return 0
done
return 1
}
prune_volumes() {
now=$(date -u +%s)
docker volume ls -q --filter dangling=true | while read -r v; do
created=$(docker volume inspect -f '{{.CreatedAt}}' "$v" 2>/dev/null) || continue
# CreatedAt is RFC 3339; keep the date and time to the second.
created=$(date -u -D '%Y-%m-%dT%H:%M:%S' -d "$(echo "$created" | cut -c1-19)" +%s 2>/dev/null) || continue
if [ $((now - created)) -ge $((max_age * 60)) ]; then
docker volume rm "$v" >/dev/null 2>&1 && log "removed volume $v"
fi
done
}
prune_images_over_threshold() {
used=$(docker_bytes)
limit=$(awk -v g="$images_above_gb" 'BEGIN { printf "%.0f\n", g * 1000 ^ 3 }')
[ "$used" -gt "$limit" ] || return 0
log "using $(gb "$used") GB, above ${images_above_gb} GB: removing unused images"
docker image ls --format '{{.ID}} {{.Repository}}:{{.Tag}}' | while read -r id ref; do
is_kept "$ref" && continue
# Fails, and is skipped, for any image a container uses.
docker image rm "$ref" >/dev/null 2>&1 && log "removed image $ref" && continue
case $ref in *'<none>'*) docker image rm "$id" >/dev/null 2>&1 && log "removed image $id" ;; esac
done
}
pass() {
before=$(docker_bytes)
docker container prune -f --filter "until=${max_age}m" >/dev/null
prune_volumes
docker image prune -f --filter "until=${max_age}m" >/dev/null
docker builder prune -f --filter "until=${max_age}m" >/dev/null 2>&1
prune_images_over_threshold
after=$(docker_bytes)
log "pass done: $(gb "$before") GB -> $(gb "$after") GB"
}
until docker info >/dev/null 2>&1; do
log "waiting for Docker"
sleep 5
done
log "started: every ${interval}s, max age ${max_age}m, images above ${images_above_gb} GB, keep: ${keep:-none}"
while :; do
pass
[ "${PRUNE_ONCE:-0}" = 1 ] && exit 0
sleep "$interval"
done
+28
View File
@@ -73,6 +73,34 @@
giteaRootURL: https://gitea.eom.dev
existingSecret: "{{ gitea_actions_runner_token_secret }}"
existingSecretKey: token
# Native sidecar after DinD: starts once Docker's socket is up and stops
# before it. See gitea_actions_prune_* in defaults for what it removes.
postExtraInitContainers:
- name: docker-prune
image: "{{ gitea_actions_prune_image }}"
restartPolicy: Always
command: ["/bin/sh", "-c", "{{ lookup('ansible.builtin.file', 'docker-prune.sh') }}"]
env:
- name: DOCKER_HOST
value: unix:///run/dind/docker.sock
- name: PRUNE_INTERVAL_SECONDS
value: "{{ gitea_actions_prune_interval_seconds | string }}"
- name: PRUNE_MAX_AGE_MINUTES
value: "{{ gitea_actions_prune_max_age_minutes | string }}"
- name: PRUNE_IMAGES_ABOVE_GB
value: "{{ gitea_actions_prune_images_above_gb | string }}"
- name: PRUNE_KEEP_IMAGES
value: "{{ gitea_actions_prune_keep_images | join(' ') }}"
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 200m
memory: 128Mi
volumeMounts:
- name: docker-socket
mountPath: /run/dind
statefulset:
replicas: "{{ gitea_actions_runner_replicas }}"
persistence: