Add a docker-prune native sidecar (docker:29.5.2-cli, matching the deployed DinD) that runs files/docker-prune.sh against DinD's socket every 15 minutes. It removes stopped containers, unused volumes, dangling images and build cache older than 6 hours, and once Docker uses more than 30 GB it removes every unused image except the pinned job images. Running containers and anything a container uses are never touched. Settings are gitea_actions_prune_* defaults. Refs DevOps/software-infrastructure#38 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -59,6 +59,22 @@ gitea_actions_docker_data_size: 50Gi
|
||||
gitea_actions_dind_ephemeral_request: 20Gi
|
||||
gitea_actions_dind_ephemeral_limit: 55Gi
|
||||
|
||||
# Docker cleanup (S5). A sidecar runs files/docker-prune.sh against DinD every
|
||||
# interval. It removes stopped containers, unused volumes, dangling images and
|
||||
# build cache older than max_age, and when Docker uses more than
|
||||
# images_above_gb it also removes every unused image except the job images
|
||||
# below. Nothing a container uses, running or stopped, is ever removed.
|
||||
# images_above_gb is decimal GB as `docker system df` reports it; 30 GB keeps
|
||||
# well clear of the 50Gi docker-data cap with room for one large job.
|
||||
gitea_actions_prune_image: docker.io/library/docker:29.5.2-cli
|
||||
gitea_actions_prune_interval_seconds: 900
|
||||
gitea_actions_prune_max_age_minutes: 360
|
||||
gitea_actions_prune_images_above_gb: 30
|
||||
gitea_actions_prune_keep_images: >-
|
||||
{{ gitea_actions_runner_labels
|
||||
| map('regex_replace', '^[^:]+:docker://', '')
|
||||
| unique | list }}
|
||||
|
||||
# Runner and DinD CPU and memory. Sized for a 2-CPU, 16 GB worker (S3's
|
||||
# recommended node) running one job at a time. The runner process itself is
|
||||
# light; DinD holds the job containers, so it gets nearly all of the budget.
|
||||
|
||||
Executable
+101
@@ -0,0 +1,101 @@
|
||||
#!/bin/sh
|
||||
# Prunes Docker data in the runner's Docker-in-Docker so the docker-data
|
||||
# emptyDir stays below its sizeLimit. Runs as a sidecar next to DinD.
|
||||
#
|
||||
# Never touches anything a job is using:
|
||||
# - only stopped containers are pruned (Docker never prunes running ones);
|
||||
# - volumes are removed only when no container references them, and Docker
|
||||
# refuses to remove a volume that is in use;
|
||||
# - images are removed with a plain `docker image rm`, which fails for any
|
||||
# image a container (running or stopped) uses, and images in KEEP are skipped.
|
||||
#
|
||||
# Settings (environment):
|
||||
# PRUNE_INTERVAL_SECONDS seconds between passes [900]
|
||||
# PRUNE_MAX_AGE_MINUTES age before stopped containers, unused
|
||||
# volumes, dangling images and build
|
||||
# cache are removed [360]
|
||||
# PRUNE_IMAGES_ABOVE_GB when Docker uses more than this, remove
|
||||
# every unused image not in KEEP [30]
|
||||
# PRUNE_KEEP_IMAGES space-separated image refs never removed,
|
||||
# normally the runner's job images []
|
||||
# PRUNE_ONCE set to 1 to run one pass and exit (tests)
|
||||
set -u
|
||||
|
||||
interval=${PRUNE_INTERVAL_SECONDS:-900}
|
||||
max_age=${PRUNE_MAX_AGE_MINUTES:-360}
|
||||
images_above_gb=${PRUNE_IMAGES_ABOVE_GB:-30}
|
||||
keep=${PRUNE_KEEP_IMAGES:-}
|
||||
|
||||
log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) docker-prune: $*"; }
|
||||
|
||||
# Bytes Docker uses for images, containers, volumes and build cache.
|
||||
docker_bytes() {
|
||||
docker system df --format '{{.Size}}' | awk '
|
||||
function bytes(s, n, u) {
|
||||
n = s + 0; u = s; sub(/^[0-9.]+/, "", u)
|
||||
if (u == "kB" || u == "KB") return n * 1000
|
||||
if (u == "MB") return n * 1000 ^ 2
|
||||
if (u == "GB") return n * 1000 ^ 3
|
||||
if (u == "TB") return n * 1000 ^ 4
|
||||
return n
|
||||
}
|
||||
{ total += bytes($1) }
|
||||
END { printf "%.0f\n", total }'
|
||||
}
|
||||
|
||||
gb() { awk -v b="$1" 'BEGIN { printf "%.1f", b / 1000 ^ 3 }'; }
|
||||
|
||||
is_kept() {
|
||||
for k in $keep; do
|
||||
[ "$1" = "$k" ] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
prune_volumes() {
|
||||
now=$(date -u +%s)
|
||||
docker volume ls -q --filter dangling=true | while read -r v; do
|
||||
created=$(docker volume inspect -f '{{.CreatedAt}}' "$v" 2>/dev/null) || continue
|
||||
# CreatedAt is RFC 3339; keep the date and time to the second.
|
||||
created=$(date -u -D '%Y-%m-%dT%H:%M:%S' -d "$(echo "$created" | cut -c1-19)" +%s 2>/dev/null) || continue
|
||||
if [ $((now - created)) -ge $((max_age * 60)) ]; then
|
||||
docker volume rm "$v" >/dev/null 2>&1 && log "removed volume $v"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
prune_images_over_threshold() {
|
||||
used=$(docker_bytes)
|
||||
limit=$(awk -v g="$images_above_gb" 'BEGIN { printf "%.0f\n", g * 1000 ^ 3 }')
|
||||
[ "$used" -gt "$limit" ] || return 0
|
||||
log "using $(gb "$used") GB, above ${images_above_gb} GB: removing unused images"
|
||||
docker image ls --format '{{.ID}} {{.Repository}}:{{.Tag}}' | while read -r id ref; do
|
||||
is_kept "$ref" && continue
|
||||
# Fails, and is skipped, for any image a container uses.
|
||||
docker image rm "$ref" >/dev/null 2>&1 && log "removed image $ref" && continue
|
||||
case $ref in *'<none>'*) docker image rm "$id" >/dev/null 2>&1 && log "removed image $id" ;; esac
|
||||
done
|
||||
}
|
||||
|
||||
pass() {
|
||||
before=$(docker_bytes)
|
||||
docker container prune -f --filter "until=${max_age}m" >/dev/null
|
||||
prune_volumes
|
||||
docker image prune -f --filter "until=${max_age}m" >/dev/null
|
||||
docker builder prune -f --filter "until=${max_age}m" >/dev/null 2>&1
|
||||
prune_images_over_threshold
|
||||
after=$(docker_bytes)
|
||||
log "pass done: $(gb "$before") GB -> $(gb "$after") GB"
|
||||
}
|
||||
|
||||
until docker info >/dev/null 2>&1; do
|
||||
log "waiting for Docker"
|
||||
sleep 5
|
||||
done
|
||||
log "started: every ${interval}s, max age ${max_age}m, images above ${images_above_gb} GB, keep: ${keep:-none}"
|
||||
|
||||
while :; do
|
||||
pass
|
||||
[ "${PRUNE_ONCE:-0}" = 1 ] && exit 0
|
||||
sleep "$interval"
|
||||
done
|
||||
@@ -73,6 +73,34 @@
|
||||
giteaRootURL: https://gitea.eom.dev
|
||||
existingSecret: "{{ gitea_actions_runner_token_secret }}"
|
||||
existingSecretKey: token
|
||||
# Native sidecar after DinD: starts once Docker's socket is up and stops
|
||||
# before it. See gitea_actions_prune_* in defaults for what it removes.
|
||||
postExtraInitContainers:
|
||||
- name: docker-prune
|
||||
image: "{{ gitea_actions_prune_image }}"
|
||||
restartPolicy: Always
|
||||
command: ["/bin/sh", "-c", "{{ lookup('ansible.builtin.file', 'docker-prune.sh') }}"]
|
||||
env:
|
||||
- name: DOCKER_HOST
|
||||
value: unix:///run/dind/docker.sock
|
||||
- name: PRUNE_INTERVAL_SECONDS
|
||||
value: "{{ gitea_actions_prune_interval_seconds | string }}"
|
||||
- name: PRUNE_MAX_AGE_MINUTES
|
||||
value: "{{ gitea_actions_prune_max_age_minutes | string }}"
|
||||
- name: PRUNE_IMAGES_ABOVE_GB
|
||||
value: "{{ gitea_actions_prune_images_above_gb | string }}"
|
||||
- name: PRUNE_KEEP_IMAGES
|
||||
value: "{{ gitea_actions_prune_keep_images | join(' ') }}"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 128Mi
|
||||
volumeMounts:
|
||||
- name: docker-socket
|
||||
mountPath: /run/dind
|
||||
statefulset:
|
||||
replicas: "{{ gitea_actions_runner_replicas }}"
|
||||
persistence:
|
||||
|
||||
Reference in New Issue
Block a user