feat(pod-nuke): add container to auto-nuke pods on broken nodes

containers/apps/pod-nuke/Dockerfile

@@ -0,0 +1,33 @@
+FROM tccr.io/tccr/alpine:v3.21.3@sha256:8ae22cc7c6f2dbc803e2d7e8aff41496d72a3ec4c2527501baac5f94360ed20a
+
+ARG TARGETPLATFORM
+ARG VERSION
+
+# hadolint ignore=DL3002
+USER root
+
+COPY ./containers/apps/pod_nuke/includes/pod_nuke.sh /scripts/pod_nuke.sh
+
+# hadolint ignore=DL3018,DL4006
+RUN apk update && apk add --no-cache bash \
+    && apk del curl \
+    && mkdir -p /scripts /config \
+    && chmod +x /scripts/pod_nuke.sh
+
+USER root
+
+WORKDIR /scripts
+
+ENTRYPOINT ["/bin/bash", "/scripts/pod_nuke.sh"]
+
+ARG CONTAINER_NAME
+ARG CONTAINER_VER
+LABEL "maintainer"="TrueCharts <info@truecharts.org>"
+LABEL "org.opencontainers.image.source"="https://github.com/truecharts/apps"
+LABEL org.opencontainers.image.licenses="BSD-3-Clause"
+LABEL org.opencontainers.image.title="${CONTAINER_NAME}"
+LABEL org.opencontainers.image.url="https://truecharts.org/docs/charts/${CONTAINER_NAME}"
+LABEL org.opencontainers.image.version="${CONTAINER_VER}"
+LABEL org.opencontainers.image.description="Container for ${CONTAINER_NAME} by TrueCharts"
+LABEL org.opencontainers.image.authors="TrueCharts"
+LABEL org.opencontainers.image.documentation="https://truecharts.org/docs/charts/${CONTAINER_NAME}"

containers/apps/pod-nuke/LICENSE

@@ -0,0 +1,106 @@
+Business Source License 1.1
+
+Parameters
+
+Licensor:             The TrueCharts Project, it's owner and it's contributors
+Licensed Work:        The TrueCharts "Blocky" Helm Chart
+Additional Use Grant: You may use the licensed work in production, as long
+                      as it is directly sourced from a TrueCharts provided
+                      official repository, catalog or source. You may also make private
+                      modification to the directly sourced licenced work,
+                      when used in production.
+
+                      The following cases are, due to their nature, also
+                      defined as 'production use' and explicitly prohibited:
+                      - Bundling, including or displaying the licensed work
+                      with(in) another work intended for production use,
+                      with the apparent intend of facilitating and/or
+                      promoting production use by third parties in
+                      violation of this license.
+
+Change Date:          2050-01-01
+
+Change License:       3-clause BSD license
+
+For information about alternative licensing arrangements for the Software,
+please contact: legal@truecharts.org
+
+Notice
+
+The Business Source License (this document, or the “License”) is not an Open
+Source license. However, the Licensed Work will eventually be made available
+under an Open Source License, as stated in this License.
+
+License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
+“Business Source License” is a trademark of MariaDB Corporation Ab.
+
+-----------------------------------------------------------------------------
+
+Business Source License 1.1
+
+Terms
+
+The Licensor hereby grants you the right to copy, modify, create derivative
+works, redistribute, and make non-production use of the Licensed Work. The
+Licensor may make an Additional Use Grant, above, permitting limited
+production use.
+
+Effective on the Change Date, or the fourth anniversary of the first publicly
+available distribution of a specific version of the Licensed Work under this
+License, whichever comes first, the Licensor hereby grants you rights under
+the terms of the Change License, and the rights granted in the paragraph
+above terminate.
+
+If your use of the Licensed Work does not comply with the requirements
+currently in effect as described in this License, you must purchase a
+commercial license from the Licensor, its affiliated entities, or authorized
+resellers, or you must refrain from using the Licensed Work.
+
+All copies of the original and modified Licensed Work, and derivative works
+of the Licensed Work, are subject to this License. This License applies
+separately for each version of the Licensed Work and the Change Date may vary
+for each version of the Licensed Work released by Licensor.
+
+You must conspicuously display this License on each original or modified copy
+of the Licensed Work. If you receive the Licensed Work in original or
+modified form from a third party, the terms and conditions set forth in this
+License apply to your use of that work.
+
+Any use of the Licensed Work in violation of this License will automatically
+terminate your rights under this License for the current and all other
+versions of the Licensed Work.
+
+This License does not grant you any right in any trademark or logo of
+Licensor or its affiliates (provided that you may use a trademark or logo of
+Licensor as expressly required by this License).
+
+TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
+AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
+EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
+TITLE.
+
+MariaDB hereby grants you permission to use this License’s text to license
+your works, and to refer to it using the trademark “Business Source License”,
+as long as you comply with the Covenants of Licensor below.
+
+Covenants of Licensor
+
+In consideration of the right to use this License’s text and the “Business
+Source License” name and trademark, Licensor covenants to MariaDB, and to all
+other recipients of the licensed work to be provided by Licensor:
+
+1. To specify as the Change License the GPL Version 2.0 or any later version,
+   or a license that is compatible with GPL Version 2.0 or a later version,
+   where “compatible” means that software provided under the Change License can
+   be included in a program with software provided under GPL Version 2.0 or a
+   later version. Licensor may specify additional Change Licenses without
+   limitation.
+
+2. To either: (a) specify an additional grant of rights to use that does not
+   impose any additional restriction on the right granted in this License, as
+   the Additional Use Grant; or (b) insert the text “None”.
+
+3. To specify a Change Date.
+
+4. Not to modify this License in any other way.

containers/apps/pod-nuke/PLATFORM

@@ -0,0 +1 @@
+linux/amd64

containers/apps/pod-nuke/VERSION

@@ -0,0 +1 @@
+0.0.1

containers/apps/pod-nuke/includes/pod_nuke.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Ensure we're using Bash
+if [ -z "$BASH_VERSION" ]; then
+    echo "This script requires Bash. Please install it on Alpine: apk add bash"
+    exit 1
+fi
+
+# Keep track of nodes that have failed
+declare -A failed_nodes
+
+while true; do
+    # Get all nodes and their readiness status
+    while read -r node status extra; do
+        # Ensure NotReady is in the status field (handling additional conditions)
+        if echo "$status" | grep -q "NotReady"; then
+            # If node is newly failed, start a 15-minute timer
+            if [[ -z "${failed_nodes[$node]}" ]]; then
+                echo "Node $node is NotReady, starting timer."
+                failed_nodes[$node]=$(date +%s)
+            else
+                # Check if 15 minutes have passed
+                start_time=${failed_nodes[$node]}
+                current_time=$(date +%s)
+                elapsed=$((current_time - start_time))
+
+                if [[ $elapsed -ge 900 ]]; then
+                    echo "15 minutes elapsed, force removing pods from $node."
+                    kubectl get pods --all-namespaces --field-selector spec.nodeName==$node --no-headers | awk '{print $2 " " $1}' | while read -r pod namespace; do
+                        echo "Force deleting pod $pod in namespace $namespace on node $node"
+                        kubectl delete pod "$pod" -n "$namespace" --force --grace-period=0
+                    done
+                    unset failed_nodes[$node]
+                fi
+            fi
+        else
+            # If node recovers, remove it from failed tracking
+            if [[ -n "${failed_nodes[$node]}" ]]; then
+                echo "Node $node recovered, removing from tracking."
+                unset failed_nodes[$node]
+            fi
+        fi
+    done < <(kubectl get nodes --no-headers | awk '{print $1 " " $2 " " $3}')
+
+    # Sleep for a short period before checking again
+    sleep 60
+done