feat(pod-nuke): ensure we dont keep runnign on already processed node failures

containers/apps/pod-nuke/includes/pod_nuke.sh

@@ -8,6 +8,8 @@ fi
 
 # Keep track of nodes that have failed
 declare -A failed_nodes
+# Keep track of nodes where pods have already been removed
+declare -A processed_nodes
 
 while true; do
     # Get all nodes and their readiness status
@@ -15,7 +17,7 @@ while true; do
         # Ensure NotReady is in the status field (handling additional conditions)
         if echo "$status" | grep -q "NotReady"; then
             # If node is newly failed, start a 15-minute timer
-            if [[ -z "${failed_nodes[$node]}" ]]; then
+            if [[ -z "${failed_nodes[$node]}" && -z "${processed_nodes[$node]}" ]]; then
                 echo "Node $node is NotReady, starting timer."
                 failed_nodes[$node]=$(date +%s)
             else
@@ -25,19 +27,27 @@ while true; do
                 elapsed=$((current_time - start_time))
 
                 if [[ $elapsed -ge 900 ]]; then
-                    echo "15 minutes elapsed, force removing pods from $node."
-                    kubectl get pods --all-namespaces --field-selector spec.nodeName==$node --no-headers | awk '{print $2 " " $1}' | while read -r pod namespace; do
-                        echo "Force deleting pod $pod in namespace $namespace on node $node"
-                        kubectl delete pod "$pod" -n "$namespace" --force --grace-period=0
-                    done
+                    echo "15 minutes elapsed, checking for running or terminating pods on $node."
+                    pods=$(kubectl get pods --all-namespaces --field-selector spec.nodeName==$node --no-headers | awk '$4 == "Running" || $4 == "Terminating" {print $2 " " $1}')
+
+                    if [[ -z "$pods" ]]; then
+                        echo "No running or terminating pods found on $node."
+                    else
+                        echo "$pods" | while read -r pod namespace; do
+                            echo "Force deleting pod $pod in namespace $namespace on node $node"
+                            kubectl delete pod "$pod" -n "$namespace" --force --grace-period=0
+                        done
+                        processed_nodes[$node]=1  # Mark node as processed after deleting pods
+                    fi
                     unset failed_nodes[$node]
                 fi
             fi
         else
-            # If node recovers, remove it from failed tracking
-            if [[ -n "${failed_nodes[$node]}" ]]; then
+            # If node recovers, remove it from failed and processed tracking
+            if [[ -n "${failed_nodes[$node]}" || -n "${processed_nodes[$node]}" ]]; then
                 echo "Node $node recovered, removing from tracking."
                 unset failed_nodes[$node]
+                unset processed_nodes[$node]
             fi
         fi
     done < <(kubectl get nodes --no-headers | awk '{print $1 " " $2 " " $3}')