DevOps/truecharts
2
0
Fork 0
Code Issues Pull Requests Actions 29 Packages Projects Releases 1 Wiki Activity

fix(clustertool): fix kubeconfig not boing loaded correctly and put some more silence into healthchecks

Browse Source
This commit is contained in:
Kjeld Schouten
2024-11-07 23:51:25 +01:00
parent 2a2519e28a
commit 5e89c6213e
3 changed files with 17 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
clustertool/pkg/gencmd/bootstrap.go
+2 -2
View File
@@ -53,9 +53,9 @@ func RunBootstrap(args []string) {
log.Info().Msgf("Bootstrap: waiting for VIP %v to come online...", helper.TalEnv["VIP_IP"]) log.Info().Msgf("Bootstrap: waiting for VIP %v to come online...", helper.TalEnv["VIP_IP"])
nodestatus.WaitForHealth(helper.TalEnv["VIP_IP"], []string{"running"}) nodestatus.WaitForHealth(helper.TalEnv["VIP_IP"], []string{"running"})
log.Info().Msgf("Bootstrap: Configuring kubectl for VIP: %v", helper.TalEnv["VIP_IP"]) log.Info().Msgf("Bootstrap: Configuring kubeconfig/kubectl for VIP: %v", helper.TalEnv["VIP_IP"])
// Ensure kubeconfig is loaded // Ensure kubeconfig is loaded
kubeconfigcmds := GenPlain("health", helper.TalEnv["VIP_IP"], extraArgs) kubeconfigcmds := GenPlain("kubeconfig", helper.TalEnv["VIP_IP"], extraArgs)
ExecCmd(kubeconfigcmds[0]) ExecCmd(kubeconfigcmds[0])
// Desired pod names // Desired pod names
clustertool/pkg/nodestatus/health.go
+11 -6
View File
@@ -15,8 +15,8 @@ func CheckHealth(node string, status string, silent bool) error {
if err != nil { if err != nil {
errstring := "healthcheck failed. status: " + string(out) + " error: " + err.Error() errstring := "healthcheck failed. status: " + string(out) + " error: " + err.Error()
if !silent { if !silent {
log.Info().Msgf("Healthcheck: check on node : failed %v", node) log.Error().Msgf("Healthcheck: check on node : failed %v", node)
log.Info().Msgf("failed with error: %s", errstring) log.Error().Msgf("failed with error: %s", errstring)
} }
log.Error().Err(err).Str("node", node).Msg("Healthcheck failed") log.Error().Err(err).Str("node", node).Msg("Healthcheck failed")
return errors.New(errstring) return errors.New(errstring)
@@ -36,17 +36,22 @@ func CheckHealth(node string, status string, silent bool) error {
response := "Healthcheck: WARN detected node " + node + " in mode " + "maintenance" + ".\nLikely a new node, so trying commands anyway. Continuing..." response := "Healthcheck: WARN detected node " + node + " in mode " + "maintenance" + ".\nLikely a new node, so trying commands anyway. Continuing..."
log.Warn().Msg(response) log.Warn().Msg(response)
} else if status == "" && strings.Contains(out, "running") { } else if status == "" && strings.Contains(out, "running") {
_, err = CheckReadyStatus(node) _, err = CheckReadyStatus(node, silent)
if err != nil { if err != nil {
errstring := "healthcheck failed. status: " + string(out) + " error: " + err.Error() errstring := "healthcheck failed. status: " + string(out) + " error: " + err.Error()
log.Error().Err(err).Str("node", node).Msg("Healthcheck failed while checking readiness")
if !silent {
log.Error().Err(err).Str("node", node).Msg("Healthcheck failed while checking readiness")
}
return errors.New(errstring) return errors.New(errstring)
} }
} else { } else {
if !silent { if !silent {
log.Info().Msgf("Healthcheck: check on node : failed %v", node) log.Info().Msgf("Healthcheck: check on node : failed %v", node)
log.Error().Str("node", node).Msg("Healthcheck failed with unexpected status")
} }
log.Error().Str("node", node).Msg("Healthcheck failed with unexpected status")
return errors.New("healthcheck failed") return errors.New("healthcheck failed")
} }
log.Debug().Str("node", node).Msg("Health check completed successfully") log.Debug().Str("node", node).Msg("Health check completed successfully")
@@ -83,7 +88,7 @@ func WaitForHealth(node string, status []string) (string, error) {
log.Debug().Str("node", node).Str("check", check).Msg("Performing initial health check") log.Debug().Str("node", node).Str("check", check).Msg("Performing initial health check")
err := CheckHealth(node, check, true) err := CheckHealth(node, check, true)
if err == nil { if err == nil {
log.Info().Str("node", node).Str("status", check).Msg("Initial health check passed") log.Debug().Str("node", node).Str("status", check).Msg("Initial health check passed")
return check, nil return check, nil
} }
} }
clustertool/pkg/nodestatus/status.go
+4 -2
View File
@@ -75,7 +75,7 @@ func CheckStatus(node string) (string, error) {
return string(out), nil return string(out), nil
} }
func CheckReadyStatus(node string) (string, error) { func CheckReadyStatus(node string, silent bool) (string, error) {
log.Info().Str("node", node).Msg("Checking node readiness status") log.Info().Str("node", node).Msg("Checking node readiness status")
argsslice := append(baseStatusCMD(node), "-o", "jsonpath={.spec.status.ready}") argsslice := append(baseStatusCMD(node), "-o", "jsonpath={.spec.status.ready}")
@@ -83,7 +83,9 @@ func CheckReadyStatus(node string) (string, error) {
if err != nil { if err != nil {
errstring := "status: " + string(out) + " error: " + err.Error() errstring := "status: " + string(out) + " error: " + err.Error()
log.Error().Msg(errstring) if !silent {
log.Error().Msg(errstring)
}
return "ERROR", errors.New(errstring) return "ERROR", errors.New(errstring)
} }
if strings.Contains(string(out), "true") { if strings.Contains(string(out), "true") {