From 8582c6434a940889d2850e93f7c220c82a2ff71b Mon Sep 17 00:00:00 2001 From: Kjeld Schouten Date: Tue, 22 Oct 2024 13:03:29 +0200 Subject: [PATCH] feat(clustertool/spegel): improve spegel behavior for clustertool and make kubedashboard optional on clustertool --- charts/incubator/spegel/Chart.yaml | 2 +- charts/incubator/spegel/templates/_args.tpl | 1 - clustertool/cmd/adv_fluxbootstrap.go | 2 +- clustertool/cmd/adv_testcmd.go | 2 +- clustertool/cmd/helmrelease_install.go | 2 +- clustertool/cmd/helmrelease_upgrade.go | 2 +- .../embed/generic/base/clusterenv.yaml | 10 +- .../app/helm-release.yaml | 2 +- .../system/spegel/app/helm-release.yaml | 2 +- clustertool/embed/generic/patches/all.yaml | 129 ++++++-------- clustertool/pkg/gencmd/apply.go | 2 +- clustertool/pkg/initfiles/clusterenv.go | 36 ++-- clustertool/pkg/initfiles/initfiles.go | 157 +++++++++++++----- clustertool/pkg/sops/decrypt.go | 2 +- 14 files changed, 202 insertions(+), 149 deletions(-) diff --git a/charts/incubator/spegel/Chart.yaml b/charts/incubator/spegel/Chart.yaml index 2fcd25309f8..60ffca5fa50 100644 --- a/charts/incubator/spegel/Chart.yaml +++ b/charts/incubator/spegel/Chart.yaml @@ -30,4 +30,4 @@ name: spegel sources: - https://github.com/truecharts/charts/tree/master/charts/system/spegel type: application -version: 2.0.0 +version: 2.0.1 diff --git a/charts/incubator/spegel/templates/_args.tpl b/charts/incubator/spegel/templates/_args.tpl index 270403b447c..619939ab108 100644 --- a/charts/incubator/spegel/templates/_args.tpl +++ b/charts/incubator/spegel/templates/_args.tpl @@ -13,7 +13,6 @@ - --mirror-registries - http://$(NODE_IP):{{ .Values.service.main.ports.main.hostPort }} - http://$(NODE_IP):{{ .Values.service.main.ports.main.nodePort }} - - http://$(NODE_IP):{{ .Values.service.main.ports.main.port }} {{- with .Values.spegel.additionalMirrorRegistries }} {{- range . }} - {{ . | quote }} diff --git a/clustertool/cmd/adv_fluxbootstrap.go b/clustertool/cmd/adv_fluxbootstrap.go index 34930b7a089..b8545f54ee4 100644 --- a/clustertool/cmd/adv_fluxbootstrap.go +++ b/clustertool/cmd/adv_fluxbootstrap.go @@ -21,7 +21,7 @@ var fluxbootstrap = &cobra.Command{ if err := sops.DecryptFiles(); err != nil { log.Info().Msgf("Error decrypting files: %v\n", err) } - initfiles.LoadTalEnv() + initfiles.LoadTalEnv(false) _, err := talhelperCfg.LoadAndValidateFromFile(helper.TalConfigFile, []string{helper.ClusterEnvFile}, false) if err != nil { log.Fatal().Err(err).Msg("failed to parse talconfig or talenv file: %s") diff --git a/clustertool/cmd/adv_testcmd.go b/clustertool/cmd/adv_testcmd.go index 7c99e18b96f..492ab0e5689 100644 --- a/clustertool/cmd/adv_testcmd.go +++ b/clustertool/cmd/adv_testcmd.go @@ -11,7 +11,7 @@ var testcmd = &cobra.Command{ Use: "test", Short: "test run", Run: func(cmd *cobra.Command, args []string) { - initfiles.LoadTalEnv() + initfiles.LoadTalEnv(false) // err := fluxhandler.ProcessJSONFiles("./testdata/truenas_exports") // if err != nil { // log.Info().Msg("Error:", err) diff --git a/clustertool/cmd/helmrelease_install.go b/clustertool/cmd/helmrelease_install.go index 20f23195c56..27fcb5c9432 100644 --- a/clustertool/cmd/helmrelease_install.go +++ b/clustertool/cmd/helmrelease_install.go @@ -12,7 +12,7 @@ var hrinstall = &cobra.Command{ Use: "install", Short: "install a helm-release file without flux, helm-release file needs to be called helm-release.yaml", Run: func(cmd *cobra.Command, args []string) { - initfiles.LoadTalEnv() + initfiles.LoadTalEnv(false) var dir string diff --git a/clustertool/cmd/helmrelease_upgrade.go b/clustertool/cmd/helmrelease_upgrade.go index 9ef0ad52a82..5c52d0c79f3 100644 --- a/clustertool/cmd/helmrelease_upgrade.go +++ b/clustertool/cmd/helmrelease_upgrade.go @@ -12,7 +12,7 @@ var hrupgrade = &cobra.Command{ Use: "upgrade", Short: "run helm-upgrade using a helm-release file without flux", Run: func(cmd *cobra.Command, args []string) { - initfiles.LoadTalEnv() + initfiles.LoadTalEnv(false) var dir string diff --git a/clustertool/embed/generic/base/clusterenv.yaml b/clustertool/embed/generic/base/clusterenv.yaml index 128c2c6a585..a296b8121f6 100644 --- a/clustertool/embed/generic/base/clusterenv.yaml +++ b/clustertool/embed/generic/base/clusterenv.yaml @@ -7,10 +7,12 @@ MASTER1IP: 192.168.20.210 GATEWAY: 192.168.20.1 # Defines the ip range metallb is allowed to use METALLB_RANGE: 192.168.20.211-192.168.20.219 -# Sets the IP on which Spegel using loadbalancer is made available -SPEGEL_IP: 192.168.20.211 -# Sets the Kubernetes Dashboard IP. Has to be within METALLB_RANGE and not in use -DASHBOARD_IP: 192.168.20.212 +# (Optional) Sets the Kubernetes Dashboard IP. Has to be within METALLB_RANGE and not in use +# Comment out or set to "" to disable explosing KubernetesDashboard +# DASHBOARD_IP: 192.168.20.211 +# (Optional) Sets the IP on which a Spegel using loadbalancer service is made available +# Comment out or set to "" to disable explosing Spegel via Loadbalancer (still available using nodePort by default) +#SPEGEL_IP: 192.168.20.212 # Used to automatically generate a sshkey-pair for FluxCD # Has to start with ssh:// GITHUB_REPOSITORY: "" diff --git a/clustertool/embed/generic/kubernetes/apps/kubernetes-dashboard/app/helm-release.yaml b/clustertool/embed/generic/kubernetes/apps/kubernetes-dashboard/app/helm-release.yaml index 8cdb2fffb17..4617108ddc1 100644 --- a/clustertool/embed/generic/kubernetes/apps/kubernetes-dashboard/app/helm-release.yaml +++ b/clustertool/embed/generic/kubernetes/apps/kubernetes-dashboard/app/helm-release.yaml @@ -30,7 +30,7 @@ spec: values: service: main: - type: LoadBalancer + type: '{{ if ne "${DASHBOARD_IP}" "" }}LoadBalancer{{ else }}ClusterIP{{ end }}' loadBalancerIP: ${DASHBOARD_IP} ports: main: diff --git a/clustertool/embed/generic/kubernetes/system/spegel/app/helm-release.yaml b/clustertool/embed/generic/kubernetes/system/spegel/app/helm-release.yaml index 0dec4f5d0f6..7e6dacde322 100644 --- a/clustertool/embed/generic/kubernetes/system/spegel/app/helm-release.yaml +++ b/clustertool/embed/generic/kubernetes/system/spegel/app/helm-release.yaml @@ -26,5 +26,5 @@ spec: values: service: main: - type: LoadBalancer + type: '{{ if ne "${SPEGEL_IP}" "" }}LoadBalancer{{ else }}ClusterIP{{ end }}' loadBalancerIP: ${SPEGEL_IP} diff --git a/clustertool/embed/generic/patches/all.yaml b/clustertool/embed/generic/patches/all.yaml index 56b683a106e..8db76d26fa5 100644 --- a/clustertool/embed/generic/patches/all.yaml +++ b/clustertool/embed/generic/patches/all.yaml @@ -1,58 +1,47 @@ -- op: replace - path: /machine/time - value: - "disabled": false - "servers": +machine: + time: + disabled: false + servers: - "time.cloudflare.com" -- op: add - path: /machine/kernel - value: + + kernel: modules: - - "name": "dm_thin_pool" - - "name": "dm_mod" - - "name": nvme_tcp - - "name": vfio_pci - - "name": uio_pci_generic -- op: replace - path: /cluster/proxy - value: - "disabled": true -- op: add - path: /machine/kubelet/extraArgs - value: - "rotate-server-certificates": true -- op: add - path: /machine/kubelet/extraConfig - value: - "maxPods": 250 - "shutdownGracePeriod": "15s" - "shutdownGracePeriodCriticalPods": "10s" -- op: add - path: /machine/kubelet/extraMounts - value: - - "destination": "/var/openebs/local" - "type": "bind" - "source": "/var/openebs/local" - "options": - - "bind" - - "rshared" - - "rw" - - destination: /var/lib/longhorn - type: bind - source: /var/lib/longhorn - options: - - bind - - rshared - - rw -- op: replace - path: /machine/features/hostDNS - value: - enabled: true - resolveMemberNames: true - forwardKubeDNSToHost: false -- op: add - path: /machine/sysctls - value: + - name: "dm_thin_pool" + - name: "dm_mod" + - name: "nvme_tcp" + - name: "vfio_pci" + - name: "uio_pci_generic" + + kubelet: + extraArgs: + rotate-server-certificates: true + extraConfig: + maxPods: 250 + shutdownGracePeriod: "15s" + shutdownGracePeriodCriticalPods: "10s" + extraMounts: + - destination: "/var/openebs/local" + type: "bind" + source: "/var/openebs/local" + options: + - "bind" + - "rshared" + - "rw" + - destination: "/var/lib/longhorn" + type: "bind" + source: "/var/lib/longhorn" + options: + - "bind" + - "rshared" + - "rw" + + features: + hostDNS: + enabled: true + resolveMemberNames: true + forwardKubeDNSToHost: false + + sysctls: fs.inotify.max_queued_events: "65536" fs.inotify.max_user_instances: "8192" fs.inotify.max_user_watches: "524288" @@ -60,22 +49,10 @@ net.core.wmem_max: "2500000" vm.nr_hugepages: "2048" -## TODO: Check how we can have this pass checks -# - op: add -# path: /machine/udev -# value: -# # Thunderbolt -# - ACTION=="add", SUBSYSTEM=="thunderbolt", ATTR{authorized}=="0", ATTR{authorized}="1" -# # Intel GPU -# - SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="44", MODE="0660" -# # Google Coral USB Accelerator -# - SUBSYSTEMS=="usb", ATTRS{idVendor}=="1a6e", ATTRS{idProduct}=="089a", GROUP="20", MODE="0660" -# - SUBSYSTEMS=="usb", ATTRS{idVendor}=="18d1", ATTRS{idProduct}=="9302", GROUP="20", MODE="0660" - -- op: add - path: /machine/files - value: - - content: | + files: + - path: "/etc/cri/conf.d/20-customization.part" + permissions: 0 + content: | [plugins."io.containerd.grpc.v1.cri"] enable_unprivileged_ports = true enable_unprivileged_icmp = true @@ -83,10 +60,10 @@ discard_unpacked_layers = false [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] discard_unpacked_layers = false - permissions: 0 - path: /etc/cri/conf.d/20-customization.part - op: create - - content: | + + - path: "/etc/nfsmount.conf" + permissions: 420 + content: | [ NFSMount_Global_Options ] nfsvers=4.2 hard=True @@ -95,6 +72,6 @@ rsize=131072 wsize=131072 nconnect=8 - permissions: 420 - path: /etc/nfsmount.conf - op: overwrite + + registries: + config: diff --git a/clustertool/pkg/gencmd/apply.go b/clustertool/pkg/gencmd/apply.go index cbdfadb6cd5..88e766c744e 100644 --- a/clustertool/pkg/gencmd/apply.go +++ b/clustertool/pkg/gencmd/apply.go @@ -15,7 +15,7 @@ import ( ) func GenApply(node string, extraFlags []string) []string { - initfiles.LoadTalEnv() + initfiles.LoadTalEnv(false) cfg, err := talhelperCfg.LoadAndValidateFromFile(helper.TalConfigFile, []string{helper.ClusterEnvFile}, false) if err != nil { log.Fatal().Err(err).Msg("failed to parse talconfig or talenv file: %s") diff --git a/clustertool/pkg/initfiles/clusterenv.go b/clustertool/pkg/initfiles/clusterenv.go index e94548a64ed..1b366fdbf1f 100644 --- a/clustertool/pkg/initfiles/clusterenv.go +++ b/clustertool/pkg/initfiles/clusterenv.go @@ -13,8 +13,8 @@ import ( "github.com/truecharts/public/clustertool/pkg/helper" ) -func LoadTalEnv() error { - // Check if talenv.yaml file exists +func LoadTalEnv(noFail bool) error { + // Check if clusterenv.yaml file exists if _, err := os.Stat(helper.ClusterPath + "/clusterenv.yaml"); err == nil { // Load environment variables from clusterenv.yaml err := helper.LoadEnvFromFile(helper.ClusterPath+"/clusterenv.yaml", helper.TalEnv) @@ -23,11 +23,20 @@ func LoadTalEnv() error { os.Exit(1) } } else if os.IsNotExist(err) { - log.Info().Msg("clusterenv.yaml file not found, skipping environment variable loading.") + // If the file doesn't exist, check noFail to determine next steps + if noFail { + log.Info().Msg("clusterenv.yaml file not found, but skipping due to noFail being true.") + return nil // Skip execution without error + } else { + log.Info().Msg("clusterenv.yaml file not found, exiting as noFail is false.") + os.Exit(1) // Exit with error code 1 + } } else { log.Info().Msgf("Error checking clusterenv.yaml file: %v\n", err) os.Exit(1) } + + // If file exists, continue with processing clusterName() checkQuotedNumbersInFile() PostProcessTalEnv() @@ -214,14 +223,13 @@ func normalizeIPNetmask(ipNetmask string) (string, error) { } func CheckEnvVariables() { - LoadTalEnv() + LoadTalEnv(false) requiredKeys := []string{ "VIP", "MASTER1IP_IP", "MASTER1IP_NETMASK", "GATEWAY", "METALLB_RANGE", - "DASHBOARD_IP_IP", "PODNET", "SVCNET", } @@ -282,14 +290,16 @@ func CheckEnvVariables() { } // Check DASHBOARD_IP against METALLB_RANGE - inRange, err = helper.IPInRange(helper.TalEnv["DASHBOARD_IP"], helper.TalEnv["METALLB_RANGE"]) - if err != nil { - log.Info().Msgf("Error checking DASHBOARD_IP against METALLB_RANGE: %v\n", err) - os.Exit(1) - } - if !inRange { - log.Info().Msg("Cannot proceed, DASHBOARD_IP must be in the METALLB_RANGE") - os.Exit(1) + if helper.TalEnv["DASHBOARD_IP"] != "" { + inRange, err = helper.IPInRange(helper.TalEnv["DASHBOARD_IP"], helper.TalEnv["METALLB_RANGE"]) + if err != nil { + log.Info().Msgf("Error checking DASHBOARD_IP against METALLB_RANGE: %v\n", err) + os.Exit(1) + } + if !inRange { + log.Info().Msg("Cannot proceed, DASHBOARD_IP must be in the METALLB_RANGE") + os.Exit(1) + } } // Validate other CIDR/IP checks with new netmask support diff --git a/clustertool/pkg/initfiles/initfiles.go b/clustertool/pkg/initfiles/initfiles.go index c4081194274..0072e7ef6ad 100644 --- a/clustertool/pkg/initfiles/initfiles.go +++ b/clustertool/pkg/initfiles/initfiles.go @@ -151,6 +151,7 @@ func genBaseFiles() error { } func UpdateBaseFiles() error { + log.Info().Msg("Updating Base files for cluster: helper.ClusterPath") // Read filenames in source directory sourceFiles, err := readFilenamesInDir(helper.BaseCache) if err != nil { @@ -244,7 +245,7 @@ func readFilenamesInDir(dir string) ([]string, error) { } func ResetBootstrapValues() error { - LoadTalEnv() + LoadTalEnv(false) err := helper.CopyDirFiltered(helper.KubeCache, helper.ClusterPath+"/kubernetes", true, `^bootstrap-values\.yaml.ct$`) if err != nil { log.Info().Msg("Error:") @@ -309,18 +310,15 @@ func setDocker() { if helper.TalEnv["DOCKERHUB_USER"] != "" && helper.TalEnv["DOCKERHUB_PASSWORD"] != "" { // Prepare the content to append configContent := fmt.Sprintf(`# Add Dockerhub Login - - operation: replace - path: /machine/registries/config/registry-1.docker.io - value: + registry-1.docker.io: auth: username: %s password: %s - - operation: replace - path: /machine/registries/config/docker.io - value: + docker.io: auth: username: %s password: %s + `, helper.TalEnv["DOCKERHUB_USER"], helper.TalEnv["DOCKERHUB_PASSWORD"], helper.TalEnv["DOCKERHUB_USER"], helper.TalEnv["DOCKERHUB_PASSWORD"]) // Open the file in append mode or create it if it doesn't exist @@ -355,42 +353,62 @@ func setSpegel() { if helper.TalEnv["SPEGEL_IP"] != "" && helper.TalEnv["SPEGEL_IP"] != "" { // Prepare the content to append configContent := fmt.Sprintf(`# Add Dockerhub Login -- operation: replace - path: /machine/registries/mirrors/ - value: - cgr.dev: - endpoints: - - http://%s:5000 - docker.io: - endpoints: - - http://%s:5000 - ghcr.io: - endpoints: - - http://%s:5000 - quay.io: - endpoints: - - http://%s:5000 - mcr.microsoft.com: - endpoints: - - http://%s:5000 - public.ecr.aws: - endpoints: - - http://%s:5000 - gcr.io: - endpoints: - - http://%s:5000 - registry.k8s.io: - endpoints: - - http://%s:5000 - k8s.gcr.io: - endpoints: - - http://%s:5000 - tccr.io: - endpoints: - - http://%s:5000 - factory.talos.dev: - endpoints: - - http://%s:5000 + mirrors: + cgr.dev: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + - http://%s:5000 + docker.io: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + - http://%s:5000 + ghcr.io: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + - http://%s:5000 + quay.io: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + - http://%s:5000 + mcr.microsoft.com: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + - http://%s:5000 + public.ecr.aws: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + - http://%s:5000 + gcr.io: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + - http://%s:5000 + registry.k8s.io: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + - http://%s:5000 + k8s.gcr.io: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + - http://%s:5000 + tccr.io: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + - http://%s:5000 + factory.talos.dev: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + - http://%s:5000 `, helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"]) @@ -407,15 +425,62 @@ func setSpegel() { } } else { // Optional: Append a note if the environment variables are not set - emptyContent := `# No Spegel_IP provided -` + configContent := fmt.Sprintf(`# No Spegel_IP provide + mirrors: + cgr.dev: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + docker.io: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + ghcr.io: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + quay.io: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + mcr.microsoft.com: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + public.ecr.aws: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + gcr.io: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + registry.k8s.io: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + k8s.gcr.io: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + tccr.io: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + factory.talos.dev: + endpoints: + - http://127.0.0.1:30020 + - http://127.0.0.1:30021 + +`) + file, err := os.OpenFile(filepath.Join(helper.ClusterPath+"/talos/patches", "all.yaml"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { log.Fatal().Err(err).Msg("Error opening file: %s") } defer file.Close() - if _, err := file.Write([]byte(emptyContent)); err != nil { + if _, err := file.Write([]byte(configContent)); err != nil { log.Fatal().Err(err).Msg("Error writing to file: %s") } } diff --git a/clustertool/pkg/sops/decrypt.go b/clustertool/pkg/sops/decrypt.go index 5c5e2051113..cb386628649 100644 --- a/clustertool/pkg/sops/decrypt.go +++ b/clustertool/pkg/sops/decrypt.go @@ -55,7 +55,7 @@ func DecryptFiles() error { if !encryptedFound { log.Info().Msg("Nothing to decrypt") } - initfiles.LoadTalEnv() + initfiles.LoadTalEnv(true) return nil }