DevOps/truecharts
2
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases 1 Wiki Activity

feat(clustertool/spegel): improve spegel behavior for clustertool and make kubedashboard optional on clustertool

Browse Source
This commit is contained in:
Kjeld Schouten
2024-10-22 13:03:29 +02:00
parent 53fa3e02ad
commit 8582c6434a
14 changed files with 202 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
charts/incubator/spegel/Chart.yaml
+1 -1
View File
@@ -30,4 +30,4 @@ name: spegel
sources:
- https://github.com/truecharts/charts/tree/master/charts/system/spegel
type: application
version: 2.0.0
version: 2.0.1
charts/incubator/spegel/templates/_args.tpl
-1
View File
@@ -13,7 +13,6 @@
- --mirror-registries
- http://$(NODE_IP):{{ .Values.service.main.ports.main.hostPort }}
- http://$(NODE_IP):{{ .Values.service.main.ports.main.nodePort }}
- http://$(NODE_IP):{{ .Values.service.main.ports.main.port }}
{{- with .Values.spegel.additionalMirrorRegistries }}
{{- range . }}
- {{ . | quote }}
clustertool/cmd/adv_fluxbootstrap.go
+1 -1
View File
@@ -21,7 +21,7 @@ var fluxbootstrap = &cobra.Command{
if err := sops.DecryptFiles(); err != nil {
log.Info().Msgf("Error decrypting files: %v\n", err)
}
initfiles.LoadTalEnv()
initfiles.LoadTalEnv(false)
_, err := talhelperCfg.LoadAndValidateFromFile(helper.TalConfigFile, []string{helper.ClusterEnvFile}, false)
if err != nil {
log.Fatal().Err(err).Msg("failed to parse talconfig or talenv file: %s")
clustertool/cmd/adv_testcmd.go
+1 -1
View File
@@ -11,7 +11,7 @@ var testcmd = &cobra.Command{
Use: "test",
Short: "test run",
Run: func(cmd *cobra.Command, args []string) {
initfiles.LoadTalEnv()
initfiles.LoadTalEnv(false)
// err := fluxhandler.ProcessJSONFiles("./testdata/truenas_exports")
// if err != nil {
// log.Info().Msg("Error:", err)
clustertool/cmd/helmrelease_install.go
+1 -1
View File
@@ -12,7 +12,7 @@ var hrinstall = &cobra.Command{
Use: "install",
Short: "install a helm-release file without flux, helm-release file needs to be called helm-release.yaml",
Run: func(cmd *cobra.Command, args []string) {
initfiles.LoadTalEnv()
initfiles.LoadTalEnv(false)
var dir string
clustertool/cmd/helmrelease_upgrade.go
+1 -1
View File
@@ -12,7 +12,7 @@ var hrupgrade = &cobra.Command{
Use: "upgrade",
Short: "run helm-upgrade using a helm-release file without flux",
Run: func(cmd *cobra.Command, args []string) {
initfiles.LoadTalEnv()
initfiles.LoadTalEnv(false)
var dir string
clustertool/embed/generic/base/clusterenv.yaml
+6 -4
View File
@@ -7,10 +7,12 @@ MASTER1IP: 192.168.20.210
GATEWAY: 192.168.20.1
# Defines the ip range metallb is allowed to use
METALLB_RANGE: 192.168.20.211-192.168.20.219
# Sets the IP on which Spegel using loadbalancer is made available
SPEGEL_IP: 192.168.20.211
# Sets the Kubernetes Dashboard IP. Has to be within METALLB_RANGE and not in use
DASHBOARD_IP: 192.168.20.212
# (Optional) Sets the Kubernetes Dashboard IP. Has to be within METALLB_RANGE and not in use
# Comment out or set to "" to disable explosing KubernetesDashboard
# DASHBOARD_IP: 192.168.20.211
# (Optional) Sets the IP on which a Spegel using loadbalancer service is made available
# Comment out or set to "" to disable explosing Spegel via Loadbalancer (still available using nodePort by default)
#SPEGEL_IP: 192.168.20.212
# Used to automatically generate a sshkey-pair for FluxCD
# Has to start with ssh://
GITHUB_REPOSITORY: ""
clustertool/embed/generic/kubernetes/apps/kubernetes-dashboard/app/helm-release.yaml
+1 -1
View File
@@ -30,7 +30,7 @@ spec:
values:
service:
main:
type: LoadBalancer
type: '{{ if ne "${DASHBOARD_IP}" "" }}LoadBalancer{{ else }}ClusterIP{{ end }}'
loadBalancerIP: ${DASHBOARD_IP}
ports:
main:
clustertool/embed/generic/kubernetes/system/spegel/app/helm-release.yaml
+1 -1
View File
@@ -26,5 +26,5 @@ spec:
values:
service:
main:
type: LoadBalancer
type: '{{ if ne "${SPEGEL_IP}" "" }}LoadBalancer{{ else }}ClusterIP{{ end }}'
loadBalancerIP: ${SPEGEL_IP}
clustertool/embed/generic/patches/all.yaml
+53 -76
View File
@@ -1,58 +1,47 @@
- op: replace
path: /machine/time
value:
"disabled": false
"servers":
machine:
time:
disabled: false
servers:
- "time.cloudflare.com"
- op: add
path: /machine/kernel
value:
kernel:
modules:
- "name": "dm_thin_pool"
- "name": "dm_mod"
- "name": nvme_tcp
- "name": vfio_pci
- "name": uio_pci_generic
- op: replace
path: /cluster/proxy
value:
"disabled": true
- op: add
path: /machine/kubelet/extraArgs
value:
"rotate-server-certificates": true
- op: add
path: /machine/kubelet/extraConfig
value:
"maxPods": 250
"shutdownGracePeriod": "15s"
"shutdownGracePeriodCriticalPods": "10s"
- op: add
path: /machine/kubelet/extraMounts
value:
- "destination": "/var/openebs/local"
"type": "bind"
"source": "/var/openebs/local"
"options":
- "bind"
- "rshared"
- "rw"
- destination: /var/lib/longhorn
type: bind
source: /var/lib/longhorn
options:
- bind
- rshared
- rw
- op: replace
path: /machine/features/hostDNS
value:
enabled: true
resolveMemberNames: true
forwardKubeDNSToHost: false
- op: add
path: /machine/sysctls
value:
- name: "dm_thin_pool"
- name: "dm_mod"
- name: "nvme_tcp"
- name: "vfio_pci"
- name: "uio_pci_generic"
kubelet:
extraArgs:
rotate-server-certificates: true
extraConfig:
maxPods: 250
shutdownGracePeriod: "15s"
shutdownGracePeriodCriticalPods: "10s"
extraMounts:
- destination: "/var/openebs/local"
type: "bind"
source: "/var/openebs/local"
options:
- "bind"
- "rshared"
- "rw"
- destination: "/var/lib/longhorn"
type: "bind"
source: "/var/lib/longhorn"
options:
- "bind"
- "rshared"
- "rw"
features:
hostDNS:
enabled: true
resolveMemberNames: true
forwardKubeDNSToHost: false
sysctls:
fs.inotify.max_queued_events: "65536"
fs.inotify.max_user_instances: "8192"
fs.inotify.max_user_watches: "524288"
@@ -60,22 +49,10 @@
net.core.wmem_max: "2500000"
vm.nr_hugepages: "2048"
## TODO: Check how we can have this pass checks
# - op: add
# path: /machine/udev
# value:
# # Thunderbolt
# - ACTION=="add", SUBSYSTEM=="thunderbolt", ATTR{authorized}=="0", ATTR{authorized}="1"
# # Intel GPU
# - SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="44", MODE="0660"
# # Google Coral USB Accelerator
# - SUBSYSTEMS=="usb", ATTRS{idVendor}=="1a6e", ATTRS{idProduct}=="089a", GROUP="20", MODE="0660"
# - SUBSYSTEMS=="usb", ATTRS{idVendor}=="18d1", ATTRS{idProduct}=="9302", GROUP="20", MODE="0660"
- op: add
path: /machine/files
value:
- content: |
files:
- path: "/etc/cri/conf.d/20-customization.part"
permissions: 0
content: |
[plugins."io.containerd.grpc.v1.cri"]
enable_unprivileged_ports = true
enable_unprivileged_icmp = true
@@ -83,10 +60,10 @@
discard_unpacked_layers = false
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
discard_unpacked_layers = false
permissions: 0
path: /etc/cri/conf.d/20-customization.part
op: create
- content: |
- path: "/etc/nfsmount.conf"
permissions: 420
content: |
[ NFSMount_Global_Options ]
nfsvers=4.2
hard=True
@@ -95,6 +72,6 @@
rsize=131072
wsize=131072
nconnect=8
permissions: 420
path: /etc/nfsmount.conf
op: overwrite
registries:
config:
clustertool/pkg/gencmd/apply.go
+1 -1
View File
@@ -15,7 +15,7 @@ import (
)
func GenApply(node string, extraFlags []string) []string {
initfiles.LoadTalEnv()
initfiles.LoadTalEnv(false)
cfg, err := talhelperCfg.LoadAndValidateFromFile(helper.TalConfigFile, []string{helper.ClusterEnvFile}, false)
if err != nil {
log.Fatal().Err(err).Msg("failed to parse talconfig or talenv file: %s")
clustertool/pkg/initfiles/clusterenv.go
+23 -13
View File
@@ -13,8 +13,8 @@ import (
"github.com/truecharts/public/clustertool/pkg/helper"
)
func LoadTalEnv() error {
// Check if talenv.yaml file exists
func LoadTalEnv(noFail bool) error {
// Check if clusterenv.yaml file exists
if _, err := os.Stat(helper.ClusterPath + "/clusterenv.yaml"); err == nil {
// Load environment variables from clusterenv.yaml
err := helper.LoadEnvFromFile(helper.ClusterPath+"/clusterenv.yaml", helper.TalEnv)
@@ -23,11 +23,20 @@ func LoadTalEnv() error {
os.Exit(1)
}
} else if os.IsNotExist(err) {
log.Info().Msg("clusterenv.yaml file not found, skipping environment variable loading.")
// If the file doesn't exist, check noFail to determine next steps
if noFail {
log.Info().Msg("clusterenv.yaml file not found, but skipping due to noFail being true.")
return nil // Skip execution without error
} else {
log.Info().Msg("clusterenv.yaml file not found, exiting as noFail is false.")
os.Exit(1) // Exit with error code 1
}
} else {
log.Info().Msgf("Error checking clusterenv.yaml file: %v\n", err)
os.Exit(1)
}
// If file exists, continue with processing
clusterName()
checkQuotedNumbersInFile()
PostProcessTalEnv()
@@ -214,14 +223,13 @@ func normalizeIPNetmask(ipNetmask string) (string, error) {
}
func CheckEnvVariables() {
LoadTalEnv()
LoadTalEnv(false)
requiredKeys := []string{
"VIP",
"MASTER1IP_IP",
"MASTER1IP_NETMASK",
"GATEWAY",
"METALLB_RANGE",
"DASHBOARD_IP_IP",
"PODNET",
"SVCNET",
}
@@ -282,14 +290,16 @@ func CheckEnvVariables() {
}
// Check DASHBOARD_IP against METALLB_RANGE
inRange, err = helper.IPInRange(helper.TalEnv["DASHBOARD_IP"], helper.TalEnv["METALLB_RANGE"])
if err != nil {
log.Info().Msgf("Error checking DASHBOARD_IP against METALLB_RANGE: %v\n", err)
os.Exit(1)
}
if !inRange {
log.Info().Msg("Cannot proceed, DASHBOARD_IP must be in the METALLB_RANGE")
os.Exit(1)
if helper.TalEnv["DASHBOARD_IP"] != "" {
inRange, err = helper.IPInRange(helper.TalEnv["DASHBOARD_IP"], helper.TalEnv["METALLB_RANGE"])
if err != nil {
log.Info().Msgf("Error checking DASHBOARD_IP against METALLB_RANGE: %v\n", err)
os.Exit(1)
}
if !inRange {
log.Info().Msg("Cannot proceed, DASHBOARD_IP must be in the METALLB_RANGE")
os.Exit(1)
}
}
// Validate other CIDR/IP checks with new netmask support
clustertool/pkg/initfiles/initfiles.go
+111 -46
View File
@@ -151,6 +151,7 @@ func genBaseFiles() error {
}
func UpdateBaseFiles() error {
log.Info().Msg("Updating Base files for cluster: helper.ClusterPath")
// Read filenames in source directory
sourceFiles, err := readFilenamesInDir(helper.BaseCache)
if err != nil {
@@ -244,7 +245,7 @@ func readFilenamesInDir(dir string) ([]string, error) {
}
func ResetBootstrapValues() error {
LoadTalEnv()
LoadTalEnv(false)
err := helper.CopyDirFiltered(helper.KubeCache, helper.ClusterPath+"/kubernetes", true, `^bootstrap-values\.yaml.ct$`)
if err != nil {
log.Info().Msg("Error:")
@@ -309,18 +310,15 @@ func setDocker() {
if helper.TalEnv["DOCKERHUB_USER"] != "" && helper.TalEnv["DOCKERHUB_PASSWORD"] != "" {
// Prepare the content to append
configContent := fmt.Sprintf(`# Add Dockerhub Login
- operation: replace
path: /machine/registries/config/registry-1.docker.io
value:
registry-1.docker.io:
auth:
username: %s
password: %s
- operation: replace
path: /machine/registries/config/docker.io
value:
docker.io:
auth:
username: %s
password: %s
`, helper.TalEnv["DOCKERHUB_USER"], helper.TalEnv["DOCKERHUB_PASSWORD"], helper.TalEnv["DOCKERHUB_USER"], helper.TalEnv["DOCKERHUB_PASSWORD"])
// Open the file in append mode or create it if it doesn't exist
@@ -355,42 +353,62 @@ func setSpegel() {
if helper.TalEnv["SPEGEL_IP"] != "" && helper.TalEnv["SPEGEL_IP"] != "" {
// Prepare the content to append
configContent := fmt.Sprintf(`# Add Dockerhub Login
- operation: replace
path: /machine/registries/mirrors/
value:
cgr.dev:
endpoints:
- http://%s:5000
docker.io:
endpoints:
- http://%s:5000
ghcr.io:
endpoints:
- http://%s:5000
quay.io:
endpoints:
- http://%s:5000
mcr.microsoft.com:
endpoints:
- http://%s:5000
public.ecr.aws:
endpoints:
- http://%s:5000
gcr.io:
endpoints:
- http://%s:5000
registry.k8s.io:
endpoints:
- http://%s:5000
k8s.gcr.io:
endpoints:
- http://%s:5000
tccr.io:
endpoints:
- http://%s:5000
factory.talos.dev:
endpoints:
- http://%s:5000
mirrors:
cgr.dev:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
- http://%s:5000
docker.io:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
- http://%s:5000
ghcr.io:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
- http://%s:5000
quay.io:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
- http://%s:5000
mcr.microsoft.com:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
- http://%s:5000
public.ecr.aws:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
- http://%s:5000
gcr.io:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
- http://%s:5000
registry.k8s.io:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
- http://%s:5000
k8s.gcr.io:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
- http://%s:5000
tccr.io:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
- http://%s:5000
factory.talos.dev:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
- http://%s:5000
`, helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"], helper.TalEnv["SPEGEL_IP"])
@@ -407,15 +425,62 @@ func setSpegel() {
}
} else {
// Optional: Append a note if the environment variables are not set
emptyContent := `# No Spegel_IP provided
`
configContent := fmt.Sprintf(`# No Spegel_IP provide
mirrors:
cgr.dev:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
docker.io:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
ghcr.io:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
quay.io:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
mcr.microsoft.com:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
public.ecr.aws:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
gcr.io:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
registry.k8s.io:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
k8s.gcr.io:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
tccr.io:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
factory.talos.dev:
endpoints:
- http://127.0.0.1:30020
- http://127.0.0.1:30021
`)
file, err := os.OpenFile(filepath.Join(helper.ClusterPath+"/talos/patches", "all.yaml"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
log.Fatal().Err(err).Msg("Error opening file: %s")
}
defer file.Close()
if _, err := file.Write([]byte(emptyContent)); err != nil {
if _, err := file.Write([]byte(configContent)); err != nil {
log.Fatal().Err(err).Msg("Error writing to file: %s")
}
}
clustertool/pkg/sops/decrypt.go
+1 -1
View File
@@ -55,7 +55,7 @@ func DecryptFiles() error {
if !encryptedFound {
log.Info().Msg("Nothing to decrypt")
}
initfiles.LoadTalEnv()
initfiles.LoadTalEnv(true)
return nil
}