chore(clustertool): Docs and Command changes

clustertool/cmd/init.go

@@ -12,7 +12,7 @@ var initLongHelp = strings.TrimSpace(`
 ClusterTool requires a specific directory layout to ensure smooth operators and standardised environments.
 
 To ensure smooth deployment, the init function can pre-generate all required files in the right places.
-Afterwards you can edit talconfig.yaml and talenv.yaml to reflect your personal settings.
+Afterwards you can edit talconfig.yaml and clusterenv.yaml to reflect your personal settings.
 
 When done, please run clustertool genconfig to generate all configarion based on your personal settings
 

clustertool/cmd/scale.go

@@ -0,0 +1,24 @@
+package cmd
+
+import (
+    "strings"
+
+    "github.com/spf13/cobra"
+)
+
+var scaleLongHelp = strings.TrimSpace(`
+These are all advanced commands that should generally not be needed
+
+`)
+
+var scaleCmd = &cobra.Command{
+    Use:           "scale",
+    Short:         "Commands for handling TrueNAS SCALE",
+    Long:          advLongHelp,
+    SilenceUsage:  true,
+    SilenceErrors: true,
+}
+
+func init() {
+    RootCmd.AddCommand(scaleCmd)
+}

clustertool/cmd/scale_export.go

@@ -6,7 +6,7 @@ import (
 )
 
 var scaleexport = &cobra.Command{
-    Use:   "scaleexport",
+    Use:   "export",
     Short: "Export SCALE Apps to file",
     Run: func(cmd *cobra.Command, args []string) {
         scale.ExportApps()
@@ -14,5 +14,5 @@ var scaleexport = &cobra.Command{
 }
 
 func init() {
-    adv.AddCommand(scaleexport)
+    scaleCmd.AddCommand(scaleexport)
 }

clustertool/cmd/scale_migrate.go

@@ -7,7 +7,7 @@ import (
 )
 
 var scalemigrate = &cobra.Command{
-    Use:   "scalemigrate",
+    Use:   "migrate",
     Short: "Migrate exported SCALE Apps to the Talos Cluster",
     Run: func(cmd *cobra.Command, args []string) {
         err := scale.ProcessJSONFiles("./truenas_exports")
@@ -18,5 +18,5 @@ var scalemigrate = &cobra.Command{
 }
 
 func init() {
-    adv.AddCommand(scalemigrate)
+    scaleCmd.AddCommand(scalemigrate)
 }

clustertool/embed/generic/kubernetes/kube-system/cilium/app/bootstrap-values.yaml.ct

@@ -1,3 +1,3 @@
-## DO NOT ALTER THIS FILE, CHANGE DO NOT PERSIST. Alter TalEnv.yaml instead.
+## DO NOT ALTER THIS FILE, CHANGE DO NOT PERSIST. Alter clusterenv.yaml instead.
 hubble:
   enabled: false

clustertool/embed/generic/root/DOTREPLACEsops.yaml

@@ -6,8 +6,6 @@ creation_rules:
   - path_regex: ^clusters.*kubernetes.*\.secret.ya?ml
     age: REPLACEME
     encrypted_regex: "((?i)(displayname|email|pass|ca|id|bootstraptoken|secretboxencryptionsecret|secrets|secrets|password|cert|secret($|[^N])|key|token|^data$|^stringData))"
-  - path_regex: talenv.yaml
-    age: REPLACEME
   - path_regex: clusterenv.yaml
     age: REPLACEME
   - path_regex: talsecret.yaml

website/src/content/docs/clustertool/FAQ.md

@@ -0,0 +1,64 @@
+---
+sidebar:
+  order: 2
+title: Frequently Asked Questions (FAQ)
+---
+
+:::caution[Work In Progress]
+
+This program, all its features and its general design, are all a Work-In-Progress. It is not done and not widely available.
+
+All code and docs are considered Pre-Beta drafts
+
+:::
+
+
+## Does ClusterTool include storage
+
+By default ClusterTool ships with a single-node Longhorn setup pre-configured on the system-drive.
+Its storageClass gets used by any PVC by default
+
+## What are the network requirements of ClusterTool
+
+- All nodes are expected to be in the same local-area network.
+- While we technically do support non-/24 subnets, we heavily advice a /24 subnet
+- We do not support nodes directly on a WAN-IP or kube-span related setups
+
+## How do I alter the default config to a multi-node setup?
+
+- Add more masternodes to talconfig
+- Add more MASTER_IPs to ClusterEnv
+- Alter the Longhorn replicas to 2 or 3
+
+## How do I seperate Masters from Slave nodes?
+
+- Set `allowSchedulingOnControlPlanes: false` in talconfig.yaml
+- On the metallb helmrelease, set `ignoreExcludeLB: false`
+
+## There are Pending pods for Cillium
+
+We ship cillium pre-configured for multi-node, this means that on a single node they inherently keep a few pods pending
+
+## What commands do I need to run after updating clustertool?
+
+init, genconfig and, preferably, apply (after checking the configuration changes)
+
+## What do I need to do to get automatic updates?
+
+We've setup a lot for you already.
+However, to get updates to propagate we advice the use of FluxCD
+
+## Do I need to use FluxCD?
+
+While its not *technically* required, we HEAVILY advice using FluxCD.
+
+## Can I remove helm-charts installed by default?
+
+We HEAVILY adviced against it, the charts in the default stack all get loaded for cluster stability.
+In some cases, like kubernetes, you can disable the workloads or services in the charts themselves instead.
+
+## Can I alter the helmrelease files for default installed charts?
+
+Yes you can!
+While we advice sticking with the defaults where possible, we fully support altering the included helm-charts to fit your usecase.
+However, any issues after alterations are not within our scope for support.

website/src/content/docs/clustertool/functions/genconfig.md

@@ -12,9 +12,9 @@ All code and docs are considered Pre-Beta drafts
 
 :::
 
-After all your settings are entered into talconfig.yaml and talenv.yaml, Clustertool generates a complete clusterconfiguration using TalHelper and various other tools.
+After all your settings are entered into talconfig.yaml and clusterenv.yaml, Clustertool generates a complete clusterconfiguration using TalHelper and various other tools.
 
 It's important to note that running `clustertool genconfig`, again after each settings change, is absolutely imperative to be able to deploy said settings to your cluster.
 
-This does not only generate the Talos "Machine Config" files, but also ensures an updated configmap containing your "talenv.yaml" settings, is added to the /manifests/ directory, for consumption by FluxCD when added.
+This does not only generate the Talos "Machine Config" files, but also ensures an updated configmap containing your "clusterenv.yaml" settings, is added to the /manifests/ directory, for consumption by FluxCD when added.
 It also ensures the same configmap is always added by updating the patches.

website/src/content/docs/clustertool/functions/init.md

@@ -15,7 +15,7 @@ All code and docs are considered Pre-Beta drafts
 ClusterTool requires a specific directory layout to ensure smooth operators and standardised environments.
 
 To ensure smooth deployment, the init function can pre-generate all required files in the right places.
-Afterwards, you can edit talconfig.yaml and talenv.yaml to reflect your personal settings.
+Afterwards, you can edit talconfig.yaml and clusterenv.yaml to reflect your personal settings.
 
 When done, please run clustertool genconfig to generate all configurations based on your personal settings.
 
@@ -23,7 +23,7 @@ When done, please run clustertool genconfig to generate all configurations based
 ## Created Files and Folders
 
 - talconfig.yaml -> Contains your Talos Cluster layout
-- talenv.yaml -> Contains configuration options for both your Charts and Talos Cluster
+- clusterenv.yaml -> Contains configuration options for both your Charts and Talos Cluster
 - talsecret.yaml -> Contains Talos Cluster encryption keys
 - age.agekey -> Contains SOPS encryption keys which can be used to encrypt data. It's **IMPERATIVE** you save this specific file elsewhere as a backup, not doing so *will* result in future data loss
 - .sops.yaml -> Contains specifications on how to decrypt any encrypted files found

website/src/content/docs/clustertool/getting-started.md

@@ -1,6 +1,6 @@
 ---
 sidebar:
-  order: 2
+  order: 3
 title: Getting Started
 ---
 
@@ -85,7 +85,7 @@ It should start with `ssh://`, so be sure to pick the SSH repository url option
 
 ### TalConfig
 
-This file contains purely the structure of the Talos Cluster and its nodes themselves. As such, it also contains a number of `${VARIABLE}` references to `talenv.yaml`. These should **not** be removed.
+This file contains purely the structure of the Talos Cluster and its nodes themselves. As such, it also contains a number of `${VARIABLE}` references to `clusterenv.yaml`. These should **not** be removed.
 
 We generate an opinionated variant of this file, that is optimised to run with our default setup. *Making any changes outside of the nodes section, might completely break ClusterTool*
 

website/src/content/docs/clustertool/index.md

@@ -24,9 +24,9 @@ As security is important, it features integrated encryption and decryption. Base
 We do expect users to understand what Talos OS is and isn't.
 As well as understand the basics of what "Apply", "Bootstrap" and/or "Upgrade" means for Talos OS.
 
-## What isn't ClusterTool
+## What ClusterTool is not
 
-Besides bootstrapping, clustertool will NOT help you update, maintain or alter helm-charts in *any* way.
+Besides bootstrapping included helm charts, clustertool will NOT help you update, maintain or alter helm-charts in *any* way.
 How users run their cluster post-bootstrap, is on to the user to decide.
 
 This means that users are expected to have basic shell/console/terminal skills and understand the basics on how to edit, update, upgrade and install Helm charts.

website/src/content/docs/clustertool/migrations/scale.md

@@ -12,7 +12,7 @@ All code and docs are considered Pre-Beta drafts
 
 :::
 
-With the upcoming "Electric Eel" version of TrueNAS SCALE, iX-Systems has decided to completely gut the Kubernetes backend of SCALE. However, we've got you covered so you can safely keep running your TrueCharts Charts.
+With the upcoming 24.10 "Electric Eel" version of TrueNAS SCALE, iX-Systems has decided to completely gut the Kubernetes backend of SCALE. However, we've got you covered so you can safely keep running your TrueCharts Charts.
 
 Our new ClusterTool can, among other things, be used to craft yourself a new kubernetes backend to continue self-hosting using TrueCharts. For this guide we'll be using a TalosOS VM on TrueNAS SCALE, but any solid hypervisor would suffice.
 
@@ -20,6 +20,10 @@ Our new ClusterTool can, among other things, be used to craft yourself a new kub
 
 Prior to migration, we require the following to be done:
 
+### **DO NOT** update to Electric Eel
+
+As soon as you migupdaterated to TrueNAS SCALE 24.10 "Electric Eel", you cannot, in any way, migrate your TrueCharts Apps anymore.
+
 ### Ensure all "hostPath" storage is replaced with "NFS"-share based storage
 
 With the future migration to Talos using ClusterTool, you won't technically be able to reach these folders through "hostPath" anymore.
@@ -33,15 +37,14 @@ Ensure you setup the permissions on the Dataset according to [this](https://true
 
 And configure the NFS Shares according to [this](https://truecharts.org/deprecated/scale/guides/nfs-share/) guide.
 
-:::note
-
-
-With NFS-based storage, we explicitly refer to the NFS storage type as explained in [this](https://truecharts.org/deprecated/scale/guides/nfs-share/) guide. We do not advise the use of "static-PVC" NFS storage as an alternative
-
-:::
 
 We  also *heavily* advise that config storage should almost *always* be set to "PVC" and not *hostPath* nor *NFS*.  If you did decide on using NFS storage for config, we cannot take responsibility if the migration fails.
 
+### NFSv4 Requirement
+
+Our default ClusterTool Talos setup, requires NFSv4 to be enabled on any shares used.
+Not doing so beforehand, might lead to issues.
+
 ### Ensure all PVC storage has VolSync backups (not restore) Enabled
 
 While we've not fully finished our design docs, there are big chances those are going to require you to already have VolSync backups of any PVC present.
@@ -52,6 +55,99 @@ Setup VolSync according to [this](https://truecharts.org/deprecated/scale/guides
 
 *If you currently do not have VolSync installed, a migration-specific catalog will be provided at a later date to install it prior to migration*
 
+### Ensure all CNPG/Postgresql sections have their backups **enabled**
+
+CNPG/postgresql data is not backed up using volsync but uses their own backup solution.
+Its imperative to setup those backups seperately as described in [this](https://truecharts.org/deprecated/scale/guides/backup-restore/) guide.
+
+### Note on other databases and their backups
+
+We cannot guarantee the integerty of included:
+- MongoDB databases
+- Redis Databases
+- MariaDB databases
+
+Those might get wiped on migration.
+
+### Keep Existing Data Around
+
+We cannot guarantee everything goes smoothly. Hence we would advice to "unset" the Apps pool, but **do not** remove the `ix-applications` dataset before you've verified your new cluster and its Apps are fully up-and-running. Deleting of Apps and/or the `ix-applications` dataset leads to permanent irreversable dataloss.
+
+#### Cobia specific
+
+When still on TrueNAS SCALE 23.10 "Cobia", we would advice using Heavyscript to make Backups of the Apps system as well.
+
+
 ## Continue with ClusterTool Getting-Started
 
 From this point onwards, please go through the complete [ClusterTool Getting-Started](/clustertool/getting-started) guide.
+
+## Migration
+
+### Important note
+
+We do not sanitise the exported/migated configuration from SCALE.
+You yourself are responsible to move any sensitive data to `ClusterEnv.yaml` variables and references them in `helmrelease.yaml` as `${NAMEOFVARIABLE}`
+
+An easy way (GUI) to do this, is stopping the Apps and then editing all SCALE Apps and replacing the sensitive data before export.
+In a lot of cases you can even, already, change them into a `${NAMEOFVARIABLE}` format!
+
+This will "break" the Apps from running in SCALE, but guarantees the sensitive data is not ever exported.
+
+### Exporting SCALE Apps
+
+Ensure you git-cloned your Cluster git repository, setup during [ClusterTool Getting-Started](/clustertool/getting-started), onto your TrueNAS SCALE machine.
+You can run now `./clustertool scale export`, to export a complete dump of the configuration values for your SCALE Apps.
+This command does NOT remove passwords and other credentials from the export files. You're responsible yourself to (re)move them!
+
+Afterwards, ensure you run `git add . && git commit -m "Exported SCALE App config" && git push`
+
+### Converting SCALE Apps to Helm-Release files
+
+- Move back to your main machine running ClusterTool and ensure your gotten your latest additions to your Cluster GIT repository, by running: `git pull`
+- Run `./clustertool scale migrate` to convert all exported SCALE Apps to ClusterTool/FluxCD compatible Helm-Releases
+- Remove any, unwanted, changes (or duplicates of) to any charts included by clustertool by default.
+- run `./clustertool init` and `./clustertool genconfig` again to ensure a fluxcd compatible directory structure is loaded
+
+### Alterations for CNPG restore
+
+The same issues with CNPG restores discussed in [this](https://truecharts.org/deprecated/scale/guides/backup-restore/) guide, also Apply to other clusters/deployments other than TrueNAS SCALE.
+Primarily the requirement to set the bootstrap mode to recovery and having to change the revision numbers on both the backup and restore sections.
+
+So to restore your CNPG backup made on SCALE< please ensure this is done before loading onto the cluster
+
+### Loading Into the Cluster
+
+We heavily advice using FluxCD to keep all your charts synced with your github repository as-well-as allow for automated updates.
+
+In case you do not, we've made an alternative for `helm install` and `helm upgrade` that uses the values defined in the `helmrelease.yaml` files instead of `values.yaml`.
+This ensures the migration is compatible with both "plain" clusters as well as FluxCD
+
+#### The FluxCD way
+
+- Push the files to your Git Repo Flux should automatically pick it all up.
+
+#### The Helm Way
+
+Run the following for each chart you want imported (altered accordingly):
+`./clustertool helmrelease install cluster/main/kubernetes/path/to/my/helmrelease/file.yaml`
+
+If you ever need to do alterations/updates/upgrades manually, there is also a command for that:
+`./clustertool helmrelease upgrade cluster/main/kubernetes/path/to/my/helmrelease/file.yaml`
+
+
+
+## Known Issues
+
+### VolSync backups broken
+In some cases VolSync might not backup your data.
+If everything is setup correctly, you can ask our support staff to verify, there is currently not much we can do about this upstream bug.
+
+### TrueNAS SCALE 24.10 "Electric Eel" not supported
+
+Its **NOT** possible *in any way* to do the migration after updating to TrueNAS SCALE 24.10 "Electric Eel"
+
+### MariaDB, MongoDB and Redis data wont migrate
+
+This is correct, all this data will NOT migrate.
+This cannot be helped or patched.

website/src/content/docs/general/FAQ.md

@@ -6,6 +6,8 @@ title: Frequently Asked Questions (FAQ)
 
 TrueCharts Frequently Asked Questions
 
+*For the ClusterTool FAQ, see the ClusterTool section*
+
 ## Why PVC is recommended over HostPath?
 
 We recommend using PVC for app "config" storage, as it provides an easy way to roll back your application in case of a failed update. Furthermore, we currently only offer active support for this storage option. You can still use HostPath for your media and other files.

website/src/content/docs/news/2024-10-16-news.md

@@ -60,7 +60,7 @@ Follow the linked Guides to find more details on how to create your own Talos Cl
 - [VM Setup](/clustertool/virtual-machines/systemrequirements)
 - [Getting Started with Clustertool](/clustertool/getting-started/)
 - [clustertool init](/clustertool/functions/init/)
-- Edit your config (talconfig.yaml and talenv.yaml)
+- Edit your config (talconfig.yaml and clusterenv.yaml)
 - [clustertool genconfig](/clustertool/functions/genconfig/)
 - [clustertool apply](/clustertool/functions/apply/)