refactor: Big internal refactor

schemas/cluster.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "title": "OptionalCluster",
+  "title": "Cluster",
   "type": "object",
   "properties": {
     "base": {
@@ -11,8 +11,7 @@
         {
           "type": "null"
         }
-      ],
+      ]
-      "writeOnly": true
     },
     "clusterEnv": {
       "anyOf": [
@@ -32,8 +31,7 @@
       "format": "ipv4"
     },
     "default": {
-      "$ref": "#/$defs/OptionalNode",
+      "$ref": "#/$defs/OptionalNodeDeserialize"
-      "writeOnly": true
     },
     "nodes": {
       "type": [
@@ -41,7 +39,7 @@
         "null"
       ],
       "items": {
-        "$ref": "#/$defs/NodeEntry"
+        "type": "string"
       }
     },
     "secretsFile": {
@@ -76,13 +74,6 @@
         "amd64"
       ]
     },
-    "NodeEntry": {
-      "anyOf": [
-        {
-          "type": "string"
-        }
-      ]
-    },
     "NodeType": {
       "type": "string",
       "enum": [
@@ -196,7 +187,7 @@
       },
       "additionalProperties": false
     },
-    "OptionalNode": {
+    "OptionalNodeDeserialize": {
       "type": "object",
       "properties": {
         "arch": {
@@ -296,7 +287,7 @@
             "null"
           ],
           "items": {
-            "$ref": "#/$defs/Patch"
+            "type": "string"
           }
         },
         "controlPlane": {
@@ -305,7 +296,7 @@
             "null"
           ],
           "items": {
-            "$ref": "#/$defs/Patch"
+            "type": "string"
           }
         }
       },
@@ -366,26 +357,19 @@
       },
       "additionalProperties": false
     },
-    "Patch": {
-      "anyOf": [
-        {
-          "type": "string"
-        }
-      ]
-    },
     "Patches": {
       "type": "object",
       "properties": {
         "all": {
           "type": "array",
           "items": {
-            "$ref": "#/$defs/Patch"
+            "type": "string"
           }
         },
         "controlPlane": {
           "type": "array",
           "items": {
-            "$ref": "#/$defs/Patch"
+            "type": "string"
           }
         }
       },

schemas/node.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "title": "OptionalNode",
+  "title": "Node",
   "type": "object",
   "properties": {
     "arch": {
@@ -192,7 +192,7 @@
             "null"
           ],
           "items": {
-            "$ref": "#/$defs/Patch"
+            "type": "string"
           }
         },
         "controlPlane": {
@@ -201,7 +201,7 @@
             "null"
           ],
           "items": {
-            "$ref": "#/$defs/Patch"
+            "type": "string"
           }
         }
       },
@@ -236,13 +236,6 @@
       },
       "additionalProperties": false
     },
-    "Patch": {
-      "anyOf": [
-        {
-          "type": "string"
-        }
-      ]
-    },
     "Schematic": {
       "type": "string"
     },

src/cluster.rs

@@ -8,18 +8,19 @@ use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use walkdir::WalkDir;
 
-use crate::get_talos_config_path;
+use crate::environment::PathEnvironment;
-use crate::node::{Node, OptionalNode};
+use crate::get_talos_path;
+use crate::node::{Node, OptionalNodeDeserialize};
 use crate::patch::Patches;
 
 #[optional_struct]
 #[derive(Debug, Deserialize, JsonSchema, Clone)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
-pub struct Base {
+pub(crate) struct Base {
     #[serde(default)]
     pub(crate) kernel_args: Vec<String>,
     #[serde(default)]
-    pub(crate) patches: Patches,
+    pub(crate) patches: Patches<String>,
 }
 
 #[optional_struct]
@@ -30,109 +31,125 @@ pub struct Version {
     talos: semver::Version,
 }
 
-#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, PartialEq, Eq)]
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, Copy, PartialEq, Eq)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub enum ClusterEnv {
     Production,
     Staging,
 }
 
-#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, PartialEq, Eq)]
-#[serde(untagged)]
-pub enum NodeEntry {
-    Name(String),
-    #[serde(skip_deserializing)]
-    Node(Box<Node>),
-}
-
 #[optional_struct]
-#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+#[derive(Debug, Deserialize, JsonSchema, Clone)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
-pub struct Cluster {
+struct ClusterDeserialize {
-    #[serde(skip_deserializing)]
-    name: String,
     #[optional_rename(OptionalVersion)]
     #[optional_wrap]
     version: Version,
-    nodes: Vec<NodeEntry>,
+    nodes: Vec<String>,
     cluster_env: ClusterEnv,
     control_plane_ip: Ipv4Addr,
     secrets_file: PathBuf,
-    #[serde(default, skip_serializing)]
+    #[serde(default)]
     #[optional_skip_wrap]
-    pub(crate) default: OptionalNode,
+    default: OptionalNodeDeserialize,
     #[optional_rename(OptionalBase)]
     #[optional_wrap]
-    #[serde(skip_serializing)]
+    base: Base,
-    pub(crate) base: Base,
+}
-    // pub secrets_file: PathBuf,
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Cluster {
+    name: String,
+    version: Version,
+    nodes: Vec<Node>,
+    cluster_env: ClusterEnv,
+    control_plane_ip: Ipv4Addr,
+    secrets_file: PathBuf,
 }
 
 impl Cluster {
-    pub fn get(cluster_name: &str, env: &Environment) -> Self {
+    pub(crate) fn get(cluster_name: &str, env: &Environment) -> Self {
-        let path = get_talos_config_path()
+        let path = get_talos_path().join("clusters").join("default.yaml");
-            .join("clusters")
-            .join("default.yaml");
 
-        let default: OptionalCluster = if path.exists() {
+        let default: OptionalClusterDeserialize = if path.exists() {
             let content = std::fs::read_to_string(path).unwrap();
             serde_yaml::from_str(&content).unwrap()
         } else {
             Default::default()
         };
 
-        let mut path = get_talos_config_path().join("clusters").join(cluster_name);
+        let mut path = get_talos_path().join("clusters").join(cluster_name);
         path.add_extension("yaml");
         let content = std::fs::read_to_string(path).unwrap();
 
-        let mut cluster: OptionalCluster = serde_yaml::from_str(&content).unwrap();
+        let mut cluster: OptionalClusterDeserialize = serde_yaml::from_str(&content).unwrap();
-        cluster.name = Some(cluster_name.to_string());
 
         // For some reason apply on the cluster does not properly apply to the default settings...
         // So we manually apply it here first
         cluster.default = default.default.clone().apply(cluster.default);
 
-        let mut cluster: Self = default.apply(cluster).try_into().unwrap();
+        let cluster: ClusterDeserialize = default.apply(cluster).try_into().unwrap();
+        let nodes = cluster.nodes;
+        let default = cluster.default;
+        let base = cluster.base;
 
-        cluster.nodes = cluster
+        let secrets_file = get_talos_path()
-            .nodes
-            .clone()
-            .into_iter()
-            .map(|node_entry| {
-                if let NodeEntry::Name(name) = node_entry {
-                    NodeEntry::Node(Box::new(Node::get(&name, env, &cluster)))
-                } else {
-                    node_entry
-                }
-            })
-            .collect();
-
-        cluster.secrets_file = get_talos_config_path()
             .join("secrets")
             .join(cluster.secrets_file)
             .absolute()
             .unwrap();
 
-        cluster
+        let cluster = Self {
+            name: cluster_name.into(),
+            version: cluster.version,
+            nodes: vec![],
+            cluster_env: cluster.cluster_env,
+            control_plane_ip: cluster.control_plane_ip,
+            secrets_file,
+        };
+
+        let nodes = nodes
+            .iter()
+            .map(|name| Node::get(name, env, &cluster, &default, &base))
+            .collect();
+
+        Self { nodes, ..cluster }
+    }
+
+    pub fn get_all() -> Vec<Cluster> {
+        let env = PathEnvironment::new_patches();
+
+        WalkDir::new(get_talos_path().join("clusters"))
+            .into_iter()
+            .filter_map(|e| e.ok())
+            .filter(|e| e.metadata().unwrap().is_file())
+            .filter_map(|e| {
+                let mut path = PathBuf::from_str(&e.file_name().to_string_lossy()).unwrap();
+                path.set_extension("");
+
+                let name = path.to_string_lossy().to_string();
+
+                if name == "default" {
+                    return None;
+                }
+
+                Some(Cluster::get(&name, &env))
+            })
+            .collect()
+    }
+
+    pub fn nodes(&self) -> &[Node] {
+        &self.nodes
     }
 }
 
-pub fn get_clusters(env: &Environment) -> Vec<Cluster> {
+impl JsonSchema for Cluster {
-    WalkDir::new(get_talos_config_path().join("clusters"))
+    fn schema_name() -> std::borrow::Cow<'static, str> {
-        .into_iter()
+        "Cluster".into()
-        .filter_map(|e| e.ok())
+    }
-        .filter(|e| e.metadata().unwrap().is_file())
-        .filter_map(|e| {
-            let mut path = PathBuf::from_str(&e.file_name().to_string_lossy()).unwrap();
-            path.set_extension("");
 
-            let name = path.to_string_lossy().to_string();
+    fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
-
+        OptionalClusterDeserialize::json_schema(generator)
-            if name == "default" {
+    }
-                return None;
-            }
-
-            Some(Cluster::get(&name, env))
-        })
-        .collect()
 }

src/environment.rs

@@ -6,7 +6,7 @@ use std::str::FromStr;
 use minijinja::{AutoEscape, Environment, path_loader};
 use walkdir::WalkDir;
 
-use crate::{get_repo_path, get_talos_config_path};
+use crate::{get_configs_path, get_talos_path};
 
 /// Transparent wrapper around minijinja::Environment that loads templates from a path and
 /// configures better defaults. It also implements IntoIter, making it possible to iterate over all
@@ -44,6 +44,20 @@ impl<'a> PathEnvironment<'a> {
             }
         });
 
+        // Helper function for getting the path to kubeconfig files
+        env.add_filter("kubeconfig", move |names: Vec<String>| {
+            names
+                .iter()
+                .map(|name| {
+                    get_configs_path()
+                        .join(name)
+                        .join("kubeconfig")
+                        .to_string_lossy()
+                        .to_string()
+                })
+                .collect::<Vec<_>>()
+        });
+
         // Add path loader
         env.set_loader(path_loader(&path));
 
@@ -53,28 +67,8 @@ impl<'a> PathEnvironment<'a> {
         }
     }
 
-    pub fn new_patches() -> Self {
+    pub(crate) fn new_patches() -> Self {
-        Self::new(&get_talos_config_path().join("patches"))
+        Self::new(&get_talos_path().join("patches"))
-    }
-
-    pub fn new_templates(repo: &Path) -> Self {
-        let mut env = Self::new(&get_repo_path().join("templates"));
-
-        let path = repo.absolute().unwrap();
-        env.env.add_filter("kubeconfig", move |names: Vec<String>| {
-            names
-                .iter()
-                .map(|name| {
-                    path.join("configs")
-                        .join(name)
-                        .join("kubeconfig")
-                        .to_string_lossy()
-                        .to_string()
-                })
-                .collect::<Vec<_>>()
-        });
-
-        env
     }
 }
 

src/lib.rs

@@ -17,10 +17,14 @@ pub fn set_repo_path(path: impl Into<PathBuf>) {
         .expect("Repo path already initialized");
 }
 
-fn get_repo_path() -> &'static Path {
+pub fn get_repo_path() -> &'static Path {
     REPO_PATH.get().expect("Repo path not initialized")
 }
 
-fn get_talos_config_path() -> PathBuf {
+pub fn get_talos_path() -> PathBuf {
     get_repo_path().join("talos")
 }
+
+pub fn get_configs_path() -> PathBuf {
+    get_repo_path().join("configs")
+}

src/main.rs

@@ -2,9 +2,9 @@ mod cli;
 
 use clap::{CommandFactory, Parser};
 use clap_complete::{Shell, generate as generate_complete};
-use crete::cluster::get_clusters;
+use crete::cluster::Cluster;
 use crete::environment::PathEnvironment;
-use crete::set_repo_path;
+use crete::{get_repo_path, set_repo_path};
 use minijinja::context;
 use thiserror::Error;
 
@@ -19,8 +19,7 @@ enum Error {
 fn generate(opts: &GlobalOpts) -> Result<(), Error> {
     set_repo_path(&opts.repo);
 
-    let patch_env = PathEnvironment::new_patches();
+    let clusters = Cluster::get_all();
-    let clusters = get_clusters(&patch_env);
 
     if clusters.is_empty() {
         return Err(Error::NoClustersFound);
@@ -32,7 +31,8 @@ fn generate(opts: &GlobalOpts) -> Result<(), Error> {
     }
     std::fs::create_dir(&path).unwrap();
 
-    let template_env = PathEnvironment::new_templates(&opts.repo);
+    // Render templates
+    let template_env = PathEnvironment::new(&get_repo_path().join("templates"));
     for template_name in &template_env {
         let template = template_env.get_template(&template_name).unwrap();
 

src/node.rs

@@ -5,9 +5,9 @@ use optional_struct::{Applicable, optional_struct};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
-use crate::cluster::Cluster;
+use crate::cluster::{Base, Cluster};
-use crate::get_talos_config_path;
+use crate::get_talos_path;
-use crate::patch::{OptionalPatches, Patches};
+use crate::patch::{OptionalPatches, OptionalPatchesString, Patches};
 use crate::schematic::Schematic;
 use crate::secret::Secret;
 
@@ -19,6 +19,15 @@ enum NodeType {
     ControlPlane,
 }
 
+impl From<NodeType> for &str {
+    fn from(value: NodeType) -> Self {
+        match value {
+            NodeType::Worker => "worker",
+            NodeType::ControlPlane => "controlplane",
+        }
+    }
+}
+
 #[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, Copy, PartialEq, Eq)]
 #[serde(rename_all = "camelCase")]
 enum NodeArch {
@@ -59,11 +68,9 @@ struct Install {
 }
 
 #[optional_struct]
-#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, PartialEq, Eq)]
+#[derive(Debug, Deserialize, JsonSchema, Clone, PartialEq, Eq)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
-pub struct Node {
+pub(crate) struct NodeDeserialize {
-    #[serde(skip_deserializing)]
-    hostname: String,
     arch: NodeArch,
     schematic: Schematic,
     r#type: NodeType,
@@ -75,64 +82,99 @@ pub struct Node {
     #[optional_wrap]
     install: Install,
     kernel_args: Vec<String>,
-    #[optional_rename(OptionalPatches)]
+    #[optional_rename(OptionalPatchesString)]
     #[optional_wrap]
-    pub(crate) patches: Patches,
+    patches: Patches<String>,
+    sops: Secret,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Node {
+    hostname: String,
+    arch: NodeArch,
+    schematic: Schematic,
+    r#type: NodeType,
+    network: Network,
+    ntp: String,
+    install: Install,
+    kernel_args: Vec<String>,
+    patches: Patches<serde_yaml::Value>,
     sops: Secret,
 }
 
 impl Node {
-    pub fn get(node_name: &str, env: &Environment, cluster: &Cluster) -> Self {
+    pub(crate) fn get(
-        let mut path = get_talos_config_path().join("nodes").join(node_name);
+        node_name: &str,
-        let named = OptionalNode {
+        env: &Environment,
-            hostname: Some(
+        cluster: &Cluster,
-                path.file_name()
+        default: &OptionalNodeDeserialize,
-                    .expect("Path should be valid")
+        base: &Base,
-                    .to_string_lossy()
+    ) -> Self {
-                    .to_string(),
+        let mut path = get_talos_path().join("nodes").join(node_name);
-            ),
+        let hostname = path
-            ..OptionalNode::default()
+            .file_name()
-        };
+            .expect("Path should be valid")
+            .to_string_lossy()
+            .to_string();
 
         path.add_extension("yaml");
         let content = std::fs::read_to_string(path).unwrap();
 
-        let node: OptionalNode = serde_yaml::from_str(&content).unwrap();
+        let node: OptionalNodeDeserialize = serde_yaml::from_str(&content).unwrap();
 
         // We want all vectors to be empty vectors by default
         // Sadly we have to this manually
         // TODO: Find a better way of doing this
-        let default = OptionalNode {
+        let default = OptionalNodeDeserialize {
             patches: Some(OptionalPatches {
                 all: Some(vec![]),
                 control_plane: Some(vec![]),
             }),
             kernel_args: vec![].into(),
             ..Default::default()
-        };
+        }
+        .apply(default.clone());
 
         // Combine all the optional node parts into complete struct
-        let mut node: Node = default
+        let node: NodeDeserialize = default
-            // Apply cluster default settings
-            .apply(cluster.default.clone())
-            // Apply hostname based on filename
-            .apply(named)
             // Override node specific settings
             .apply(node)
             .try_into()
             .unwrap();
 
         // Prepend the cluster base values
-        let mut kernel_args = cluster.base.kernel_args.clone();
+        let mut kernel_args = base.kernel_args.clone();
         kernel_args.extend(node.kernel_args);
-        node.kernel_args = kernel_args;
 
-        let patches = cluster.base.patches.clone().extend(node.patches);
+        let patches = base.patches.clone().extend(node.patches);
-        node.patches = patches;
+
+        let node = Node {
+            hostname,
+            arch: node.arch,
+            schematic: node.schematic,
+            r#type: node.r#type,
+            network: node.network,
+            ntp: node.ntp,
+            install: node.install,
+            kernel_args,
+            patches: Default::default(),
+            sops: node.sops,
+        };
 
         // Render patches
-        node.patches = node.patches.clone().render(env, cluster, &node);
+        let patches = patches.render(env, cluster, &node);
 
-        node
+        Node { patches, ..node }
+    }
+}
+
+impl JsonSchema for Node {
+    fn schema_name() -> std::borrow::Cow<'static, str> {
+        "Node".into()
+    }
+
+    fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
+        OptionalNodeDeserialize::json_schema(generator)
     }
 }

src/patch.rs

@@ -6,46 +6,40 @@ use serde::{Deserialize, Serialize};
 use crate::cluster::Cluster;
 use crate::node::Node;
 
-#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, PartialEq, Eq)]
-#[serde(untagged)]
-pub(crate) enum Patch {
-    Name(String),
-    #[serde(skip_deserializing)]
-    #[schemars(with = "serde_json::Value")]
-    Resolved(serde_yaml::Value),
-}
-
 #[optional_struct]
 #[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, PartialEq, Eq, Default)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
-pub struct Patches {
+pub(crate) struct Patches<T> {
-    pub(crate) all: Vec<Patch>,
+    pub(crate) all: Vec<T>,
-    pub(crate) control_plane: Vec<Patch>,
+    pub(crate) control_plane: Vec<T>,
 }
 
-fn render(patches: Vec<Patch>, env: &Environment, cluster: &Cluster, node: &Node) -> Vec<Patch> {
+pub(crate) type OptionalPatchesString = OptionalPatches<String>;
+
+fn render(
+    patches: Vec<String>,
+    env: &Environment,
+    cluster: &Cluster,
+    node: &Node,
+) -> Vec<serde_yaml::Value> {
     patches
         .into_iter()
-        .map(|patch| {
+        .map(|name| {
-            if let Patch::Name(name) = patch {
+            let content = env
-                let content = env
+                .get_template(&name)
-                    .get_template(&name)
+                .unwrap()
-                    .unwrap()
+                .render(context! {
-                    .render(context! {
+                    node,
-                        node,
+                    cluster
-                        cluster
+                })
-                    })
+                .unwrap();
-                    .unwrap();
 
-                Patch::Resolved(serde_yaml::from_str(&content).unwrap())
+            serde_yaml::from_str(&content).unwrap()
-            } else {
-                patch
-            }
         })
         .collect()
 }
 
-impl Patches {
+impl Patches<String> {
     pub(crate) fn extend(mut self, other: Self) -> Self {
         self.all.extend(other.all);
         self.control_plane.extend(other.control_plane);
@@ -56,8 +50,13 @@ impl Patches {
         }
     }
 
-    pub(crate) fn render(self, env: &Environment, cluster: &Cluster, node: &Node) -> Self {
+    pub(crate) fn render(
-        Self {
+        self,
+        env: &Environment,
+        cluster: &Cluster,
+        node: &Node,
+    ) -> Patches<serde_yaml::Value> {
+        Patches {
             all: render(self.all.clone(), env, cluster, node),
             control_plane: render(self.control_plane.clone(), env, cluster, node),
         }

src/schematic.rs

@@ -1,14 +1,14 @@
 use schemars::JsonSchema;
 use serde::{Deserialize, Deserializer, Serialize};
 
-use crate::get_talos_config_path;
+use crate::get_talos_path;
 
 fn deserialize_schematic<'de, D>(deserializer: D) -> Result<String, D::Error>
 where
     D: Deserializer<'de>,
 {
     let name: String = Deserialize::deserialize(deserializer)?;
-    let path = get_talos_config_path().join("schematics").join(name);
+    let path = get_talos_path().join("schematics").join(name);
     let content = std::fs::read_to_string(path).unwrap().trim().to_owned();
 
     let client = reqwest::blocking::Client::new();
@@ -31,3 +31,9 @@ where
 
 #[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, PartialEq, Eq)]
 pub(crate) struct Schematic(#[serde(deserialize_with = "deserialize_schematic")] String);
+
+impl std::fmt::Display for Schematic {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(&self.0)
+    }
+}

src/secret.rs

@@ -1,7 +1,7 @@
 use schemars::JsonSchema;
 use serde::{Deserialize, Deserializer, Serialize};
 
-use crate::get_talos_config_path;
+use crate::get_talos_path;
 
 #[derive(Debug, Deserialize, JsonSchema)]
 #[serde(rename_all = "camelCase", untagged)]
@@ -18,7 +18,7 @@ where
     let value = match secret {
         SecretHelper::String(value) => value,
         SecretHelper::File { file } => {
-            let path = get_talos_config_path().join("secrets").join(file);
+            let path = get_talos_path().join("secrets").join(file);
             std::fs::read_to_string(path).unwrap().trim().to_owned()
         }
     };

xtask/src/main.rs

@@ -1,8 +1,8 @@
 use std::fs::File;
 use std::io::Write;
 
-use crete::cluster::OptionalCluster;
+use crete::cluster::Cluster;
-use crete::node::OptionalNode;
+use crete::node::Node;
 use repo_path::repo_path;
 use schemars::{JsonSchema, schema_for};
 
@@ -19,6 +19,6 @@ where
 }
 
 fn main() {
-    write::<OptionalCluster>("cluster");
+    write::<Cluster>("cluster");
-    write::<OptionalNode>("node");
+    write::<Node>("node");
 }