infra/crete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases 4 Activity

refactor: Big internal refactor

Browse Source
This commit is contained in:
Dreaded_X
2026-04-01 06:18:57 +02:00
parent a7578a6b16
commit 7a83a8a4be
11 changed files with 240 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files
schemas/cluster.json
+9 -25
View File
@@ -1,6 +1,6 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "OptionalCluster",
"title": "Cluster",
"type": "object",
"properties": {
"base": {
@@ -11,8 +11,7 @@
{
"type": "null"
}
],
"writeOnly": true
]
},
"clusterEnv": {
"anyOf": [
@@ -32,8 +31,7 @@
"format": "ipv4"
},
"default": {
"$ref": "#/$defs/OptionalNode",
"writeOnly": true
"$ref": "#/$defs/OptionalNodeDeserialize"
},
"nodes": {
"type": [
@@ -41,7 +39,7 @@
"null"
],
"items": {
"$ref": "#/$defs/NodeEntry"
"type": "string"
}
},
"secretsFile": {
@@ -76,13 +74,6 @@
"amd64"
]
},
"NodeEntry": {
"anyOf": [
{
"type": "string"
}
]
},
"NodeType": {
"type": "string",
"enum": [
@@ -196,7 +187,7 @@
},
"additionalProperties": false
},
"OptionalNode": {
"OptionalNodeDeserialize": {
"type": "object",
"properties": {
"arch": {
@@ -296,7 +287,7 @@
"null"
],
"items": {
"$ref": "#/$defs/Patch"
"type": "string"
}
},
"controlPlane": {
@@ -305,7 +296,7 @@
"null"
],
"items": {
"$ref": "#/$defs/Patch"
"type": "string"
}
}
},
@@ -366,26 +357,19 @@
},
"additionalProperties": false
},
"Patch": {
"anyOf": [
{
"type": "string"
}
]
},
"Patches": {
"type": "object",
"properties": {
"all": {
"type": "array",
"items": {
"$ref": "#/$defs/Patch"
"type": "string"
}
},
"controlPlane": {
"type": "array",
"items": {
"$ref": "#/$defs/Patch"
"type": "string"
}
}
},
schemas/node.json
+3 -10
View File
@@ -1,6 +1,6 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "OptionalNode",
"title": "Node",
"type": "object",
"properties": {
"arch": {
@@ -192,7 +192,7 @@
"null"
],
"items": {
"$ref": "#/$defs/Patch"
"type": "string"
}
},
"controlPlane": {
@@ -201,7 +201,7 @@
"null"
],
"items": {
"$ref": "#/$defs/Patch"
"type": "string"
}
}
},
@@ -236,13 +236,6 @@
},
"additionalProperties": false
},
"Patch": {
"anyOf": [
{
"type": "string"
}
]
},
"Schematic": {
"type": "string"
},
src/cluster.rs
+68 -51
View File
@@ -8,18 +8,19 @@ use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use walkdir::WalkDir;
use crate::get_talos_config_path;
use crate::node::{Node, OptionalNode};
use crate::environment::PathEnvironment;
use crate::get_talos_path;
use crate::node::{Node, OptionalNodeDeserialize};
use crate::patch::Patches;
#[optional_struct]
#[derive(Debug, Deserialize, JsonSchema, Clone)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct Base {
pub(crate) struct Base {
#[serde(default)]
pub(crate) kernel_args: Vec<String>,
#[serde(default)]
pub(crate) patches: Patches,
pub(crate) patches: Patches<String>,
}
#[optional_struct]
@@ -30,95 +31,96 @@ pub struct Version {
talos: semver::Version,
}
#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, PartialEq, Eq)]
#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, Copy, PartialEq, Eq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub enum ClusterEnv {
Production,
Staging,
}
#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, PartialEq, Eq)]
#[serde(untagged)]
pub enum NodeEntry {
Name(String),
#[serde(skip_deserializing)]
Node(Box<Node>),
}
#[optional_struct]
#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
#[derive(Debug, Deserialize, JsonSchema, Clone)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct Cluster {
#[serde(skip_deserializing)]
name: String,
struct ClusterDeserialize {
#[optional_rename(OptionalVersion)]
#[optional_wrap]
version: Version,
nodes: Vec<NodeEntry>,
nodes: Vec<String>,
cluster_env: ClusterEnv,
control_plane_ip: Ipv4Addr,
secrets_file: PathBuf,
#[serde(default, skip_serializing)]
#[serde(default)]
#[optional_skip_wrap]
pub(crate) default: OptionalNode,
default: OptionalNodeDeserialize,
#[optional_rename(OptionalBase)]
#[optional_wrap]
#[serde(skip_serializing)]
pub(crate) base: Base,
// pub secrets_file: PathBuf,
base: Base,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Cluster {
name: String,
version: Version,
nodes: Vec<Node>,
cluster_env: ClusterEnv,
control_plane_ip: Ipv4Addr,
secrets_file: PathBuf,
}
impl Cluster {
pub fn get(cluster_name: &str, env: &Environment) -> Self {
let path = get_talos_config_path()
.join("clusters")
.join("default.yaml");
pub(crate) fn get(cluster_name: &str, env: &Environment) -> Self {
let path = get_talos_path().join("clusters").join("default.yaml");
let default: OptionalCluster = if path.exists() {
let default: OptionalClusterDeserialize = if path.exists() {
let content = std::fs::read_to_string(path).unwrap();
serde_yaml::from_str(&content).unwrap()
} else {
Default::default()
};
let mut path = get_talos_config_path().join("clusters").join(cluster_name);
let mut path = get_talos_path().join("clusters").join(cluster_name);
path.add_extension("yaml");
let content = std::fs::read_to_string(path).unwrap();
let mut cluster: OptionalCluster = serde_yaml::from_str(&content).unwrap();
cluster.name = Some(cluster_name.to_string());
let mut cluster: OptionalClusterDeserialize = serde_yaml::from_str(&content).unwrap();
// For some reason apply on the cluster does not properly apply to the default settings...
// So we manually apply it here first
cluster.default = default.default.clone().apply(cluster.default);
let mut cluster: Self = default.apply(cluster).try_into().unwrap();
let cluster: ClusterDeserialize = default.apply(cluster).try_into().unwrap();
let nodes = cluster.nodes;
let default = cluster.default;
let base = cluster.base;
cluster.nodes = cluster
.nodes
.clone()
.into_iter()
.map(|node_entry| {
if let NodeEntry::Name(name) = node_entry {
NodeEntry::Node(Box::new(Node::get(&name, env, &cluster)))
} else {
node_entry
}
})
.collect();
cluster.secrets_file = get_talos_config_path()
let secrets_file = get_talos_path()
.join("secrets")
.join(cluster.secrets_file)
.absolute()
.unwrap();
cluster
}
let cluster = Self {
name: cluster_name.into(),
version: cluster.version,
nodes: vec![],
cluster_env: cluster.cluster_env,
control_plane_ip: cluster.control_plane_ip,
secrets_file,
};
let nodes = nodes
.iter()
.map(|name| Node::get(name, env, &cluster, &default, &base))
.collect();
Self { nodes, ..cluster }
}
pub fn get_clusters(env: &Environment) -> Vec<Cluster> {
WalkDir::new(get_talos_config_path().join("clusters"))
pub fn get_all() -> Vec<Cluster> {
let env = PathEnvironment::new_patches();
WalkDir::new(get_talos_path().join("clusters"))
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| e.metadata().unwrap().is_file())
@@ -132,7 +134,22 @@ pub fn get_clusters(env: &Environment) -> Vec<Cluster> {
return None;
}
Some(Cluster::get(&name, env))
Some(Cluster::get(&name, &env))
})
.collect()
}
pub fn nodes(&self) -> &[Node] {
&self.nodes
}
}
impl JsonSchema for Cluster {
fn schema_name() -> std::borrow::Cow<'static, str> {
"Cluster".into()
}
fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
OptionalClusterDeserialize::json_schema(generator)
}
}
src/environment.rs
+17 -23
View File
@@ -6,7 +6,7 @@ use std::str::FromStr;
use minijinja::{AutoEscape, Environment, path_loader};
use walkdir::WalkDir;
use crate::{get_repo_path, get_talos_config_path};
use crate::{get_configs_path, get_talos_path};
/// Transparent wrapper around minijinja::Environment that loads templates from a path and
/// configures better defaults. It also implements IntoIter, making it possible to iterate over all
@@ -44,6 +44,20 @@ impl<'a> PathEnvironment<'a> {
}
});
// Helper function for getting the path to kubeconfig files
env.add_filter("kubeconfig", move |names: Vec<String>| {
names
.iter()
.map(|name| {
get_configs_path()
.join(name)
.join("kubeconfig")
.to_string_lossy()
.to_string()
})
.collect::<Vec<_>>()
});
// Add path loader
env.set_loader(path_loader(&path));
@@ -53,28 +67,8 @@ impl<'a> PathEnvironment<'a> {
}
}
pub fn new_patches() -> Self {
Self::new(&get_talos_config_path().join("patches"))
}
pub fn new_templates(repo: &Path) -> Self {
let mut env = Self::new(&get_repo_path().join("templates"));
let path = repo.absolute().unwrap();
env.env.add_filter("kubeconfig", move |names: Vec<String>| {
names
.iter()
.map(|name| {
path.join("configs")
.join(name)
.join("kubeconfig")
.to_string_lossy()
.to_string()
})
.collect::<Vec<_>>()
});
env
pub(crate) fn new_patches() -> Self {
Self::new(&get_talos_path().join("patches"))
}
}
src/lib.rs
+6 -2
View File
@@ -17,10 +17,14 @@ pub fn set_repo_path(path: impl Into<PathBuf>) {
.expect("Repo path already initialized");
}
fn get_repo_path() -> &'static Path {
pub fn get_repo_path() -> &'static Path {
REPO_PATH.get().expect("Repo path not initialized")
}
fn get_talos_config_path() -> PathBuf {
pub fn get_talos_path() -> PathBuf {
get_repo_path().join("talos")
}
pub fn get_configs_path() -> PathBuf {
get_repo_path().join("configs")
}
src/main.rs
+5 -5
View File
@@ -2,9 +2,9 @@ mod cli;
use clap::{CommandFactory, Parser};
use clap_complete::{Shell, generate as generate_complete};
use crete::cluster::get_clusters;
use crete::cluster::Cluster;
use crete::environment::PathEnvironment;
use crete::set_repo_path;
use crete::{get_repo_path, set_repo_path};
use minijinja::context;
use thiserror::Error;
@@ -19,8 +19,7 @@ enum Error {
fn generate(opts: &GlobalOpts) -> Result<(), Error> {
set_repo_path(&opts.repo);
let patch_env = PathEnvironment::new_patches();
let clusters = get_clusters(&patch_env);
let clusters = Cluster::get_all();
if clusters.is_empty() {
return Err(Error::NoClustersFound);
@@ -32,7 +31,8 @@ fn generate(opts: &GlobalOpts) -> Result<(), Error> {
}
std::fs::create_dir(&path).unwrap();
let template_env = PathEnvironment::new_templates(&opts.repo);
// Render templates
let template_env = PathEnvironment::new(&get_repo_path().join("templates"));
for template_name in &template_env {
let template = template_env.get_template(&template_name).unwrap();
src/node.rs
+74 -32
View File
@@ -5,9 +5,9 @@ use optional_struct::{Applicable, optional_struct};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use crate::cluster::Cluster;
use crate::get_talos_config_path;
use crate::patch::{OptionalPatches, Patches};
use crate::cluster::{Base, Cluster};
use crate::get_talos_path;
use crate::patch::{OptionalPatches, OptionalPatchesString, Patches};
use crate::schematic::Schematic;
use crate::secret::Secret;
@@ -19,6 +19,15 @@ enum NodeType {
ControlPlane,
}
impl From<NodeType> for &str {
fn from(value: NodeType) -> Self {
match value {
NodeType::Worker => "worker",
NodeType::ControlPlane => "controlplane",
}
}
}
#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, Copy, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
enum NodeArch {
@@ -59,11 +68,9 @@ struct Install {
}
#[optional_struct]
#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, PartialEq, Eq)]
#[derive(Debug, Deserialize, JsonSchema, Clone, PartialEq, Eq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct Node {
#[serde(skip_deserializing)]
hostname: String,
pub(crate) struct NodeDeserialize {
arch: NodeArch,
schematic: Schematic,
r#type: NodeType,
@@ -75,64 +82,99 @@ pub struct Node {
#[optional_wrap]
install: Install,
kernel_args: Vec<String>,
#[optional_rename(OptionalPatches)]
#[optional_rename(OptionalPatchesString)]
#[optional_wrap]
pub(crate) patches: Patches,
patches: Patches<String>,
sops: Secret,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Node {
hostname: String,
arch: NodeArch,
schematic: Schematic,
r#type: NodeType,
network: Network,
ntp: String,
install: Install,
kernel_args: Vec<String>,
patches: Patches<serde_yaml::Value>,
sops: Secret,
}
impl Node {
pub fn get(node_name: &str, env: &Environment, cluster: &Cluster) -> Self {
let mut path = get_talos_config_path().join("nodes").join(node_name);
let named = OptionalNode {
hostname: Some(
path.file_name()
pub(crate) fn get(
node_name: &str,
env: &Environment,
cluster: &Cluster,
default: &OptionalNodeDeserialize,
base: &Base,
) -> Self {
let mut path = get_talos_path().join("nodes").join(node_name);
let hostname = path
.file_name()
.expect("Path should be valid")
.to_string_lossy()
.to_string(),
),
..OptionalNode::default()
};
.to_string();
path.add_extension("yaml");
let content = std::fs::read_to_string(path).unwrap();
let node: OptionalNode = serde_yaml::from_str(&content).unwrap();
let node: OptionalNodeDeserialize = serde_yaml::from_str(&content).unwrap();
// We want all vectors to be empty vectors by default
// Sadly we have to this manually
// TODO: Find a better way of doing this
let default = OptionalNode {
let default = OptionalNodeDeserialize {
patches: Some(OptionalPatches {
all: Some(vec![]),
control_plane: Some(vec![]),
}),
kernel_args: vec![].into(),
..Default::default()
};
}
.apply(default.clone());
// Combine all the optional node parts into complete struct
let mut node: Node = default
// Apply cluster default settings
.apply(cluster.default.clone())
// Apply hostname based on filename
.apply(named)
let node: NodeDeserialize = default
// Override node specific settings
.apply(node)
.try_into()
.unwrap();
// Prepend the cluster base values
let mut kernel_args = cluster.base.kernel_args.clone();
let mut kernel_args = base.kernel_args.clone();
kernel_args.extend(node.kernel_args);
node.kernel_args = kernel_args;
let patches = cluster.base.patches.clone().extend(node.patches);
node.patches = patches;
let patches = base.patches.clone().extend(node.patches);
let node = Node {
hostname,
arch: node.arch,
schematic: node.schematic,
r#type: node.r#type,
network: node.network,
ntp: node.ntp,
install: node.install,
kernel_args,
patches: Default::default(),
sops: node.sops,
};
// Render patches
node.patches = node.patches.clone().render(env, cluster, &node);
let patches = patches.render(env, cluster, &node);
node
Node { patches, ..node }
}
}
impl JsonSchema for Node {
fn schema_name() -> std::borrow::Cow<'static, str> {
"Node".into()
}
fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
OptionalNodeDeserialize::json_schema(generator)
}
}
src/patch.rs
+21 -22
View File
@@ -6,28 +6,25 @@ use serde::{Deserialize, Serialize};
use crate::cluster::Cluster;
use crate::node::Node;
#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, PartialEq, Eq)]
#[serde(untagged)]
pub(crate) enum Patch {
Name(String),
#[serde(skip_deserializing)]
#[schemars(with = "serde_json::Value")]
Resolved(serde_yaml::Value),
}
#[optional_struct]
#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, PartialEq, Eq, Default)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct Patches {
pub(crate) all: Vec<Patch>,
pub(crate) control_plane: Vec<Patch>,
pub(crate) struct Patches<T> {
pub(crate) all: Vec<T>,
pub(crate) control_plane: Vec<T>,
}
fn render(patches: Vec<Patch>, env: &Environment, cluster: &Cluster, node: &Node) -> Vec<Patch> {
pub(crate) type OptionalPatchesString = OptionalPatches<String>;
fn render(
patches: Vec<String>,
env: &Environment,
cluster: &Cluster,
node: &Node,
) -> Vec<serde_yaml::Value> {
patches
.into_iter()
.map(|patch| {
if let Patch::Name(name) = patch {
.map(|name| {
let content = env
.get_template(&name)
.unwrap()
@@ -37,15 +34,12 @@ fn render(patches: Vec<Patch>, env: &Environment, cluster: &Cluster, node: &Node
})
.unwrap();
Patch::Resolved(serde_yaml::from_str(&content).unwrap())
} else {
patch
}
serde_yaml::from_str(&content).unwrap()
})
.collect()
}
impl Patches {
impl Patches<String> {
pub(crate) fn extend(mut self, other: Self) -> Self {
self.all.extend(other.all);
self.control_plane.extend(other.control_plane);
@@ -56,8 +50,13 @@ impl Patches {
}
}
pub(crate) fn render(self, env: &Environment, cluster: &Cluster, node: &Node) -> Self {
Self {
pub(crate) fn render(
self,
env: &Environment,
cluster: &Cluster,
node: &Node,
) -> Patches<serde_yaml::Value> {
Patches {
all: render(self.all.clone(), env, cluster, node),
control_plane: render(self.control_plane.clone(), env, cluster, node),
}
src/schematic.rs
+8 -2
View File
@@ -1,14 +1,14 @@
use schemars::JsonSchema;
use serde::{Deserialize, Deserializer, Serialize};
use crate::get_talos_config_path;
use crate::get_talos_path;
fn deserialize_schematic<'de, D>(deserializer: D) -> Result<String, D::Error>
where
D: Deserializer<'de>,
{
let name: String = Deserialize::deserialize(deserializer)?;
let path = get_talos_config_path().join("schematics").join(name);
let path = get_talos_path().join("schematics").join(name);
let content = std::fs::read_to_string(path).unwrap().trim().to_owned();
let client = reqwest::blocking::Client::new();
@@ -31,3 +31,9 @@ where
#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, PartialEq, Eq)]
pub(crate) struct Schematic(#[serde(deserialize_with = "deserialize_schematic")] String);
impl std::fmt::Display for Schematic {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
src/secret.rs
+2 -2
View File
@@ -1,7 +1,7 @@
use schemars::JsonSchema;
use serde::{Deserialize, Deserializer, Serialize};
use crate::get_talos_config_path;
use crate::get_talos_path;
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase", untagged)]
@@ -18,7 +18,7 @@ where
let value = match secret {
SecretHelper::String(value) => value,
SecretHelper::File { file } => {
let path = get_talos_config_path().join("secrets").join(file);
let path = get_talos_path().join("secrets").join(file);
std::fs::read_to_string(path).unwrap().trim().to_owned()
}
};
xtask/src/main.rs
+4 -4
View File
@@ -1,8 +1,8 @@
use std::fs::File;
use std::io::Write;
use crete::cluster::OptionalCluster;
use crete::node::OptionalNode;
use crete::cluster::Cluster;
use crete::node::Node;
use repo_path::repo_path;
use schemars::{JsonSchema, schema_for};
@@ -19,6 +19,6 @@ where
}
fn main() {
write::<OptionalCluster>("cluster");
write::<OptionalNode>("node");
write::<Cluster>("cluster");
write::<Node>("node");
}