feat: Migrate to serde-saphyr

Fixes: #30

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "ipxe/ipxe"]
-	path = ipxe/ipxe
-	url = git@github.com:rkubes/ipxe

.release-plz.toml

@@ -1,5 +1,14 @@
 [workspace]
 pr_name = "chore(release): Release v{{ version }}"
+changelog_path = "./CHANGELOG.md"
+
+[[package]]
+name = "crete"
+changelog_include = ["ipxe"]
+
+[[package]]
+name = "ipxe"
+release = false
 
 [[package]]
 name = "xtask"

Cargo.toml

@@ -10,8 +10,8 @@ license = "GPL-3.0-or-later"
 members = ["ipxe", "xtask"]
 
 [workspace.dependencies]
-schemars = { version = "1.2.2", features = ["semver1"] }
-serde_json = "1.0.151"
+schemars = { version = "1.2.3", features = ["semver1"] }
+serde_json = "1.0.154"
 
 [dependencies]
 async-walkdir = "2.2.0"
@@ -19,8 +19,8 @@ byteorder = "1.5.0"
 clap = { version = "4.6.7", features = ["derive"] }
 clap_complete = "4.6.11"
 futures = "0.3.34"
-gix-discover = { version = "0.56.0", features = ["sha1"] }
-minijinja = { version = "2.24.0", features = ["json", "loader"] }
+gix-discover = { version = "0.57.0", features = ["sha1"] }
+minijinja = { version = "3.0.1", features = ["json", "serde"] }
 netascii = "0.1.0"
 num_enum = "0.7.6"
 optional_struct = "0.5.2"
@@ -34,11 +34,12 @@ serde = { version = "1.0.229", features = ["derive"] }
 serde_json = { workspace = true }
 serde_yaml = "0.9.34"
 thiserror = "2.0.21"
-tokio = { version = "1.52.1", features = ["full"] }
+tokio = { version = "1.54.0", features = ["full"] }
 ipxe = { path = "./ipxe" }
 axum = { version = "0.8.9", features = ["macros"] }
 socket2 = "0.6.5"
 socket-pktinfo = "0.4.1"
+serde-saphyr = "2.0.0"
 
 [lib]
 test = false

ipxe/Cargo.toml

@@ -4,7 +4,13 @@ version = "0.1.0"
 edition = "2024"
 
 [build-dependencies]
-build-print = "1.0.1"
+diffy = "0.5.2"
+gix = { version = "0.89.0", default-features = false, features = [
+  "blocking-network-client",
+  "worktree-mutation",
+  "sha1",
+  "blocking-http-transport-reqwest-rust-tls",
+] }
 
 [dependencies]
 include_dir = "0.7.4"

ipxe/build.rs

@@ -1,57 +1,180 @@
-use std::env;
-use std::path::PathBuf;
-use std::process::{Command, Stdio};
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::sync::atomic::AtomicBool;
 
-use build_print::error;
+use diffy::apply;
+use diffy::patch_set::{FileOperation, ParseOptions, PatchSet};
+use gix::Repository;
+use gix::progress::Discard;
+
+const URL: &str = "https://github.com/ipxe/ipxe";
+const REVISION: &str = "2fa08affe9a1a1422766f54b2fb045234b571f06";
+const FILES: &[&str] = &["bin/ipxe.pxe", "bin-x86_64-efi/ipxe.efi"];
+
+const SRC_DIR: &str = "src";
 
 fn main() {
-    let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap())
+    let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap())
         .canonicalize()
         .unwrap();
-    let src_dir = manifest_dir.join("ipxe").join("src");
+    let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap())
+        .canonicalize()
+        .unwrap();
+    let repo_dir = out_dir.join("repo");
+
+    let repo = checkout(&out_dir, &repo_dir);
+
+    apply_patches(&repo, &manifest_dir, &repo_dir);
+    println!("cargo::rerun-if-changed=patches");
+
+    build(&manifest_dir, &repo_dir);
+    println!("cargo::rerun-if-changed=src/embed.ipxe");
+
+    copy_files(&repo_dir, &out_dir);
+}
+
+fn checkout(out_dir: &Path, repo_dir: &Path) -> Repository {
+    // Stamp file is used instead of comparing against HEAD to guard against incomplete clones
+    let stamp = out_dir.join("checkout.stamp");
+    if std::fs::read_to_string(&stamp).is_ok_and(|stamp| stamp == REVISION)
+        && let Ok(repo) = gix::open(repo_dir)
+    {
+        return repo;
+    }
+
+    // Cleanup any left-overs
+    std::fs::remove_file(&stamp).ok();
+    std::fs::remove_dir_all(repo_dir).ok();
+
+    std::fs::create_dir_all(repo_dir).unwrap();
+
+    let mut prepare = gix::prepare_clone(URL, repo_dir)
+        .unwrap()
+        .with_revision(Some(REVISION))
+        .unwrap();
+
+    let interrupt = AtomicBool::new(false);
+    let (mut checkout, _) = prepare.fetch_then_checkout(Discard, &interrupt).unwrap();
+    let (repo, _) = checkout.main_worktree(Discard, &interrupt).unwrap();
+
+    std::fs::write(&stamp, REVISION).unwrap();
+
+    repo
+}
+
+fn apply_patches(repo: &Repository, manifest_dir: &Path, repo_dir: &Path) {
+    let commit_id = gix::ObjectId::from_hex(REVISION.as_bytes()).unwrap();
+    let tree = repo.find_commit(commit_id).unwrap().tree().unwrap();
+
+    // Get all the patches and sort them based on name
+    let mut patch_files: Vec<PathBuf> = std::fs::read_dir(manifest_dir.join("patches"))
+        .unwrap()
+        .map(|e| e.unwrap().path())
+        .filter(|e| e.extension().is_some_and(|e| e == "patch"))
+        .collect();
+    patch_files.sort();
+
+    let mut patched = HashMap::new();
+    for path in patch_files {
+        let text = std::fs::read_to_string(&path).unwrap();
+        let patch_set = PatchSet::parse(&text, ParseOptions::gitdiff());
+
+        for patch in patch_set {
+            let patch = patch.unwrap();
+
+            match patch.operation().strip_prefix(1) {
+                FileOperation::Modify { original, modified } => {
+                    assert_eq!(original, modified);
+                    let path = original.to_string();
+
+                    // Either get the current in-memory patched file, otherwise get the prisine file
+                    let current = patched
+                        .remove(&path)
+                        .unwrap_or_else(|| pristine(&tree, &path));
+
+                    let new = apply(&current, patch.patch().as_text().unwrap()).unwrap();
+                    patched.insert(path, new);
+                }
+                _ => todo!(),
+            }
+        }
+    }
+
+    // If the file was previously patched, but now isn't restore the pristine file
+    for path in modified_files(repo) {
+        if !patched.contains_key(&path) {
+            std::fs::write(repo_dir.join(&path), pristine(&tree, &path)).unwrap();
+        }
+    }
+
+    // Write all the patched files
+    for (path, content) in &patched {
+        let path = repo_dir.join(path);
+        let content = content.as_bytes();
+
+        // But only if the file is actually different from the one on disk
+        // This prevents changing the mtime unnecessarily
+        if std::fs::read(&path).ok().as_deref() != Some(content) {
+            std::fs::write(path, content).unwrap();
+        }
+    }
+}
+
+fn pristine(tree: &gix::Tree<'_>, path: &str) -> String {
+    let entry = tree.lookup_entry_by_path(path).unwrap().unwrap();
+
+    String::from_utf8(entry.object().unwrap().data.clone()).unwrap()
+}
+
+fn modified_files(repo: &gix::Repository) -> Vec<String> {
+    repo.status(Discard)
+        .unwrap()
+        .untracked_files(gix::status::UntrackedFiles::None)
+        .into_index_worktree_iter(Vec::<gix::bstr::BString>::new())
+        .unwrap()
+        .map(|item| item.unwrap().rela_path().to_string())
+        .collect()
+}
+
+fn build(manifest_dir: &Path, repo_dir: &Path) {
+    let src_dir = repo_dir.join(SRC_DIR);
 
     let mut command = Command::new("make");
     command
-        .stdout(Stdio::piped())
         .current_dir(&src_dir)
         .env_remove("DEBUG")
         .env_remove("PROFILE")
-        .args([
-            "-j",
-            &std::thread::available_parallelism()
-                .unwrap()
-                .get()
-                .to_string(),
-        ])
         .arg(format!(
             "EMBED={}",
             manifest_dir
                 .join("src")
                 .join("embed.ipxe")
-                .to_string_lossy(),
-        ));
+                .to_string_lossy()
+        ))
+        .args(FILES);
 
-    let files = ["bin/ipxe.pxe", "bin-x86_64-efi/ipxe.efi"].map(PathBuf::from);
-    for file in &files {
-        command.arg(file);
+    if let Ok(flags) = std::env::var("CARGO_MAKEFLAGS") {
+        command.env("MAKEFLAGS", flags);
     }
 
     let output = command.output().unwrap();
+
     if !output.status.success() {
-        error!("{}", String::from_utf8_lossy(&output.stderr));
+        panic!("make failed:\n{}", String::from_utf8_lossy(&output.stderr));
+    }
 }
 
-    let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap())
-        .canonicalize()
-        .unwrap()
-        .join("ipxe");
+fn copy_files(repo_dir: &Path, out_dir: &Path) {
+    let src_dir = repo_dir.join(SRC_DIR);
+    let bin_dir = out_dir.join("bin");
 
-    std::fs::remove_dir_all(&out_dir).ok();
-    for file in &files {
+    std::fs::create_dir_all(&bin_dir).unwrap();
+    for file in FILES {
+        let file = Path::new(file);
         let input = src_dir.join(file);
-        let output = out_dir.join(file.file_name().unwrap());
+        let output = bin_dir.join(file.file_name().unwrap());
 
-        std::fs::create_dir_all(output.parent().unwrap()).unwrap();
         std::fs::copy(input, output).unwrap();
     }
 }

ipxe/patches/0001_enable_protocols.patch

@@ -0,0 +1,19 @@
+diff --git a/src/config/general.h b/src/config/general.h
+index 43858dceb..54cb1b657 100644
+--- a/src/config/general.h
++++ b/src/config/general.h
+@@ -60,10 +60,10 @@ FILE_SECBOOT ( PERMITTED );
+ //#define HTTP_ENC_PEERDIST	/* PeerDist content encoding */
+
+ /* Disable protocols not historically included in BIOS builds */
+-#if defined ( PLATFORM_pcbios )
+-  #undef DOWNLOAD_PROTO_HTTPS
+-  #undef HTTP_AUTH_NTLM
+-#endif
++// #if defined ( PLATFORM_pcbios )
++//   #undef DOWNLOAD_PROTO_HTTPS
++//   #undef HTTP_AUTH_NTLM
++// #endif
+
+ /*****************************************************************************
+  *

ipxe/src/lib.rs

@@ -1,3 +1,3 @@
 use include_dir::{Dir, include_dir};
 
-pub static IPXE_DIR: Dir<'_> = include_dir!("$OUT_DIR/ipxe");
+pub static IPXE_DIR: Dir<'_> = include_dir!("$OUT_DIR/bin");

src/cluster.rs

@@ -102,7 +102,7 @@ impl Cluster {
         path.add_extension("yaml");
         let content = tokio::fs::read_to_string(path).await.unwrap();
 
-        let mut cluster: OptionalClusterDeserialize = serde_yaml::from_str(&content).unwrap();
+        let mut cluster: OptionalClusterDeserialize = serde_saphyr::from_str(&content).unwrap();
 
         // For some reason apply on the cluster does not properly apply to the default settings...
         // So we manually apply it here first

src/environment.rs

@@ -4,8 +4,8 @@ use std::path::{Path, PathBuf};
 
 use async_walkdir::WalkDir;
 use futures::{Stream, StreamExt};
-use minijinja::value::ViaDeserialize;
-use minijinja::{AutoEscape, Environment, path_loader};
+use minijinja::syntax::SyntaxConfig;
+use minijinja::{AutoEscape, Environment, path_loader, value};
 
 use crate::{get_configs_path, get_talos_path};
 
@@ -19,21 +19,27 @@ pub struct PathEnvironment<'a> {
 }
 
 fn apply_netmask(
-    ip: ViaDeserialize<Ipv4Addr>,
-    netmask: ViaDeserialize<Ipv4Addr>,
+    ip: value::Serde<Ipv4Addr>,
+    netmask: value::Serde<Ipv4Addr>,
 ) -> Result<String, minijinja::Error> {
     Ok((*ip & *netmask).to_string())
 }
 
 pub fn configure_environment(env: &mut Environment) {
     // Configure jinja
-    env.set_trim_blocks(true);
-    env.set_lstrip_blocks(true);
+    // Configure jinja
+    env.set_syntax(
+        SyntaxConfig::builder()
+            .trim_blocks(true)
+            .lstrip_blocks(true)
+            .build()
+            .expect("Syntax config should be valid"),
+    );
     env.set_undefined_behavior(minijinja::UndefinedBehavior::Strict);
     env.set_auto_escape_callback(|_| AutoEscape::None);
 
     // Add filters
-    env.add_filter("to_prefix", |netmask: ViaDeserialize<Ipv4Addr>| {
+    env.add_filter("to_prefix", |netmask: value::Serde<Ipv4Addr>| {
         let mask = netmask.to_bits();
         let prefix = mask.leading_ones();
 

src/file.rs

@@ -1,20 +1,18 @@
 use crete::cluster::Cluster;
 use crete::environment::configure_environment;
 use ipxe::IPXE_DIR;
-use minijinja::{Environment, context};
+use minijinja::{Environment, context, value};
 use regex::regex;
 
-// TODO: Make this a proper async function
-pub fn get_file(
+pub async fn get_file(
     filename: impl ToString,
     http_url: impl ToString,
-) -> impl Future<Output = Option<(Vec<u8>, &'static str)>> {
+) -> Option<(Vec<u8>, &'static str)> {
     let re = regex!(r"^talos/(?<cluster>.+)/(?<node>.+)\.ya?ml$");
 
     let filename: String = filename.to_string();
     let re = re.clone();
 
-    async move {
     const IPXE_PATTERN: &str = "ipxe/";
     if filename == format!("{IPXE_PATTERN}boot.ipxe") {
         let mut env = Environment::new();
@@ -28,7 +26,7 @@ pub fn get_file(
         let clusters = Cluster::get_all().await;
         let content = template
             .render(context! {
-                    clusters => clusters.values().collect::<Vec<_>>(),
+                clusters => value::Serde(clusters.values().collect::<Vec<_>>()),
                 httpUrl => http_url.to_string()
             })
             .unwrap();
@@ -55,4 +53,3 @@ pub fn get_file(
         None
     }
 }
-}

src/main.rs

@@ -16,7 +16,7 @@ use crete::talosconfig::{get_current_context, set_context};
 use crete::{get_configs_path, get_repo_path, set_repo_path, tftp};
 use futures::StreamExt;
 use futures::future::join_all;
-use minijinja::context;
+use minijinja::{context, value};
 
 use crate::cli::{Cli, Commands, GlobalOpts};
 use crate::error::Error;
@@ -44,8 +44,8 @@ async fn generate(opts: &GlobalOpts) -> Result<(), Error> {
 
         let content = template
             .render(context! {
-                clusters => clusters.values().collect::<Vec<_>>(),
-                root => opts.repo
+                clusters => value::Serde(clusters.values().collect::<Vec<_>>()),
+                root => value::Serde(&opts.repo)
             })
             .unwrap();
 
@@ -114,13 +114,10 @@ async fn main() -> Result<(), Error> {
                 axum::serve(listener, app).await.unwrap();
             });
 
-            tftp::serve(ip, move |filename, ip| {
-                let filename = filename.to_owned();
-                async move {
+            tftp::serve(ip, move |filename, ip| async move {
                 get_file(filename, format!("http://{ip}:{port}"))
                     .await
                     .map(|(file, _)| file)
-                }
             })
             .await?;
         }

src/node.rs

@@ -102,7 +102,7 @@ pub struct Node {
     ntp: String,
     install: Install,
     kernel_args: Vec<String>,
-    patches: Patches<serde_yaml::Value>,
+    patches: Patches<serde_json::Value>,
     sops: Secret,
 }
 
@@ -124,7 +124,7 @@ impl Node {
         path.add_extension("yaml");
         let content = tokio::fs::read_to_string(path).await.unwrap();
 
-        let node: OptionalNodeDeserialize = serde_yaml::from_str(&content).unwrap();
+        let node: OptionalNodeDeserialize = serde_saphyr::from_str(&content).unwrap();
 
         // We want all vectors to be empty vectors by default
         // Sadly we have to this manually

src/patch.rs

@@ -1,4 +1,4 @@
-use minijinja::{Environment, context};
+use minijinja::{Environment, context, value};
 use optional_struct::optional_struct;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
@@ -21,7 +21,7 @@ async fn render(
     env: &Environment<'static>,
     cluster: &Cluster,
     node: &Node,
-) -> Vec<serde_yaml::Value> {
+) -> Vec<serde_json::Value> {
     let env = env.clone();
     let cluster = cluster.clone();
     let node = node.clone();
@@ -35,12 +35,12 @@ async fn render(
                     .get_template(&name)
                     .unwrap()
                     .render(context! {
-                        node,
-                        cluster
+                        node => value::Serde(&node),
+                        cluster => value::Serde(&cluster)
                     })
                     .unwrap();
 
-                serde_yaml::from_str(&content).unwrap()
+                serde_saphyr::from_str(&content).unwrap()
             })
             .collect()
     })
@@ -64,7 +64,7 @@ impl Patches<String> {
         env: &Environment<'static>,
         cluster: &Cluster,
         node: &Node,
-    ) -> Patches<serde_yaml::Value> {
+    ) -> Patches<serde_json::Value> {
         Patches {
             all: render(self.all.clone(), env, cluster, node).await,
             control_plane: render(self.control_plane.clone(), env, cluster, node).await,

src/talosconfig.rs

@@ -20,7 +20,7 @@ pub async fn get_current_context() -> Option<String> {
     tokio::fs::read(&TalosConfig::merged_path())
         .await
         .ok()
-        .and_then(|config| serde_yaml::from_slice(&config).ok())
+        .and_then(|config| serde_saphyr::from_slice(&config).ok())
         .map(|config: TalosConfig| config.context)
 }
 

src/tftp/server.rs

@@ -5,7 +5,7 @@ use std::time::Duration;
 use socket_pktinfo::{PktInfo, PktInfoUdpSocket};
 use socket2::Domain;
 use thiserror::Error;
-use tokio::io::unix::AsyncFd;
+use tokio::io::unix::{AsyncFd, AsyncFdRegisterError};
 use tokio::net::UdpSocket;
 
 use crate::tftp::packet::{ErrorCode, Mode, PACKET_SIZE};
@@ -29,6 +29,8 @@ pub enum Error {
     UnexpectedPacket(Packet),
     #[error("Timeout")]
     Timeout,
+    #[error("AsyncFd: {0}")]
+    AsyncFd(#[from] AsyncFdRegisterError<PktInfoUdpSocket>),
 }
 
 pub async fn handle_connection<F, Fut>(
@@ -37,7 +39,7 @@ pub async fn handle_connection<F, Fut>(
     callback: F,
 ) -> Result<(), Error>
 where
-    F: Fn(&str, IpAddr) -> Fut,
+    F: Fn(String, IpAddr) -> Fut,
     Fut: Future<Output = Option<Vec<u8>>>,
 {
     let src = info.addr_src;
@@ -59,7 +61,7 @@ where
     let filename = request.filename();
     println!("{src}: {filename} [{:?}]", request.mode());
 
-    let Some(data) = callback(filename, info.addr_dst).await else {
+    let Some(data) = callback(filename.to_string(), info.addr_dst).await else {
         Packet::error(ErrorCode::FileNotFound, "Unknown file")
             .send_to(&socket, &src)
             .await?;
@@ -152,7 +154,7 @@ where
 
 pub async fn serve<F, Fut>(ip: IpAddr, callback: F) -> Result<!, Error>
 where
-    F: Fn(&str, IpAddr) -> Fut + Clone + Send + 'static,
+    F: Fn(String, IpAddr) -> Fut + Clone + Send + 'static,
     Fut: Future<Output = Option<Vec<u8>>> + Send + 'static,
 {
     let socket = PktInfoUdpSocket::new(Domain::IPV4)?;
@@ -161,7 +163,8 @@ where
     let addr: SocketAddr = (ip, 69).into();
     socket.bind(&addr.into())?;
 
-    let async_fd = AsyncFd::new(socket)?;
+    let async_fd = unsafe { AsyncFd::register(socket)? };
+
     let mut buf = [0u8; PACKET_SIZE];
 
     println!("Tftp server started on {addr}");

typos.toml

@@ -0,0 +1,2 @@
+[default.extend-words]
+rela = "rela"