feat(config)!: Reworked how configuration is loaded

The environment variable `AUTOMATION_CONFIG` has been renamed to
`AUTOMATION__ENTRYPOINT` and can now also be set in `automation.toml` by
specifying:
```
automation = "<path>"
```

Directly accessing the environment variables in lua in no longer
possible. To pass in configuration or secrets you can now instead make
use of the `variables` and `secrets` modules.

To set values in these modules you can either specify them in
`automation.toml`:
```
[variables]
<name> = <value>

[secrets]
<name> = <value>
```
Note that these values will get converted to a string.

You can also specify the environment variables
`AUTOMATION__VARIABLES__<name>` and `AUTOMATION__SECRETS__<name>` to
set variables and secrets respectively. By adding the suffix `__FILE` to
the environment variable name the contents of a file can be loaded into
the variable or secret.

Note that variables and secrets are identical in functionality and the
name difference exists purely to make it clear that secret values are
meant to be kept secret.

.gitea/workflows/build.yml

@@ -34,9 +34,9 @@ jobs:
                     --name automation_rs \
                     --network mqtt \
                     -e RUST_LOG=automation=debug \
-                    -e MQTT_PASSWORD=${{ secrets.MQTT_PASSWORD }} \
-                    -e HUE_TOKEN=${{ secrets.HUE_TOKEN }} \
-                    -e NTFY_TOPIC=${{ secrets.NTFY_TOPIC }} \
+                    -e AUTOMATION__SECRETS__MQTT_PASSWORD=${{ secrets.MQTT_PASSWORD }} \
+                    -e AUTOMATION__SECRETS__HUE_TOKEN=${{ secrets.HUE_TOKEN }} \
+                    -e AUTOMATION__SECRETS__NTFY_TOPIC=${{ secrets.NTFY_TOPIC }} \
                     git.huizinga.dev/dreaded_x/automation_rs@${{ needs.build.outputs.digest }}
 
                   docker network connect web automation_rs

.gitignore

@@ -1,2 +1,3 @@
 /target
 .env
+automation.toml

Cargo.lock

@@ -95,6 +95,7 @@ dependencies = [
  "automation_devices",
  "automation_lib",
  "axum",
+ "config",
  "dotenvy",
  "google_home",
  "hostname",
@@ -327,6 +328,19 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "config"
+version = "0.15.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0faa974509d38b33ff89282db9c3295707ccf031727c0de9772038ec526852ba"
+dependencies = [
+ "async-trait",
+ "pathdiff",
+ "serde",
+ "toml",
+ "winnow",
+]
+
 [[package]]
 name = "core-foundation"
 version = "0.9.4"
@@ -466,7 +480,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -1262,6 +1276,12 @@ dependencies = [
  "windows-targets",
 ]
 
+[[package]]
+name = "pathdiff"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3"
+
 [[package]]
 name = "percent-encoding"
 version = "2.3.2"
@@ -1409,7 +1429,7 @@ dependencies = [
  "once_cell",
  "socket2",
  "tracing",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -1586,7 +1606,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -1784,6 +1804,15 @@ dependencies = [
  "syn 2.0.106",
 ]
 
+[[package]]
+name = "serde_spanned"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40734c41988f7306bb04f0ecf60ec0f3f1caa34290e4e8ea471dcd3346483b83"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde_urlencoded"
 version = "0.7.1"
@@ -2035,6 +2064,37 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "toml"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75129e1dc5000bfbaa9fee9d1b21f974f9fbad9daec557a521ee6e080825f6e8"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_parser",
+ "winnow",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_parser"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10"
+dependencies = [
+ "winnow",
+]
+
 [[package]]
 name = "tower"
 version = "0.5.2"
@@ -2491,6 +2551,15 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
+[[package]]
+name = "winnow"
+version = "0.7.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "winsafe"
 version = "0.0.19"

Cargo.toml

@@ -69,6 +69,10 @@ async-trait = { workspace = true }
 automation_devices = { workspace = true }
 automation_lib = { workspace = true }
 axum = { workspace = true }
+config = { version = "0.15.15", default-features = false, features = [
+  "async",
+  "toml",
+] }
 dotenvy = { workspace = true }
 google_home = { workspace = true }
 hostname = { workspace = true }

Dockerfile

@@ -21,6 +21,6 @@ RUN cargo auditable build --release
 
 FROM gcr.io/distroless/cc-debian12:nonroot AS runtime
 COPY --from=builder /app/target/release/automation /app/automation
-ENV AUTOMATION_CONFIG=/app/config.lua
+ENV AUTOMATION__ENTRYPOINT=/app/config.lua
 COPY ./config.lua /app/config.lua
 CMD [ "/app/automation" ]

config.lua

@@ -1,16 +1,13 @@
 local device_manager = require("device_manager")
 local utils = require("utils")
+local secrets = require("secrets")
+local debug = require("variables").debug or false
 
 print(_VERSION)
 
 local host = utils.get_hostname()
 print("Running @" .. host)
 
-local debug, value = pcall(utils.get_env, "DEBUG")
-if debug and value ~= "true" then
-	debug = false
-end
-
 local function mqtt_z2m(topic)
 	return "zigbee2mqtt/" .. topic
 end
@@ -28,12 +25,12 @@ local mqtt_client = require("mqtt").new({
 	port = 8883,
 	client_name = "automation-" .. host,
 	username = "mqtt",
-	password = utils.get_env("MQTT_PASSWORD"),
+	password = secrets.mqtt_password,
 	tls = host == "zeus" or host == "hephaestus",
 })
 
 local ntfy = Ntfy.new({
-	topic = utils.get_env("NTFY_TOPIC"),
+	topic = secrets.ntfy_topic,
 })
 device_manager:add(ntfy)
 
@@ -147,7 +144,7 @@ on_light:add(function(light)
 end)
 
 local hue_ip = "10.0.0.102"
-local hue_token = utils.get_env("HUE_TOKEN")
+local hue_token = secrets.hue_token
 
 local hue_bridge = HueBridge.new({
 	identifier = "hue_bridge",

src/config.rs

@@ -0,0 +1,17 @@
+use std::collections::HashMap;
+
+use serde::Deserialize;
+
+#[derive(Debug, Deserialize)]
+pub struct Config {
+    #[serde(default = "default_entrypoint")]
+    pub entrypoint: String,
+    #[serde(default)]
+    pub variables: HashMap<String, String>,
+    #[serde(default)]
+    pub secrets: HashMap<String, String>,
+}
+
+fn default_entrypoint() -> String {
+    "./config.lua".into()
+}

src/main.rs

@@ -1,4 +1,6 @@
 #![feature(iter_intersperse)]
+mod config;
+mod secret;
 mod web;
 
 use std::net::SocketAddr;
@@ -6,6 +8,7 @@ use std::path::Path;
 use std::process;
 use std::time::{SystemTime, UNIX_EPOCH};
 
+use ::config::{Environment, File};
 use automation_lib::config::{FulfillmentConfig, MqttConfig};
 use automation_lib::device_manager::DeviceManager;
 use automation_lib::helpers;
@@ -14,6 +17,7 @@ use axum::extract::{FromRef, State};
 use axum::http::StatusCode;
 use axum::routing::post;
 use axum::{Json, Router};
+use config::Config;
 use dotenvy::dotenv;
 use google_home::{GoogleHome, Request, Response};
 use mlua::LuaSerdeExt;
@@ -22,6 +26,8 @@ use tokio::net::TcpListener;
 use tracing::{debug, error, info, warn};
 use web::{ApiError, User};
 
+use crate::secret::EnvironmentSecretFile;
+
 #[derive(Clone)]
 struct AppState {
     pub openid_url: String,
@@ -69,7 +75,21 @@ async fn app() -> anyhow::Result<()> {
     dotenv().ok();
 
     tracing_subscriber::fmt::init();
-    // console_subscriber::init();
+
+    let config: Config = ::config::Config::builder()
+        .add_source(
+            File::with_name(&format!("{}.toml", std::env!("CARGO_PKG_NAME"))).required(false),
+        )
+        .add_source(
+            Environment::default()
+                .prefix(std::env!("CARGO_PKG_NAME"))
+                .separator("__"),
+        )
+        .add_source(EnvironmentSecretFile::default())
+        .build()
+        .unwrap()
+        .try_deserialize()
+        .unwrap();
 
     info!("Starting automation_rs...");
 
@@ -133,11 +153,10 @@ async fn app() -> anyhow::Result<()> {
 
     lua.register_module("device_manager", device_manager.clone())?;
 
+    lua.register_module("variables", lua.to_value(&config.variables)?)?;
+    lua.register_module("secrets", lua.to_value(&config.secrets)?)?;
+
     let utils = lua.create_table()?;
-    let get_env = lua.create_function(|_lua, name: String| {
-        std::env::var(name).map_err(mlua::ExternalError::into_lua_err)
-    })?;
-    utils.set("get_env", get_env)?;
     let get_hostname = lua.create_function(|_lua, ()| {
         hostname::get()
             .map(|name| name.to_str().unwrap_or("unknown").to_owned())
@@ -151,17 +170,13 @@ async fn app() -> anyhow::Result<()> {
             .as_millis())
     })?;
     utils.set("get_epoch", get_epoch)?;
-
     lua.register_module("utils", utils)?;
 
     automation_devices::register_with_lua(&lua)?;
     helpers::register_with_lua(&lua)?;
 
-    // TODO: Make this not hardcoded
-    let config_filename = std::env::var("AUTOMATION_CONFIG").unwrap_or("./config.lua".into());
-    let config_path = Path::new(&config_filename);
-
-    let fulfillment_config: mlua::Value = lua.load(config_path).eval_async().await?;
+    let entrypoint = Path::new(&config.entrypoint);
+    let fulfillment_config: mlua::Value = lua.load(entrypoint).eval_async().await?;
     let fulfillment_config: FulfillmentConfig = lua.from_value(fulfillment_config)?;
 
     // Create google home fulfillment route

src/secret.rs

@@ -0,0 +1,43 @@
+use std::str::from_utf8;
+
+use config::{ConfigError, Source, Value, ValueKind};
+
+#[derive(Debug, Clone, Default)]
+pub struct EnvironmentSecretFile {}
+
+const SUFFIX: &str = "__file";
+const PREFIX: &str = concat!(std::env!("CARGO_PKG_NAME"), "__");
+
+impl Source for EnvironmentSecretFile {
+    fn clone_into_box(&self) -> Box<dyn Source + Send + Sync> {
+        Box::new((*self).clone())
+    }
+
+    fn collect(&self) -> Result<config::Map<String, config::Value>, ConfigError> {
+        Ok(std::env::vars()
+            .flat_map(|(key, value): (String, String)| {
+                let key = key.to_lowercase();
+                if !key.starts_with(PREFIX) {
+                    return None;
+                }
+
+                if !key.ends_with(SUFFIX) {
+                    return None;
+                }
+
+                let suffix_length = key.len() - SUFFIX.len();
+                let key = key[PREFIX.len()..suffix_length].replace("__", ".");
+
+                if key.is_empty() {
+                    return None;
+                }
+
+                let content = from_utf8(&std::fs::read(&value).unwrap())
+                    .unwrap()
+                    .to_owned();
+
+                Some((key, Value::new(Some(&value), ValueKind::String(content))))
+            })
+            .collect())
+    }
+}