Deepmerge node configs

tools/render

@@ -12,7 +12,7 @@ import git
 import requests
 import yaml
 from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template
-from mergedeep import merge
+from mergedeep import Strategy, merge
 from netaddr import IPAddress
 
 REPO = git.Repo(sys.path[0], search_parent_directories=True)
@@ -38,12 +38,24 @@ TEMPLATES = Environment(
 )
 
 
+# When we try to make a deep copy of the nodes dict it fails as the Template
+# does not implement __deepcopy__, so this wrapper type facilitates that
+class TemplateWrapper:
+    def __init__(self, template: Template):
+        self.template = template
+
+    def __deepcopy__(self, memo):
+        # NOTE: This is not a true deepcopy, but since we know we won't modify
+        # the template this is fine.
+        return self
+
+
 def render_templates(node: dict, args: dict):
     class Inner(json.JSONEncoder):
         def default(self, o):
-            if isinstance(o, Template):
+            if isinstance(o, TemplateWrapper):
                 try:
-                    rendered = o.render(args | {"node": node})
+                    rendered = o.template.render(args | {"node": node})
                 except Exception as e:
                     e.add_note(f"While rendering for: {node['hostname']}")
                     raise e
@@ -84,7 +96,7 @@ def template_constructor(environment: Environment):
         patch_name = loader.construct_scalar(node)
         try:
             template = environment.get_template(f"{patch_name}.yaml")
-            return template
+            return TemplateWrapper(template)
         except Exception:
             raise yaml.MarkedYAMLError("Failed to load patch", node.start_mark)
 
@@ -125,7 +137,12 @@ def get_defaults(directory: pathlib.Path, root: pathlib.Path):
 
     # Stop recursion when reaching root directory
     if directory != root:
-        return get_defaults(directory.parent, root) | yml_data
+        return merge(
+            {},
+            get_defaults(directory.parent, root),
+            yml_data,
+            strategy=Strategy.TYPESAFE_REPLACE,
+        )
     else:
         return yml_data
 
@@ -143,7 +160,7 @@ def main():
         config = yaml.safe_load(fyaml)
 
     with open(ROOT.joinpath("secrets.yaml")) as fyaml:
-        merge(config, yaml.safe_load(fyaml))
+        merge(config, yaml.safe_load(fyaml), strategy=Strategy.TYPESAFE_REPLACE)
 
     template_args = {
         "config": config,
@@ -157,7 +174,12 @@ def main():
 
         with open(fullname) as fyaml:
             yml_data = yaml.load(fyaml, Loader=get_loader(fullname.parent))
-        yml_data = get_defaults(fullname.parent, NODES) | yml_data
+        yml_data = merge(
+            {},
+            get_defaults(fullname.parent, NODES),
+            yml_data,
+            strategy=Strategy.TYPESAFE_REPLACE,
+        )
         yml_data["hostname"] = fullname.stem
         yml_data["filename"] = filename
         nodes.append(yml_data)
@@ -172,11 +194,13 @@ def main():
         )
     )
 
-    # Get all clusters
+    # HACK: We can't hash a dict, so we first convert it to json, the use set
+    # to get all the unique entries, and then convert it back
     # NOTE: This assumes that all nodes in the cluster use the same definition for the cluster
-    clusters = [
-        dict(s) for s in set(frozenset(node["cluster"].items()) for node in nodes)
-    ]
+    clusters = list(
+        json.loads(cluster)
+        for cluster in set(json.dumps(node["cluster"]) for node in nodes)
+    )
 
     template_args |= {"nodes": nodes, "clusters": clusters}