kubernetes, bare metaling

configuration.nix

@@ -33,9 +33,12 @@
 
     ./modules/nfs.nix
 
-    ./vms/kube-vm
+    ./kubenetes
+
+    # ./vms/kube-vm
     # ./vms/kube-vm2
-    ./vms/kube-daddy
+    # ./vms/kube-daddy
+    # ./vms/kube-networking.nix
     # ./networking/wireguard-kube.nix
 
     # ./modules/de.nix

flake.lock

@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1777655179,
-        "narHash": "sha256-Rx7RvgxgFeoaJUddpuVbJ2jaaAp7qH6wV9PwBmLvfz4=",
+        "lastModified": 1777894865,
+        "narHash": "sha256-agINDb/tr4v2uaVmgE/i0dY1M2JJdzUI/Caup/MWEGU=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "feda41500ec53fcd4e3131de7b0441bce08fd3e9",
+        "rev": "9c6f1307e1d76a2285d8001e1b8bc281bfe15dac",
         "type": "github"
       },
       "original": {
@@ -48,11 +48,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1777181277,
-        "narHash": "sha256-yVJbd07ortDRAttDFmDV5p220aOLTHgVAx//0nW/xW8=",
+        "lastModified": 1777787189,
+        "narHash": "sha256-2KUbS/HhzWW3kkkY1+RiWj9mJ76VEXw8lBJzcCFKzfY=",
         "owner": "nix-community",
         "repo": "nix-index-database",
-        "rev": "b8eb7acee0f7604fe1bf6a5b3dcf5254369180fa",
+        "rev": "2dea2b920e7127b3afa8506713f23536651de312",
         "type": "github"
       },
       "original": {
@@ -63,11 +63,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1777268161,
-        "narHash": "sha256-bxrdOn8SCOv8tN4JbTF/TXq7kjo9ag4M+C8yzzIRYbE=",
+        "lastModified": 1777578337,
+        "narHash": "sha256-Ad49moKWeXtKBJNy2ebiTQUEgdLyvGmTeykAQ9xM+Z4=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "1c3fe55ad329cbcb28471bb30f05c9827f724c76",
+        "rev": "15f4ee454b1dce334612fa6843b3e05cf546efab",
         "type": "github"
       },
       "original": {

kubenetes/containerd.nix

@@ -0,0 +1,69 @@
+{ config, lib, pkgs, ... }: let
+nvidiaEnabled = builtins.elem "nvidia" config.services.xserver.videoDrivers;
+in {
+  config = lib.mkMerge [
+    (lib.mkIf nvidiaEnabled {
+  virtualisation.docker.enableNvidia = true;
+  virtualisation.docker.enable = true;
+  hardware.nvidia-container-toolkit = {
+    enable = true;
+    mount-nvidia-executables = true;
+    mount-nvidia-docker-1-directories = true;
+    extraArgs = [ "--device-name-strategy=uuid" ];
+  };
+
+  environment.systemPackages = with pkgs; [ nvidia-docker (lib.getOutput "tools" config.hardware.nvidia-container-toolkit.package) runc ];
+  services.envfs.enable = true;
+  virtualisation.docker.daemon.settings.features.cdi = true;
+
+  virtualisation.containerd = {
+    enable = true;
+    settings = {
+      plugins = {
+        "io.containerd.grpc.v1.cri" = {
+          # enable_cdi = true;
+          # cdi_spec_dirs = [ "/etc/cdi" "/var/run/cdi" ];
+          containerd = {
+            # default_runtime_name = "runc";
+            runtimes.runc.options = { SystemdCgroup = false; };
+            default_runtime_name = "nvidia";
+            runtimes = {
+              nvidia = {
+                privileged_without_host_devices = false;
+                runtime_type = "io.containerd.runc.v2";
+                options = {
+                  BinaryName = "${lib.getOutput "tools" config.hardware.nvidia-container-toolkit.package}/bin/nvidia-container-runtime";
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+      })
+    (lib.mkIf (!nvidiaEnabled ) {
+  virtualisation.containerd = {
+    enable = true;
+settings = {
+    version = 2;
+    plugins."io.containerd.grpc.v1.cri" = {
+      # This is the critical part for Kubeadm
+      containerd.runtimes.runc = {
+        runtime_type = "io.containerd.runc.v2";
+        options.SystemdCgroup = true;
+      };
+      
+      # # Keep your existing settings
+      # containerd.snapshotter = lib.mkIf config.boot.zfs.enabled (lib.mkOptionDefault "zfs");
+      # cni.bin_dir = lib.mkOptionDefault "${pkgs.cni-plugins}/bin";
+    };
+  };
+    };
+      })
+  ];
+
+
+  # # Tell the Kubelet to use containerd
+  # services.kubernetes.kubelet.containerRuntimeEndpoint = "unix:///run/containerd/containerd.sock";
+}

kubenetes/default.nix

@@ -0,0 +1,4 @@
+{ pkgs, ... }: {
+  imports = [ ./kubelet.nix ./containerd.nix ];
+  environment.systemPackages = with pkgs; [ kubernetes cri-tools ];
+}

kubenetes/kubelet.nix

@@ -11,7 +11,11 @@
 
     serviceConfig = {
       # Use the kubelet binary from the Nix store
-      ExecStart = "${pkgs.kubernetes}/bin/kubelet";
+      ExecStart = ''${pkgs.kubernetes}/bin/kubelet \
+      --config=/var/lib/kubelet/config.yaml \
+      --kubeconfig=/etc/kubernetes/kubelet.conf \
+      --pod-manifest-path=/etc/kubernetes/manifests
+      '';
 
       Restart = "always";
       RestartSec = 10;

modules/drivers/nvidia.nix

@@ -1,8 +1,7 @@
-{
-  config,
-  pkgs,
-  lib,
-  ...
+{ config
+, pkgs
+, lib
+, ...
 }: {
   nixpkgs.config.nvidia.acceptLicense = true;
   services.xserver.videoDrivers = [ "nvidia" ];
@@ -18,7 +17,7 @@
   };
 
   hardware.nvidia = {
-    package = config.boot.kernelPackages.nvidiaPackages.stable;
+    package = config.boot.kernelPackages.nvidiaPackages.legacy_580;
     modesetting.enable = true;
     open = false;
     nvidiaSettings = true;

vms/kube-daddy/default.nix

@@ -6,124 +6,4 @@
     config = ./kube-daddy.nix;
   };
 
-  systemd.services.kube-iptable = {
-    after = [ "network.target" ];
-    wantedBy = [ "multi-user.target" ];
-
-    serviceConfig = {
-      Type = "oneshot";
-      ExecStart = "${pkgs.iptables}/bin/iptables -t nat -I POSTROUTING 1 -s 10.0.0.0/24 -o enp8s0 -j MASQUERADE ";
-      RemainAfterExit = true;
-      User = "root";
-    };
-
-    stopIfChanged = true;
-  };
-
-  networking = {
-    bridges = {
-      "br0" = {
-        interfaces = [
-          "microvm-tap1"
-          "microvm-tap2"
-        ];
-      };
-    };
-
-    interfaces.br0.ipv4.addresses = [
-      {
-        address = "10.0.0.1";
-        prefixLength = 24;
-      }
-    ];
-
-    localCommands = ''
-      ip tuntap add dev microvm-tap1 mode tap user root || true
-      ip tuntap add dev microvm-tap2 mode tap user root || true
-      ip link set microvm-tap1 up
-      ip link set microvm-tap2 up
-    '';
-
-    nat = {
-      enable = true;
-      externalInterface = "enp8s0";
-      internalIPs = [ "10.0.0.0/24" ];
-      forwardPorts = [
-        {
-          sourcePort = 8877;
-          destination = "10.0.0.2:8888";
-          proto = "tcp";
-        }
-        # { # Access this directly from host by 10.0.0.2:4325
-        #   sourcePort = 4325; # argocd
-        #   destination = "10.0.0.2:8080";
-        #   proto = "tcp";
-        # }
-        {
-          # Netbird
-          sourcePort = 3478;
-          destination = "10.0.0.2:3478";
-          proto = "udp";
-        }
-        {
-          sourcePort = 6443;
-          destination = "10.0.0.2:6443";
-          proto = "tcp";
-        }
-        {
-          sourcePort = 4123;
-          destination = "10.0.0.2:4123";
-          proto = "tcp";
-        }
-        {
-          sourcePort = 8472;
-          destination = "10.0.0.2:8472";
-          proto = "udp";
-        }
-        {
-          sourcePort = 2379;
-          destination = "10.0.0.2:2379";
-          proto = "udp";
-        }
-        {
-          sourcePort = 2380;
-          destination = "10.0.0.2:2380";
-          proto = "udp";
-        }
-        {
-          sourcePort = 2379;
-          proto = "tcp";
-          destination = "10.0.0.2:2379";
-        }
-        {
-          sourcePort = 2380;
-          destination = "10.0.0.2:2380";
-          proto = "tcp";
-        }
-        {
-          sourcePort = 4001;
-          destination = "10.0.0.2:4001";
-          proto = "udp";
-        }
-        {
-          sourcePort = 4001;
-          destination = "10.0.0.2:4001";
-          proto = "tcp";
-        }
-        # If your app uses UDP (like HTTP/3 or QUIC), add this too:
-        # { sourcePort = 8888; destination = "10.0.0.2:8888"; proto = "udp"; }
-      ];
-    };
-
-    # 5. Update Firewall to trust the Bridge
-    firewall.trustedInterfaces = [ "br0" ];
-  };
-
-  systemd.tmpfiles.rules = [
-    "d /var/lib/microvms/shared 0755 microvm kvm -"
-    "d /var/lib/microvms/shared/kube 0755 microvm kvm -"
-    "d /var/lib/microvms/shared/docking 0755 microvm kvm -"
-    "d /var/lib/microvms/shared/.config 0755 microvm kvm -"
-    "d /var/lib/microvms/shared/.local 0755 microvm kvm -"
-  ];
 }

vms/kube-networking.nix

@@ -0,0 +1,123 @@
+{ pkgs, ... }:
+{
+  systemd.services.kube-iptable = {
+    after = [ "network.target" ];
+    wantedBy = [ "multi-user.target" ];
+
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = "${pkgs.iptables}/bin/iptables -t nat -I POSTROUTING 1 -s 10.0.0.0/24 -o enp8s0 -j MASQUERADE ";
+      RemainAfterExit = true;
+      User = "root";
+    };
+
+    stopIfChanged = true;
+  };
+
+  networking = {
+    bridges = {
+      "br0" = {
+        interfaces = [
+          "microvm-tap1"
+          "microvm-tap2"
+        ];
+      };
+    };
+
+    interfaces.br0.ipv4.addresses = [
+      {
+        address = "10.0.0.1";
+        prefixLength = 24;
+      }
+    ];
+
+    localCommands = ''
+      ip tuntap add dev microvm-tap1 mode tap user root || true
+      ip tuntap add dev microvm-tap2 mode tap user root || true
+      ip link set microvm-tap1 up
+      ip link set microvm-tap2 up
+    '';
+
+    nat = {
+      enable = true;
+      externalInterface = "enp8s0";
+      internalIPs = [ "10.0.0.0/24" ];
+      forwardPorts = [
+        {
+          sourcePort = 8877;
+          destination = "10.0.0.2:8888";
+          proto = "tcp";
+        }
+        # { # Access this directly from host by 10.0.0.2:4325
+        #   sourcePort = 4325; # argocd
+        #   destination = "10.0.0.2:8080";
+        #   proto = "tcp";
+        # }
+        {
+          # Netbird
+          sourcePort = 3478;
+          destination = "10.0.0.2:3478";
+          proto = "udp";
+        }
+        {
+          sourcePort = 6443;
+          destination = "10.0.0.2:6443";
+          proto = "tcp";
+        }
+        {
+          sourcePort = 4123;
+          destination = "10.0.0.2:4123";
+          proto = "tcp";
+        }
+        {
+          sourcePort = 8472;
+          destination = "10.0.0.2:8472";
+          proto = "udp";
+        }
+        {
+          sourcePort = 2379;
+          destination = "10.0.0.2:2379";
+          proto = "udp";
+        }
+        {
+          sourcePort = 2380;
+          destination = "10.0.0.2:2380";
+          proto = "udp";
+        }
+        {
+          sourcePort = 2379;
+          proto = "tcp";
+          destination = "10.0.0.2:2379";
+        }
+        {
+          sourcePort = 2380;
+          destination = "10.0.0.2:2380";
+          proto = "tcp";
+        }
+        {
+          sourcePort = 4001;
+          destination = "10.0.0.2:4001";
+          proto = "udp";
+        }
+        {
+          sourcePort = 4001;
+          destination = "10.0.0.2:4001";
+          proto = "tcp";
+        }
+        # If your app uses UDP (like HTTP/3 or QUIC), add this too:
+        # { sourcePort = 8888; destination = "10.0.0.2:8888"; proto = "udp"; }
+      ];
+    };
+
+    # 5. Update Firewall to trust the Bridge
+    firewall.trustedInterfaces = [ "br0" ];
+  };
+
+  systemd.tmpfiles.rules = [
+    "d /var/lib/microvms/shared 0755 microvm kvm -"
+    "d /var/lib/microvms/shared/kube 0755 microvm kvm -"
+    "d /var/lib/microvms/shared/docking 0755 microvm kvm -"
+    "d /var/lib/microvms/shared/.config 0755 microvm kvm -"
+    "d /var/lib/microvms/shared/.local 0755 microvm kvm -"
+  ];
+}

vms/kube-vm/kube-vm.nix

@@ -45,6 +45,8 @@
   microvm = {
     # Choose your hypervisor: "qemu", "firecracker", "cloud-hypervisor", etc.
     hypervisor = "qemu";
+    vcpu = 8;
+    mem = 8192 / 3;
 
     # Create a tap interface or user networking
     interfaces = [{
@@ -65,11 +67,28 @@
     volumes = [{
       image = "/var/lib/microvms/kube-vm/kube-vm.img";
       mountPoint = "/";
-      size = 512 * 4; # Size in MB
+      size = 512 * 8; # Size in MB
     }];
   };
 
-  boot.kernelModules = [ "br_netfilter" ];
+  boot.kernel.sysctl = {
+    "net.ipv4.ip_forward" = 1;
+    "net.ipv6.conf.all.forwarding" = 1; 
+  };
+
+    systemd.services."load-kernel-modules" = {
+    enable = true;
+    description = "Modprobe kernel modules";
+    # before = [ "flannel.service" ];
+    wantedBy = [
+      "multi-user.target"
+      # "flannel.service"
+    ];
+
+    script = ''
+      ${pkgs.kmod}/bin/modprobe br_netfilter
+    '';
+  };
 
   networking = {
     hostName = "kube-vm";

vms/kube-vm/kubernetes/kubernetes.nix

@@ -1,4 +0,0 @@
-{ pkgs, ... }: {
-  imports = [ ./kublet.nix ];
-  environment.systemPackages = with pkgs; [ kubernetes ];
-}