From 14097100228e5d3b80f155100898131a959496d6 Mon Sep 17 00:00:00 2001 From: BOTAlex Date: Thu, 7 May 2026 13:47:57 +0200 Subject: [PATCH 1/2] kubernetes, bare metaling --- configuration.nix | 7 +- flake.lock | 18 +-- kubenetes/containerd.nix | 69 ++++++++++ .../kubernetes => kubenetes}/kubelet.nix | 6 +- kubenetes/kubernetes.nix | 4 + modules/drivers/nvidia.nix | 35 +++-- vms/kube-daddy/default.nix | 120 ----------------- vms/kube-networking.nix | 123 ++++++++++++++++++ vms/kube-vm/kube-vm.nix | 23 +++- vms/kube-vm/kubernetes/kubernetes.nix | 4 - 10 files changed, 253 insertions(+), 156 deletions(-) create mode 100644 kubenetes/containerd.nix rename {vms/kube-vm/kubernetes => kubenetes}/kubelet.nix (74%) create mode 100644 kubenetes/kubernetes.nix create mode 100644 vms/kube-networking.nix delete mode 100644 vms/kube-vm/kubernetes/kubernetes.nix diff --git a/configuration.nix b/configuration.nix index fbc31d7..70968d0 100755 --- a/configuration.nix +++ b/configuration.nix @@ -33,9 +33,12 @@ ./modules/nfs.nix - ./vms/kube-vm + ./kubenetes + + # ./vms/kube-vm # ./vms/kube-vm2 - ./vms/kube-daddy + # ./vms/kube-daddy + # ./vms/kube-networking.nix # ./networking/wireguard-kube.nix # ./modules/de.nix diff --git a/flake.lock b/flake.lock index 9b0f100..721bce6 100755 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1777655179, - "narHash": "sha256-Rx7RvgxgFeoaJUddpuVbJ2jaaAp7qH6wV9PwBmLvfz4=", + "lastModified": 1777894865, + "narHash": "sha256-agINDb/tr4v2uaVmgE/i0dY1M2JJdzUI/Caup/MWEGU=", "owner": "nix-community", "repo": "home-manager", - "rev": "feda41500ec53fcd4e3131de7b0441bce08fd3e9", + "rev": "9c6f1307e1d76a2285d8001e1b8bc281bfe15dac", "type": "github" }, "original": { @@ -48,11 +48,11 @@ ] }, "locked": { - "lastModified": 1777181277, - "narHash": "sha256-yVJbd07ortDRAttDFmDV5p220aOLTHgVAx//0nW/xW8=", + "lastModified": 1777787189, + "narHash": "sha256-2KUbS/HhzWW3kkkY1+RiWj9mJ76VEXw8lBJzcCFKzfY=", "owner": "nix-community", "repo": "nix-index-database", - "rev": "b8eb7acee0f7604fe1bf6a5b3dcf5254369180fa", + "rev": "2dea2b920e7127b3afa8506713f23536651de312", "type": "github" }, "original": { @@ -63,11 +63,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1777268161, - "narHash": "sha256-bxrdOn8SCOv8tN4JbTF/TXq7kjo9ag4M+C8yzzIRYbE=", + "lastModified": 1777578337, + "narHash": "sha256-Ad49moKWeXtKBJNy2ebiTQUEgdLyvGmTeykAQ9xM+Z4=", "owner": "nixos", "repo": "nixpkgs", - "rev": "1c3fe55ad329cbcb28471bb30f05c9827f724c76", + "rev": "15f4ee454b1dce334612fa6843b3e05cf546efab", "type": "github" }, "original": { diff --git a/kubenetes/containerd.nix b/kubenetes/containerd.nix new file mode 100644 index 0000000..bac1d29 --- /dev/null +++ b/kubenetes/containerd.nix @@ -0,0 +1,69 @@ +{ config, lib, pkgs, ... }: let +nvidiaEnabled = builtins.elem "nvidia" config.services.xserver.videoDrivers; +in { + config = lib.mkMerge [ + (lib.mkIf nvidiaEnabled { + virtualisation.docker.enableNvidia = true; + virtualisation.docker.enable = true; + hardware.nvidia-container-toolkit = { + enable = true; + mount-nvidia-executables = true; + mount-nvidia-docker-1-directories = true; + extraArgs = [ "--device-name-strategy=uuid" ]; + }; + + environment.systemPackages = with pkgs; [ nvidia-docker (lib.getOutput "tools" config.hardware.nvidia-container-toolkit.package) runc ]; + services.envfs.enable = true; + virtualisation.docker.daemon.settings.features.cdi = true; + + virtualisation.containerd = { + enable = true; + settings = { + plugins = { + "io.containerd.grpc.v1.cri" = { + # enable_cdi = true; + # cdi_spec_dirs = [ "/etc/cdi" "/var/run/cdi" ]; + containerd = { + # default_runtime_name = "runc"; + runtimes.runc.options = { SystemdCgroup = false; }; + default_runtime_name = "nvidia"; + runtimes = { + nvidia = { + privileged_without_host_devices = false; + runtime_type = "io.containerd.runc.v2"; + options = { + BinaryName = "${lib.getOutput "tools" config.hardware.nvidia-container-toolkit.package}/bin/nvidia-container-runtime"; + }; + }; + }; + }; + }; + }; + }; + }; + }) + (lib.mkIf (!nvidiaEnabled ) { + virtualisation.containerd = { + enable = true; +settings = { + version = 2; + plugins."io.containerd.grpc.v1.cri" = { + # This is the critical part for Kubeadm + containerd.runtimes.runc = { + runtime_type = "io.containerd.runc.v2"; + options.SystemdCgroup = true; + }; + + # # Keep your existing settings + # containerd.snapshotter = lib.mkIf config.boot.zfs.enabled (lib.mkOptionDefault "zfs"); + # cni.bin_dir = lib.mkOptionDefault "${pkgs.cni-plugins}/bin"; + }; + }; + }; + }) + ]; + + + # # Tell the Kubelet to use containerd + # services.kubernetes.kubelet.containerRuntimeEndpoint = "unix:///run/containerd/containerd.sock"; +} diff --git a/vms/kube-vm/kubernetes/kubelet.nix b/kubenetes/kubelet.nix similarity index 74% rename from vms/kube-vm/kubernetes/kubelet.nix rename to kubenetes/kubelet.nix index 853886f..fac03c1 100644 --- a/vms/kube-vm/kubernetes/kubelet.nix +++ b/kubenetes/kubelet.nix @@ -11,7 +11,11 @@ serviceConfig = { # Use the kubelet binary from the Nix store - ExecStart = "${pkgs.kubernetes}/bin/kubelet"; + ExecStart = ''${pkgs.kubernetes}/bin/kubelet \ + --config=/var/lib/kubelet/config.yaml \ + --kubeconfig=/etc/kubernetes/kubelet.conf \ + --pod-manifest-path=/etc/kubernetes/manifests + ''; Restart = "always"; RestartSec = 10; diff --git a/kubenetes/kubernetes.nix b/kubenetes/kubernetes.nix new file mode 100644 index 0000000..b8fba36 --- /dev/null +++ b/kubenetes/kubernetes.nix @@ -0,0 +1,4 @@ +{ pkgs, ... }: { + imports = [ ./kubelet.nix ./containerd.nix ]; + environment.systemPackages = with pkgs; [ kubernetes cri-tools ]; +} diff --git a/modules/drivers/nvidia.nix b/modules/drivers/nvidia.nix index c77892d..c6f58d0 100644 --- a/modules/drivers/nvidia.nix +++ b/modules/drivers/nvidia.nix @@ -1,13 +1,12 @@ -{ - config, - pkgs, - lib, - ... +{ config +, pkgs +, lib +, ... }: { nixpkgs.config.nvidia.acceptLicense = true; - services.xserver.videoDrivers = ["nvidia"]; + services.xserver.videoDrivers = [ "nvidia" ]; - boot.kernelParams = ["nvidia.NVreg_PreserveVideoMemoryAllocations=1"]; + boot.kernelParams = [ "nvidia.NVreg_PreserveVideoMemoryAllocations=1" ]; hardware.graphics = { enable = true; @@ -18,7 +17,7 @@ }; hardware.nvidia = { - package = config.boot.kernelPackages.nvidiaPackages.stable; + package = config.boot.kernelPackages.nvidiaPackages.legacy_580; modesetting.enable = true; open = false; nvidiaSettings = true; @@ -43,23 +42,23 @@ specialisation = { Battery.configuration = { - system.nixos.tags = ["Battery"]; + system.nixos.tags = [ "Battery" ]; boot.extraModprobeConfig = '' blacklist nouveau options nouveau modeset=0 ''; services.udev.extraRules = '' - # Remove NVIDIA USB xHCI Host Controller devices, if present - ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x0c0330", ATTR{power/control}="auto", ATTR{remove}="1" - # Remove NVIDIA USB Type-C UCSI devices, if present - ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x0c8000", ATTR{power/control}="auto", ATTR{remove}="1" - # Remove NVIDIA Audio devices, if present - ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x040300", ATTR{power/control}="auto", ATTR{remove}="1" - # Remove NVIDIA VGA/3D controller devices - ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x03[0-9]*", ATTR{power/control}="auto", ATTR{remove}="1" + # Remove NVIDIA USB xHCI Host Controller devices, if present + ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x0c0330", ATTR{power/control}="auto", ATTR{remove}="1" + # Remove NVIDIA USB Type-C UCSI devices, if present + ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x0c8000", ATTR{power/control}="auto", ATTR{remove}="1" + # Remove NVIDIA Audio devices, if present + ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x040300", ATTR{power/control}="auto", ATTR{remove}="1" + # Remove NVIDIA VGA/3D controller devices + ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x03[0-9]*", ATTR{power/control}="auto", ATTR{remove}="1" ''; - boot.blacklistedKernelModules = ["nouveau" "nvidia" "nvidia_drm" "nvidia_modeset"]; + boot.blacklistedKernelModules = [ "nouveau" "nvidia" "nvidia_drm" "nvidia_modeset" ]; }; }; } diff --git a/vms/kube-daddy/default.nix b/vms/kube-daddy/default.nix index e26d5e9..a6c49ea 100644 --- a/vms/kube-daddy/default.nix +++ b/vms/kube-daddy/default.nix @@ -6,124 +6,4 @@ config = ./kube-daddy.nix; }; - systemd.services.kube-iptable = { - after = [ "network.target" ]; - wantedBy = [ "multi-user.target" ]; - - serviceConfig = { - Type = "oneshot"; - ExecStart = "${pkgs.iptables}/bin/iptables -t nat -I POSTROUTING 1 -s 10.0.0.0/24 -o enp8s0 -j MASQUERADE "; - RemainAfterExit = true; - User = "root"; - }; - - stopIfChanged = true; - }; - - networking = { - bridges = { - "br0" = { - interfaces = [ - "microvm-tap1" - "microvm-tap2" - ]; - }; - }; - - interfaces.br0.ipv4.addresses = [ - { - address = "10.0.0.1"; - prefixLength = 24; - } - ]; - - localCommands = '' - ip tuntap add dev microvm-tap1 mode tap user root || true - ip tuntap add dev microvm-tap2 mode tap user root || true - ip link set microvm-tap1 up - ip link set microvm-tap2 up - ''; - - nat = { - enable = true; - externalInterface = "enp8s0"; - internalIPs = [ "10.0.0.0/24" ]; - forwardPorts = [ - { - sourcePort = 8877; - destination = "10.0.0.2:8888"; - proto = "tcp"; - } - # { # Access this directly from host by 10.0.0.2:4325 - # sourcePort = 4325; # argocd - # destination = "10.0.0.2:8080"; - # proto = "tcp"; - # } - { - # Netbird - sourcePort = 3478; - destination = "10.0.0.2:3478"; - proto = "udp"; - } - { - sourcePort = 6443; - destination = "10.0.0.2:6443"; - proto = "tcp"; - } - { - sourcePort = 4123; - destination = "10.0.0.2:4123"; - proto = "tcp"; - } - { - sourcePort = 8472; - destination = "10.0.0.2:8472"; - proto = "udp"; - } - { - sourcePort = 2379; - destination = "10.0.0.2:2379"; - proto = "udp"; - } - { - sourcePort = 2380; - destination = "10.0.0.2:2380"; - proto = "udp"; - } - { - sourcePort = 2379; - proto = "tcp"; - destination = "10.0.0.2:2379"; - } - { - sourcePort = 2380; - destination = "10.0.0.2:2380"; - proto = "tcp"; - } - { - sourcePort = 4001; - destination = "10.0.0.2:4001"; - proto = "udp"; - } - { - sourcePort = 4001; - destination = "10.0.0.2:4001"; - proto = "tcp"; - } - # If your app uses UDP (like HTTP/3 or QUIC), add this too: - # { sourcePort = 8888; destination = "10.0.0.2:8888"; proto = "udp"; } - ]; - }; - - # 5. Update Firewall to trust the Bridge - firewall.trustedInterfaces = [ "br0" ]; - }; - - systemd.tmpfiles.rules = [ - "d /var/lib/microvms/shared 0755 microvm kvm -" - "d /var/lib/microvms/shared/kube 0755 microvm kvm -" - "d /var/lib/microvms/shared/docking 0755 microvm kvm -" - "d /var/lib/microvms/shared/.config 0755 microvm kvm -" - "d /var/lib/microvms/shared/.local 0755 microvm kvm -" - ]; } diff --git a/vms/kube-networking.nix b/vms/kube-networking.nix new file mode 100644 index 0000000..0424d7b --- /dev/null +++ b/vms/kube-networking.nix @@ -0,0 +1,123 @@ +{ pkgs, ... }: +{ + systemd.services.kube-iptable = { + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Type = "oneshot"; + ExecStart = "${pkgs.iptables}/bin/iptables -t nat -I POSTROUTING 1 -s 10.0.0.0/24 -o enp8s0 -j MASQUERADE "; + RemainAfterExit = true; + User = "root"; + }; + + stopIfChanged = true; + }; + + networking = { + bridges = { + "br0" = { + interfaces = [ + "microvm-tap1" + "microvm-tap2" + ]; + }; + }; + + interfaces.br0.ipv4.addresses = [ + { + address = "10.0.0.1"; + prefixLength = 24; + } + ]; + + localCommands = '' + ip tuntap add dev microvm-tap1 mode tap user root || true + ip tuntap add dev microvm-tap2 mode tap user root || true + ip link set microvm-tap1 up + ip link set microvm-tap2 up + ''; + + nat = { + enable = true; + externalInterface = "enp8s0"; + internalIPs = [ "10.0.0.0/24" ]; + forwardPorts = [ + { + sourcePort = 8877; + destination = "10.0.0.2:8888"; + proto = "tcp"; + } + # { # Access this directly from host by 10.0.0.2:4325 + # sourcePort = 4325; # argocd + # destination = "10.0.0.2:8080"; + # proto = "tcp"; + # } + { + # Netbird + sourcePort = 3478; + destination = "10.0.0.2:3478"; + proto = "udp"; + } + { + sourcePort = 6443; + destination = "10.0.0.2:6443"; + proto = "tcp"; + } + { + sourcePort = 4123; + destination = "10.0.0.2:4123"; + proto = "tcp"; + } + { + sourcePort = 8472; + destination = "10.0.0.2:8472"; + proto = "udp"; + } + { + sourcePort = 2379; + destination = "10.0.0.2:2379"; + proto = "udp"; + } + { + sourcePort = 2380; + destination = "10.0.0.2:2380"; + proto = "udp"; + } + { + sourcePort = 2379; + proto = "tcp"; + destination = "10.0.0.2:2379"; + } + { + sourcePort = 2380; + destination = "10.0.0.2:2380"; + proto = "tcp"; + } + { + sourcePort = 4001; + destination = "10.0.0.2:4001"; + proto = "udp"; + } + { + sourcePort = 4001; + destination = "10.0.0.2:4001"; + proto = "tcp"; + } + # If your app uses UDP (like HTTP/3 or QUIC), add this too: + # { sourcePort = 8888; destination = "10.0.0.2:8888"; proto = "udp"; } + ]; + }; + + # 5. Update Firewall to trust the Bridge + firewall.trustedInterfaces = [ "br0" ]; + }; + + systemd.tmpfiles.rules = [ + "d /var/lib/microvms/shared 0755 microvm kvm -" + "d /var/lib/microvms/shared/kube 0755 microvm kvm -" + "d /var/lib/microvms/shared/docking 0755 microvm kvm -" + "d /var/lib/microvms/shared/.config 0755 microvm kvm -" + "d /var/lib/microvms/shared/.local 0755 microvm kvm -" + ]; +} diff --git a/vms/kube-vm/kube-vm.nix b/vms/kube-vm/kube-vm.nix index f1b6ca9..315af5d 100644 --- a/vms/kube-vm/kube-vm.nix +++ b/vms/kube-vm/kube-vm.nix @@ -45,6 +45,8 @@ microvm = { # Choose your hypervisor: "qemu", "firecracker", "cloud-hypervisor", etc. hypervisor = "qemu"; + vcpu = 8; + mem = 8192 / 3; # Create a tap interface or user networking interfaces = [{ @@ -65,11 +67,28 @@ volumes = [{ image = "/var/lib/microvms/kube-vm/kube-vm.img"; mountPoint = "/"; - size = 512 * 4; # Size in MB + size = 512 * 8; # Size in MB }]; }; - boot.kernelModules = [ "br_netfilter" ]; + boot.kernel.sysctl = { + "net.ipv4.ip_forward" = 1; + "net.ipv6.conf.all.forwarding" = 1; + }; + + systemd.services."load-kernel-modules" = { + enable = true; + description = "Modprobe kernel modules"; + # before = [ "flannel.service" ]; + wantedBy = [ + "multi-user.target" + # "flannel.service" + ]; + + script = '' + ${pkgs.kmod}/bin/modprobe br_netfilter + ''; + }; networking = { hostName = "kube-vm"; diff --git a/vms/kube-vm/kubernetes/kubernetes.nix b/vms/kube-vm/kubernetes/kubernetes.nix deleted file mode 100644 index 2a61a73..0000000 --- a/vms/kube-vm/kubernetes/kubernetes.nix +++ /dev/null @@ -1,4 +0,0 @@ -{ pkgs, ... }: { - imports = [ ./kublet.nix ]; - environment.systemPackages = with pkgs; [ kubernetes ]; -} From 1c2ce93ca13a0ec4518e5c2599f6e95d6c3438c0 Mon Sep 17 00:00:00 2001 From: BOTAlex Date: Thu, 7 May 2026 13:48:41 +0200 Subject: [PATCH 2/2] kubernetes, bare metaling --- kubenetes/{kubernetes.nix => default.nix} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename kubenetes/{kubernetes.nix => default.nix} (100%) diff --git a/kubenetes/kubernetes.nix b/kubenetes/default.nix similarity index 100% rename from kubenetes/kubernetes.nix rename to kubenetes/default.nix