botalex/nixos-server
1
0
Fork 0
mirror of https://github.com/MagicBOTAlex/nixos-server.git synced 2026-05-25 01:12:31 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

kubernetes, bare metaling

Browse source
This commit is contained in:
BOTAlex 2026-05-07 13:47:57 +02:00
parent fc6edfa41b
commit 1409710022
10 changed files with 253 additions and 156 deletions
Download patch file Download diff file Expand all files Collapse all files

7
configuration.nix
View file

@ -33,9 +33,12 @@
./modules/nfs.nix
./vms/kube-vm
./kubenetes
# ./vms/kube-vm
# ./vms/kube-vm2
./vms/kube-daddy
# ./vms/kube-daddy
# ./vms/kube-networking.nix
# ./networking/wireguard-kube.nix
# ./modules/de.nix

18
flake.lock generated
View file

@ -7,11 +7,11 @@
]
},
"locked": {
"lastModified": 1777655179,
"narHash": "sha256-Rx7RvgxgFeoaJUddpuVbJ2jaaAp7qH6wV9PwBmLvfz4=",
"lastModified": 1777894865,
"narHash": "sha256-agINDb/tr4v2uaVmgE/i0dY1M2JJdzUI/Caup/MWEGU=",
"owner": "nix-community",
"repo": "home-manager",
"rev": "feda41500ec53fcd4e3131de7b0441bce08fd3e9",
"rev": "9c6f1307e1d76a2285d8001e1b8bc281bfe15dac",
"type": "github"
},
"original": {
@ -48,11 +48,11 @@
]
},
"locked": {
"lastModified": 1777181277,
"narHash": "sha256-yVJbd07ortDRAttDFmDV5p220aOLTHgVAx//0nW/xW8=",
"lastModified": 1777787189,
"narHash": "sha256-2KUbS/HhzWW3kkkY1+RiWj9mJ76VEXw8lBJzcCFKzfY=",
"owner": "nix-community",
"repo": "nix-index-database",
"rev": "b8eb7acee0f7604fe1bf6a5b3dcf5254369180fa",
"rev": "2dea2b920e7127b3afa8506713f23536651de312",
"type": "github"
},
"original": {
@ -63,11 +63,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1777268161,
"narHash": "sha256-bxrdOn8SCOv8tN4JbTF/TXq7kjo9ag4M+C8yzzIRYbE=",
"lastModified": 1777578337,
"narHash": "sha256-Ad49moKWeXtKBJNy2ebiTQUEgdLyvGmTeykAQ9xM+Z4=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "1c3fe55ad329cbcb28471bb30f05c9827f724c76",
"rev": "15f4ee454b1dce334612fa6843b3e05cf546efab",
"type": "github"
},
"original": {

69
kubenetes/containerd.nix Normal file
View file

@ -0,0 +1,69 @@
{ config, lib, pkgs, ... }: let
nvidiaEnabled = builtins.elem "nvidia" config.services.xserver.videoDrivers;
in {
config = lib.mkMerge [
(lib.mkIf nvidiaEnabled {
virtualisation.docker.enableNvidia = true;
virtualisation.docker.enable = true;
hardware.nvidia-container-toolkit = {
enable = true;
mount-nvidia-executables = true;
mount-nvidia-docker-1-directories = true;
extraArgs = [ "--device-name-strategy=uuid" ];
};
environment.systemPackages = with pkgs; [ nvidia-docker (lib.getOutput "tools" config.hardware.nvidia-container-toolkit.package) runc ];
services.envfs.enable = true;
virtualisation.docker.daemon.settings.features.cdi = true;
virtualisation.containerd = {
enable = true;
settings = {
plugins = {
"io.containerd.grpc.v1.cri" = {
# enable_cdi = true;
# cdi_spec_dirs = [ "/etc/cdi" "/var/run/cdi" ];
containerd = {
# default_runtime_name = "runc";
runtimes.runc.options = { SystemdCgroup = false; };
default_runtime_name = "nvidia";
runtimes = {
nvidia = {
privileged_without_host_devices = false;
runtime_type = "io.containerd.runc.v2";
options = {
BinaryName = "${lib.getOutput "tools" config.hardware.nvidia-container-toolkit.package}/bin/nvidia-container-runtime";
};
};
};
};
};
};
};
};
})
(lib.mkIf (!nvidiaEnabled ) {
virtualisation.containerd = {
enable = true;
settings = {
version = 2;
plugins."io.containerd.grpc.v1.cri" = {
# This is the critical part for Kubeadm
containerd.runtimes.runc = {
runtime_type = "io.containerd.runc.v2";
options.SystemdCgroup = true;
};
# # Keep your existing settings
# containerd.snapshotter = lib.mkIf config.boot.zfs.enabled (lib.mkOptionDefault "zfs");
# cni.bin_dir = lib.mkOptionDefault "${pkgs.cni-plugins}/bin";
};
};
};
})
];
# # Tell the Kubelet to use containerd
# services.kubernetes.kubelet.containerRuntimeEndpoint = "unix:///run/containerd/containerd.sock";
}

6
vms/kube-vm/kubernetes/kubelet.nix → kubenetes/kubelet.nix
View file

@ -11,7 +11,11 @@
serviceConfig = {
# Use the kubelet binary from the Nix store
ExecStart = "${pkgs.kubernetes}/bin/kubelet";
ExecStart = ''${pkgs.kubernetes}/bin/kubelet \
--config=/var/lib/kubelet/config.yaml \
--kubeconfig=/etc/kubernetes/kubelet.conf \
--pod-manifest-path=/etc/kubernetes/manifests
'';
Restart = "always";
RestartSec = 10;

4
kubenetes/kubernetes.nix Normal file
View file

@ -0,0 +1,4 @@
{ pkgs, ... }: {
imports = [ ./kubelet.nix ./containerd.nix ];
environment.systemPackages = with pkgs; [ kubernetes cri-tools ];
}

35
modules/drivers/nvidia.nix
View file

@ -1,13 +1,12 @@
{
config,
pkgs,
lib,
...
{ config
, pkgs
, lib
, ...
}: {
nixpkgs.config.nvidia.acceptLicense = true;
services.xserver.videoDrivers = ["nvidia"];
services.xserver.videoDrivers = [ "nvidia" ];
boot.kernelParams = ["nvidia.NVreg_PreserveVideoMemoryAllocations=1"];
boot.kernelParams = [ "nvidia.NVreg_PreserveVideoMemoryAllocations=1" ];
hardware.graphics = {
enable = true;
@ -18,7 +17,7 @@
};
hardware.nvidia = {
package = config.boot.kernelPackages.nvidiaPackages.stable;
package = config.boot.kernelPackages.nvidiaPackages.legacy_580;
modesetting.enable = true;
open = false;
nvidiaSettings = true;
@ -43,23 +42,23 @@
specialisation = {
Battery.configuration = {
system.nixos.tags = ["Battery"];
system.nixos.tags = [ "Battery" ];
boot.extraModprobeConfig = ''
blacklist nouveau
options nouveau modeset=0
'';
services.udev.extraRules = ''
# Remove NVIDIA USB xHCI Host Controller devices, if present
ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x0c0330", ATTR{power/control}="auto", ATTR{remove}="1"
# Remove NVIDIA USB Type-C UCSI devices, if present
ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x0c8000", ATTR{power/control}="auto", ATTR{remove}="1"
# Remove NVIDIA Audio devices, if present
ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x040300", ATTR{power/control}="auto", ATTR{remove}="1"
# Remove NVIDIA VGA/3D controller devices
ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x03[0-9]*", ATTR{power/control}="auto", ATTR{remove}="1"
# Remove NVIDIA USB xHCI Host Controller devices, if present
ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x0c0330", ATTR{power/control}="auto", ATTR{remove}="1"
# Remove NVIDIA USB Type-C UCSI devices, if present
ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x0c8000", ATTR{power/control}="auto", ATTR{remove}="1"
# Remove NVIDIA Audio devices, if present
ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x040300", ATTR{power/control}="auto", ATTR{remove}="1"
# Remove NVIDIA VGA/3D controller devices
ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x03[0-9]*", ATTR{power/control}="auto", ATTR{remove}="1"
'';
boot.blacklistedKernelModules = ["nouveau" "nvidia" "nvidia_drm" "nvidia_modeset"];
boot.blacklistedKernelModules = [ "nouveau" "nvidia" "nvidia_drm" "nvidia_modeset" ];
};
};
}

120
vms/kube-daddy/default.nix
View file

@ -6,124 +6,4 @@
config = ./kube-daddy.nix;
};
systemd.services.kube-iptable = {
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${pkgs.iptables}/bin/iptables -t nat -I POSTROUTING 1 -s 10.0.0.0/24 -o enp8s0 -j MASQUERADE ";
RemainAfterExit = true;
User = "root";
};
stopIfChanged = true;
};
networking = {
bridges = {
"br0" = {
interfaces = [
"microvm-tap1"
"microvm-tap2"
];
};
};
interfaces.br0.ipv4.addresses = [
{
address = "10.0.0.1";
prefixLength = 24;
}
];
localCommands = ''
ip tuntap add dev microvm-tap1 mode tap user root || true
ip tuntap add dev microvm-tap2 mode tap user root || true
ip link set microvm-tap1 up
ip link set microvm-tap2 up
'';
nat = {
enable = true;
externalInterface = "enp8s0";
internalIPs = [ "10.0.0.0/24" ];
forwardPorts = [
{
sourcePort = 8877;
destination = "10.0.0.2:8888";
proto = "tcp";
}
# { # Access this directly from host by 10.0.0.2:4325
# sourcePort = 4325; # argocd
# destination = "10.0.0.2:8080";
# proto = "tcp";
# }
{
# Netbird
sourcePort = 3478;
destination = "10.0.0.2:3478";
proto = "udp";
}
{
sourcePort = 6443;
destination = "10.0.0.2:6443";
proto = "tcp";
}
{
sourcePort = 4123;
destination = "10.0.0.2:4123";
proto = "tcp";
}
{
sourcePort = 8472;
destination = "10.0.0.2:8472";
proto = "udp";
}
{
sourcePort = 2379;
destination = "10.0.0.2:2379";
proto = "udp";
}
{
sourcePort = 2380;
destination = "10.0.0.2:2380";
proto = "udp";
}
{
sourcePort = 2379;
proto = "tcp";
destination = "10.0.0.2:2379";
}
{
sourcePort = 2380;
destination = "10.0.0.2:2380";
proto = "tcp";
}
{
sourcePort = 4001;
destination = "10.0.0.2:4001";
proto = "udp";
}
{
sourcePort = 4001;
destination = "10.0.0.2:4001";
proto = "tcp";
}
# If your app uses UDP (like HTTP/3 or QUIC), add this too:
# { sourcePort = 8888; destination = "10.0.0.2:8888"; proto = "udp"; }
];
};
# 5. Update Firewall to trust the Bridge
firewall.trustedInterfaces = [ "br0" ];
};
systemd.tmpfiles.rules = [
"d /var/lib/microvms/shared 0755 microvm kvm -"
"d /var/lib/microvms/shared/kube 0755 microvm kvm -"
"d /var/lib/microvms/shared/docking 0755 microvm kvm -"
"d /var/lib/microvms/shared/.config 0755 microvm kvm -"
"d /var/lib/microvms/shared/.local 0755 microvm kvm -"
];
}

123
vms/kube-networking.nix Normal file
View file

@ -0,0 +1,123 @@
{ pkgs, ... }:
{
systemd.services.kube-iptable = {
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${pkgs.iptables}/bin/iptables -t nat -I POSTROUTING 1 -s 10.0.0.0/24 -o enp8s0 -j MASQUERADE ";
RemainAfterExit = true;
User = "root";
};
stopIfChanged = true;
};
networking = {
bridges = {
"br0" = {
interfaces = [
"microvm-tap1"
"microvm-tap2"
];
};
};
interfaces.br0.ipv4.addresses = [
{
address = "10.0.0.1";
prefixLength = 24;
}
];
localCommands = ''
ip tuntap add dev microvm-tap1 mode tap user root || true
ip tuntap add dev microvm-tap2 mode tap user root || true
ip link set microvm-tap1 up
ip link set microvm-tap2 up
'';
nat = {
enable = true;
externalInterface = "enp8s0";
internalIPs = [ "10.0.0.0/24" ];
forwardPorts = [
{
sourcePort = 8877;
destination = "10.0.0.2:8888";
proto = "tcp";
}
# { # Access this directly from host by 10.0.0.2:4325
# sourcePort = 4325; # argocd
# destination = "10.0.0.2:8080";
# proto = "tcp";
# }
{
# Netbird
sourcePort = 3478;
destination = "10.0.0.2:3478";
proto = "udp";
}
{
sourcePort = 6443;
destination = "10.0.0.2:6443";
proto = "tcp";
}
{
sourcePort = 4123;
destination = "10.0.0.2:4123";
proto = "tcp";
}
{
sourcePort = 8472;
destination = "10.0.0.2:8472";
proto = "udp";
}
{
sourcePort = 2379;
destination = "10.0.0.2:2379";
proto = "udp";
}
{
sourcePort = 2380;
destination = "10.0.0.2:2380";
proto = "udp";
}
{
sourcePort = 2379;
proto = "tcp";
destination = "10.0.0.2:2379";
}
{
sourcePort = 2380;
destination = "10.0.0.2:2380";
proto = "tcp";
}
{
sourcePort = 4001;
destination = "10.0.0.2:4001";
proto = "udp";
}
{
sourcePort = 4001;
destination = "10.0.0.2:4001";
proto = "tcp";
}
# If your app uses UDP (like HTTP/3 or QUIC), add this too:
# { sourcePort = 8888; destination = "10.0.0.2:8888"; proto = "udp"; }
];
};
# 5. Update Firewall to trust the Bridge
firewall.trustedInterfaces = [ "br0" ];
};
systemd.tmpfiles.rules = [
"d /var/lib/microvms/shared 0755 microvm kvm -"
"d /var/lib/microvms/shared/kube 0755 microvm kvm -"
"d /var/lib/microvms/shared/docking 0755 microvm kvm -"
"d /var/lib/microvms/shared/.config 0755 microvm kvm -"
"d /var/lib/microvms/shared/.local 0755 microvm kvm -"
];
}

23
vms/kube-vm/kube-vm.nix
View file

@ -45,6 +45,8 @@
microvm = {
# Choose your hypervisor: "qemu", "firecracker", "cloud-hypervisor", etc.
hypervisor = "qemu";
vcpu = 8;
mem = 8192 / 3;
# Create a tap interface or user networking
interfaces = [{
@ -65,11 +67,28 @@
volumes = [{
image = "/var/lib/microvms/kube-vm/kube-vm.img";
mountPoint = "/";
size = 512 * 4; # Size in MB
size = 512 * 8; # Size in MB
}];
};
boot.kernelModules = [ "br_netfilter" ];
boot.kernel.sysctl = {
"net.ipv4.ip_forward" = 1;
"net.ipv6.conf.all.forwarding" = 1;
};
systemd.services."load-kernel-modules" = {
enable = true;
description = "Modprobe kernel modules";
# before = [ "flannel.service" ];
wantedBy = [
"multi-user.target"
# "flannel.service"
];
script = ''
${pkgs.kmod}/bin/modprobe br_netfilter
'';
};
networking = {
hostName = "kube-vm";

4
vms/kube-vm/kubernetes/kubernetes.nix
View file

@ -1,4 +0,0 @@
{ pkgs, ... }: {
imports = [ ./kublet.nix ];
environment.systemPackages = with pkgs; [ kubernetes ];
}