Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add configs for kubevirt-lab-1 #109

Merged
merged 1 commit into from
Mar 28, 2024
Merged

feat: add configs for kubevirt-lab-1 #109

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 22 additions & 8 deletions hosts/k8s/disko-config/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,28 +4,42 @@ Generate LUKS keyfile to encrypt the root partition, it's used by disko.

```bash
# partition the usb stick
parted /dev/sdb -- mklabel gpt
parted /dev/sdb -- mkpart primary 2M 512MB
parted /dev/sdb -- mkpart primary 512MB 1024MB
mkfs.fat -F 32 -n NIXOS_DSC /dev/sdb1
mkfs.fat -F 32 -n NIXOS_K3S /dev/sdb2
DEV=/dev/sdX
parted $DEV -- mklabel gpt
parted $DEV -- mkpart primary 2M 512MB
parted $DEV -- mkpart primary 512MB 1024MB
mkfs.fat -F 32 -n NIXOS_DSC ${DEV}1
mkfs.fat -F 32 -n NIXOS_K3S ${DEV}2

# Generate a keyfile from the true random number generator
KEYFILE=./kubevirt-luks-keyfile
dd bs=8192 count=4 iflag=fullblock if=/dev/random of=$KEYFILE
dd bs=512 count=64 iflag=fullblock if=/dev/random of=$KEYFILE

# generate token for k3s
K3S_TOKEN_FILE=./kubevirt-k3s-token
K3S_TOKEN=$(grep -ao '[A-Za-z0-9]' < /dev/random | head -64 | tr -d '\n' ; echo "")
echo $K3S_TOKEN > $K3S_TOKEN_FILE

# copy the keyfile and token to the usb stick

KEYFILE=./kubevirt-luks-keyfile
DEVICE=/dev/disk/by-label/NIXOS_DSC
dd bs=8192 count=4 iflag=fullblock if=$KEYFILE of=$DEVICE
# seek=128 skip N obs-sized output blocks to avoid overwriting the filesystem header
dd bs=512 count=64 iflag=fullblock seek=128 if=$KEYFILE of=$DEVICE

K3S_TOKEN_FILE=./kubevirt-k3s-token
USB_PATH=/run/media/ryan/NIXOS_K3S
cp $K3S_TOKEN_FILE $USB_PATH
```

### 2. Partition the SSD & install NixOS via disko

```bash
# enter an shell with git/vim/ssh-agent/gnumake available
nix-shell -p git vim gnumake
# clone this repository
git clone https://github.com/ryan4yin/nix-config.git

cd nix-config
sudo nix run --experimental-features "nix-command flakes" 'github:nix-community/disko#disko-install' -- \
--write-efi-boot-entries --flake .#kubevirt-shoryu --disk main /dev/nvme0n1
```
169 changes: 87 additions & 82 deletions hosts/k8s/disko-config/kubevirt-disko-fs.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
{
# required by impermanence
fileSystems."/persistent".neededForBoot = true;

# contains the k3s's token
fileSystems."/run/media/nixos_k3s" = {
device = "/dev/disk/by-label/NIXOS_K3S";
Expand All @@ -7,90 +10,92 @@
};

disko.devices = {
disk = {
sda = {
type = "disk";
device = "/dev/nvme0n1";
content = {
type = "gpt";
partitions = {
# The EFI & Boot partition
ESP = {
size = "630M";
type = "EF00";
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot/efi";
mountOptions = [
"defaults"
];
};
};
# The root partition
luks = {
size = "100%";
content = {
type = "luks";
name = "crypted";
settings = {
keyFile = "/dev/disk/by-label/NIXOS_DSC"; # The keyfile is stored on a USB stick
keyFileSize = 8192 * 4; # The maxium size of the keyfile is 8192 bytes
keyFileOffset = 0;
fallbackToPassword = true;
allowDiscards = true;
};
# Whether to add a boot.initrd.luks.devices entry for the specified disk.
initrdUnlock = true;
nodev."/" = {
fsType = "tmpfs";
mountOptions = [
"size=4G"
"defaults"
# set mode to 755, otherwise systemd will set it to 777, which cause problems.
# relatime: Update inode access times relative to modify or change time.
"mode=755"
];
};

# encrypt the root partition with luks2 and argon2id, will prompt for a passphrase, which will be used to unlock the partition.
# cryptsetup luksFormat
extraFormatArgs = [
"--type luks2"
"--cipher aes-xts-plain64"
"--hash sha512"
"--iter-time 5000"
"--key-size 256"
"--pbkdf argon2id"
# use true random data from /dev/random, will block until enough entropy is available
"--use-random"
];
extraOpenArgs = [
"--timeout 10"
];
content = {
type = "btrfs";
extraArgs = ["-f"];
subvolumes = {
"@root" = {
mountpoint = "/";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@home" = {
mountpoint = "/home";
mountOptions = ["compress-force=zstd:1"];
};
"@lib" = {
mountpoint = "/var/lib";
mountOptions = ["compress-force=zstd:1"];
};
disk.main = {
type = "disk";
# When using disko-install, we will overwrite this value from the commandline
device = "/dev/nvme0n1"; # The device to partition
content = {
type = "gpt";
partitions = {
# The EFI & Boot partition
ESP = {
size = "630M";
type = "EF00";
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
mountOptions = [
"defaults"
];
};
};
# The root partition
luks = {
size = "100%";
content = {
type = "luks";
name = "encrypted";
settings = {
keyFile = "/dev/disk/by-label/NIXOS_DSC"; # The keyfile is stored on a USB stick
# The maximum size of the keyfile is 8192 bytes
keyFileSize = 512 * 64; # match the `bs * count` of the `dd` command
keyFileOffset = 512 * 128; # match the `bs * skip` of the `dd` command
fallbackToPassword = true;
allowDiscards = true;
};
# Whether to add a boot.initrd.luks.devices entry for the specified disk.
initrdUnlock = true;

"@nix" = {
mountpoint = "/nix";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@tmp" = {
mountpoint = "/tmp";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@snapshots" = {
mountpoint = "/snapshots";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@swap" = {
mountpoint = "/swap";
swap.swapfile.size = "8192M";
};
# encrypt the root partition with luks2 and argon2id, will prompt for a passphrase, which will be used to unlock the partition.
# cryptsetup luksFormat
extraFormatArgs = [
"--type luks2"
"--cipher aes-xts-plain64"
"--hash sha512"
"--iter-time 5000"
"--key-size 256"
"--pbkdf argon2id"
# use true random data from /dev/random, will block until enough entropy is available
"--use-random"
];
extraOpenArgs = [
"--timeout 10"
];
content = {
type = "btrfs";
extraArgs = ["-f"];
subvolumes = {
"@nix" = {
mountpoint = "/nix";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@persistent" = {
mountpoint = "/persistent";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@tmp" = {
mountpoint = "/tmp";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@snapshots" = {
mountpoint = "/snapshots";
mountOptions = ["compress-force=zstd:1" "noatime"];
};
"@swap" = {
mountpoint = "/swap";
swap.swapfile.size = "16384M";
};
};
};
Expand Down
6 changes: 4 additions & 2 deletions hosts/k8s/kubevirt-shoryu/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@
}: let
# MoreFine - S500Plus
hostName = "kubevirt-shoryu"; # Define your hostname.

coreModule = mylib.genKubeVirtCoreModule {
inherit pkgs hostName;
inherit (myvars) networking;
};
k3sModule = mylib.genK3sServerModule {
inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = config.age.secrets."k3s-prod-1-token".path;
tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
# the first node in the cluster should be the one to initialize the cluster
clusterInit = true;
};
Expand All @@ -26,6 +26,8 @@ in {
++ [
disko.nixosModules.default
../disko-config/kubevirt-disko-fs.nix
./hardware-configuration.nix
./impermanence.nix
coreModule
k3sModule
];
Expand Down
49 changes: 49 additions & 0 deletions hosts/k8s/kubevirt-shoryu/hardware-configuration.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
{
config,
lib,
pkgs,
modulesPath,
...
}: {
imports = [
(modulesPath + "/installer/scan/not-detected.nix")
];

boot.loader = {
# depending on how you configured your disk mounts, change this to /boot or /boot/efi.
efi.efiSysMountPoint = "/boot/";
efi.canTouchEfiVariables = true;
systemd-boot.enable = true;
};
# clear /tmp on boot to get a stateless /tmp directory.
boot.tmp.cleanOnBoot = true;

boot.initrd.availableKernelModules = ["xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod"];
boot.kernelModules = ["kvm-amd" "vfio-pci"];
boot.extraModprobeConfig = "options kvm_amd nested=1"; # for amd cpu

# Enable binfmt emulation of aarch64-linux, this is required for cross compilation.
boot.binfmt.emulatedSystems = ["aarch64-linux" "riscv64-linux"];
# supported file systems, so we can mount any removable disks with these filesystems
boot.supportedFilesystems = [
"ext4"
"btrfs"
"xfs"
"ntfs"
"fat"
"vfat"
"cifs" # mount windows share
];

# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp5s0.useDHCP = lib.mkDefault true;
# networking.interfaces.wlo1.useDHCP = lib.mkDefault true;

nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}
53 changes: 53 additions & 0 deletions hosts/k8s/kubevirt-shoryu/impermanence.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
{
impermanence,
pkgs,
...
}: {
imports = [
impermanence.nixosModules.impermanence
];

environment.systemPackages = [
# `sudo ncdu -x /`
pkgs.ncdu
];

# NOTE: impermanence only mounts the directory/file list below to /persistent
# If the directory/file already exists in the root filesystem, you should
# move those files/directories to /persistent first!
environment.persistence."/persistent" = {
# sets the mount option x-gvfs-hide on all the bind mounts
# to hide them from the file manager
hideMounts = true;
directories = [
"/etc/NetworkManager/system-connections"
"/etc/ssh"
"/etc/nix/inputs"
"/etc/secureboot" # lanzaboote - secure boot
# my secrets
"/etc/agenix/"

"/var/log"
"/var/lib"

# k3s related
"/etc/iscsi"
"/etc/rancher"
];
files = [
"/etc/machine-id"
];

# the following directories will be passed to /persistent/home/$USER
users.ryan = {
directories = [
"codes"
"nix-config"
"tmp"
];
files = [
".config/nushell/history.txt"
];
};
};
}
4 changes: 3 additions & 1 deletion hosts/k8s/kubevirt-shushou/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
k3sModule = mylib.genK3sServerModule {
inherit pkgs;
kubeconfigFile = "/home/${myvars.username}/.kube/config";
tokenFile = config.age.secrets."k3s-prod-1-token".path;
tokenFile = "/run/media/nixos_k3s/kubevirt-k3s-token";
serverIp = myvars.networking.hostsAddr.${k3sServerName}.ipv4;
};
in {
Expand All @@ -25,6 +25,8 @@ in {
++ [
disko.nixosModules.default
../disko-config/kubevirt-disko-fs.nix
../kubevirt-shoryu/hardware-configuration.nix
../kubevirt-shoryu/impermanence.nix
coreModule
k3sModule
];
Expand Down
Loading