initial commit

modules/nixos/virtualisation/default.nix

@@ -0,0 +1,92 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.virtualisation;
+
+  boolToZeroOne = x: if x then "1" else "0";
+
+  aclString = strings.concatMapStringsSep ''
+    ,
+  '' strings.escapeNixString cfg.libvirtd.deviceACL;
+
+  inherit (lib)
+    mkDefault
+    mkOption
+    optionalString
+    optionals
+    strings
+    types
+    ;
+in
+{
+  imports = [
+    ./hugepages.nix
+    ./kvmfr.nix
+    ./quickemu.nix
+    ./vfio.nix
+  ];
+
+  options.virtualisation = {
+    libvirtd = {
+      deviceACL = mkOption {
+        type = types.listOf types.str;
+        default = [ ];
+        example = [
+          "/dev/kvm"
+          "/dev/net/tun"
+          "/dev/vfio/vfio"
+        ];
+        description = "List of device paths that QEMU processes are allowed to access.";
+      };
+
+      clearEmulationCapabilities = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Whether to remove privileged Linux capabilities from QEMU processes after they start.";
+      };
+    };
+  };
+
+  config = {
+    virtualisation = {
+      libvirtd = {
+        enable = mkDefault true;
+        onBoot = mkDefault "ignore";
+        onShutdown = mkDefault "shutdown";
+        qemu.runAsRoot = mkDefault false;
+        qemu.verbatimConfig = ''
+          clear_emulation_capabilities = ${boolToZeroOne cfg.libvirtd.clearEmulationCapabilities}
+        ''
+        + optionalString (cfg.libvirtd.deviceACL != [ ]) ''
+          cgroup_device_acl = [
+            ${aclString}
+          ]
+        '';
+        qemu.swtpm.enable = mkDefault true; # TPM 2.0
+      };
+      spiceUSBRedirection.enable = mkDefault true;
+    };
+
+    users.users."qemu-libvirtd" = {
+      extraGroups = optionals (!cfg.libvirtd.qemu.runAsRoot) [
+        "kvm"
+        "input"
+      ];
+      isSystemUser = true;
+    };
+
+    programs.virt-manager.enable = mkDefault true;
+
+    environment.systemPackages = [
+      (pkgs.writeShellScriptBin "iommu-groups" (builtins.readFile ./iommu-groups.sh))
+      pkgs.dnsmasq
+      pkgs.qemu_full
+      pkgs.virtio-win
+    ];
+  };
+}

modules/nixos/virtualisation/hugepages.nix

@@ -0,0 +1,45 @@
+{
+  config,
+  lib,
+  ...
+}:
+
+let
+  cfg = config.virtualisation.hugepages;
+
+  inherit (lib)
+    mkEnableOption
+    mkOption
+    optionals
+    types
+    ;
+in
+{
+  options.virtualisation.hugepages = {
+    enable = mkEnableOption "huge pages.";
+
+    defaultPageSize = mkOption {
+      type = types.strMatching "[0-9]*[kKmMgG]";
+      default = "1M";
+      description = "Default size of huge pages. You can use suffixes K, M, and G to specify KB, MB, and GB.";
+    };
+
+    pageSize = mkOption {
+      type = types.strMatching "[0-9]*[kKmMgG]";
+      default = "1M";
+      description = "Size of huge pages that are allocated at boot. You can use suffixes K, M, and G to specify KB, MB, and GB.";
+    };
+
+    numPages = mkOption {
+      type = types.ints.positive;
+      default = 1;
+      description = "Number of huge pages to allocate at boot.";
+    };
+  };
+
+  config.boot.kernelParams = optionals cfg.enable [
+    "default_hugepagesz=${cfg.defaultPageSize}"
+    "hugepagesz=${cfg.pageSize}"
+    "hugepages=${toString cfg.numPages}"
+  ];
+}

modules/nixos/virtualisation/iommu-groups.sh

@@ -0,0 +1,6 @@
+shopt -s nullglob
+for d in /sys/kernel/iommu_groups/*/devices/*; do
+    n=${d#*/iommu_groups/*}; n=${n%%/*}
+    printf 'IOMMU Group %s ' "$n"
+    lspci -nns "${d##*/}"
+done;

modules/nixos/virtualisation/kvmfr.nix

@@ -0,0 +1,157 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.virtualisation.kvmfr;
+
+  sizeFromResolution =
+    resolution:
+    let
+      ceilToPowerOf2 =
+        let
+          findNextPower = target: power: if power >= target then power else findNextPower target (power * 2);
+        in
+        n: if n <= 1 then 1 else findNextPower n 1;
+      pixelSize = if resolution.pixelFormat == "rgb24" then 3 else 4;
+      bytes = resolution.width * resolution.height * pixelSize * 2;
+    in
+    ceilToPowerOf2 (bytes / 1024 / 1024 + 10);
+
+  deviceSizes = map (device: device.size) cfg.devices;
+
+  devices = imap (index: _deviceConfig: "/dev/kvmfr${toString index}") cfg.devices;
+
+  udevPackage = pkgs.writeTextDir "/lib/udev/rules.d/99-kvmfr.rules" (
+    concatStringsSep "\n" (
+      imap0 (index: deviceConfig: ''
+        SUBSYSTEM=="kvmfr", KERNEL=="kvmfr${toString index}", OWNER="${deviceConfig.permissions.user}", GROUP="${deviceConfig.permissions.group}", MODE="${deviceConfig.permissions.mode}", TAG+="systemd"
+      '') cfg.devices
+    )
+  );
+
+  apparmorAbstraction = concatStringsSep "\n" (map (device: "${device} rw") devices);
+
+  permissionsType = types.submodule {
+    options = {
+      user = mkOption {
+        type = types.str;
+        default = "root";
+        description = "Owner of the shared memory device.";
+      };
+      group = mkOption {
+        type = types.str;
+        default = "root";
+        description = "Group of the shared memory device.";
+      };
+      mode = mkOption {
+        type = types.str;
+        default = "0600";
+        description = "Mode of the shared memory device.";
+      };
+    };
+  };
+
+  resolutionType = types.submodule {
+    options = {
+      width = mkOption {
+        type = types.number;
+        description = "Maximum horizontal video size that should be supported by this device.";
+      };
+
+      height = mkOption {
+        type = types.number;
+        description = "Maximum vertical video size that should be supported by this device.";
+      };
+
+      pixelFormat = mkOption {
+        type = types.enum [
+          "rgba32"
+          "rgb24"
+        ];
+        description = "Pixel format to use.";
+        default = "rgba32";
+      };
+    };
+  };
+
+  deviceType = (
+    types.submodule (
+      { config, options, ... }:
+      {
+        options = {
+          resolution = mkOption {
+            type = types.nullOr resolutionType;
+            default = null;
+            description = "Automatically calculate the minimum device size for a specific resolution. Overrides `size` if set.";
+          };
+
+          size = mkOption {
+            type = types.number;
+            description = "Size for the kvmfr device in megabytes.";
+          };
+
+          permissions = mkOption {
+            type = permissionsType;
+            default = { };
+            description = "Permissions of the kvmfr device.";
+          };
+        };
+
+        config = {
+          size = mkIf (config.resolution != null) (sizeFromResolution config.resolution);
+        };
+      }
+    )
+  );
+
+  inherit (lib)
+    concatStringsSep
+    imap
+    imap0
+    mkIf
+    mkOption
+    optionals
+    types
+    ;
+in
+{
+  options.virtualisation.kvmfr = {
+    enable = mkOption {
+      type = types.bool;
+      default = false;
+      description = "Whether to enable the kvmfr kernel module.";
+    };
+
+    devices = mkOption {
+      type = types.listOf deviceType;
+      default = [ ];
+      description = "List of devices to create.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    boot.extraModulePackages = with config.boot.kernelPackages; [ kvmfr ];
+    services.udev.packages = optionals (cfg.devices != [ ]) [ udevPackage ];
+
+    environment.systemPackages = with pkgs; [ looking-glass-client ];
+    environment.etc = {
+      "modules-load.d/kvmfr.conf".text = ''
+        kvmfr
+      '';
+
+      "modprobe.d/kvmfr.conf".text = ''
+        options kvmfr static_size_mb=${concatStringsSep "," (map (size: toString size) deviceSizes)}
+      '';
+
+      "apparmor.d/local/abstractions/libvirt-qemu" = mkIf config.security.apparmor.enable {
+        text = apparmorAbstraction;
+      };
+    };
+
+    virtualisation.libvirtd.deviceACL = devices;
+  };
+}

modules/nixos/virtualisation/quickemu.nix

@@ -0,0 +1,28 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.virtualisation.quickemu;
+
+  inherit (lib) mkEnableOption mkIf;
+in
+{
+  options.virtualisation.quickemu = {
+    enable = mkEnableOption "quickemu";
+  };
+
+  config = mkIf cfg.enable {
+    environment.systemPackages = with pkgs; [
+      quickemu
+    ];
+
+    boot.extraModprobeConfig = ''
+      options kvm_amd nested=1
+      options kvm ignore_msrs=1 report_ignored_msrs=0
+    '';
+  };
+}

modules/nixos/virtualisation/vfio.nix

@@ -0,0 +1,103 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.virtualisation.vfio;
+
+  inherit (lib)
+    mkEnableOption
+    mkIf
+    mkOption
+    optional
+    optionals
+    types
+    versionOlder
+    ;
+in
+{
+  options.virtualisation.vfio = {
+    enable = mkEnableOption "VFIO Configuration";
+
+    IOMMUType = mkOption {
+      type = types.enum [
+        "intel"
+        "amd"
+      ];
+      example = "intel";
+      description = "Type of the IOMMU used";
+    };
+
+    devices = mkOption {
+      type = types.listOf (types.strMatching "[0-9a-f]{4}:[0-9a-f]{4}");
+      default = [ ];
+      example = [
+        "10de:1b80"
+        "10de:10f0"
+      ];
+      description = "PCI IDs of devices to bind to vfio-pci";
+    };
+
+    disableEFIfb = mkOption {
+      type = types.bool;
+      default = false;
+      example = true;
+      description = "Disables the usage of the EFI framebuffer on boot.";
+    };
+
+    blacklistNvidia = mkOption {
+      type = types.bool;
+      default = false;
+      description = "Add Nvidia GPU modules to blacklist";
+    };
+
+    ignoreMSRs = mkOption {
+      type = types.bool;
+      default = false;
+      example = true;
+      description = "Enables or disables kvm guest access to model-specific registers";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.udev.extraRules = ''
+      SUBSYSTEM=="vfio", OWNER="root", GROUP="kvm"
+    '';
+
+    boot.kernelParams =
+      (
+        if cfg.IOMMUType == "intel" then
+          [
+            "intel_iommu=on"
+            "intel_iommu=igfx_off"
+          ]
+        else
+          [ "amd_iommu=on" ]
+      )
+      ++ optional (cfg.devices != [ ]) ("vfio-pci.ids=" + builtins.concatStringsSep "," cfg.devices)
+      ++ (optional cfg.disableEFIfb "video=efifb:off")
+      ++ (
+        optionals cfg.ignoreMSRs [
+          "kvm.ignore_msrs=1"
+          "kvm.report_ignored_msrs=0"
+        ]
+        ++ optional cfg.blacklistNvidia "modprobe.blacklist=nouveau,nvidia,nvidia_drm,nvidia"
+      );
+
+    boot.initrd.kernelModules = [
+      "vfio_pci"
+      "vfio_iommu_type1"
+      "vfio"
+    ]
+    ++ optionals (versionOlder pkgs.linux.version "6.2") [ "vfio_virqfd" ];
+
+    # boot.blacklistedKernelModules = optionals cfg.blacklistNvidia [
+    #   "nouveau"
+    #   "nvidia"
+    #   "nvidia_drm"
+    # ];
+  };
+}