Working multi-GPU/single-GPU setup

2025-12-10 00:43:56 -05:00 · 2023-03-16 21:44:50 -04:00 · 2023-03-16 21:44:50 -04:00 · 1d06b35262
commit 1d06b35262
parent 84b15db0af
12 changed files with 317 additions and 87 deletions
--- a/flake.lock
+++ b/flake.lock
@ -7,11 +7,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1678671394,
-        "narHash": "sha256-fe+OMC3+0BsSnvSWm+oYbMgQup32TABfOsvJH23J368=",
+        "lastModified": 1678951780,
+        "narHash": "sha256-Bve469g/b+AQ8cBIjlmaDVW/XZdZK9HUh8/UwV9Wbdc=",
        "owner": "nix-community",
        "repo": "disko",
-        "rev": "d19c4fa316c4ea7127a89304c01839cbbcaf1f64",
+        "rev": "a2dec6e4e0fe54a9e1610299ac3aaeeb48eab50c",
        "type": "github"
      },
      "original": {
@ -74,11 +74,11 @@
    },
    "flake-utils_2": {
      "locked": {
-        "lastModified": 1676283394,
-        "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=",
+        "lastModified": 1678901627,
+        "narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=",
        "owner": "numtide",
        "repo": "flake-utils",
-        "rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073",
+        "rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6",
        "type": "github"
      },
      "original": {
@ -119,11 +119,11 @@
    },
    "master": {
      "locked": {
-        "lastModified": 1678673372,
-        "narHash": "sha256-OJuIe9Mf7RC/MBibdEQBxg5KVR/5eo/kYPN9M+IGESs=",
+        "lastModified": 1679022282,
+        "narHash": "sha256-jdMD81dOM3+ynAgkmp3MoOKwNz57iCFUoamSq2r3Ozk=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "6bfb3ffb7067f298dc612a190f7e07cf97b3f312",
+        "rev": "9e27d7aa10bb84f1ace7193538764e85a8caeea2",
        "type": "github"
      },
      "original": {
@ -325,11 +325,11 @@
    },
    "stable": {
      "locked": {
-        "lastModified": 1678500271,
-        "narHash": "sha256-tRBLElf6f02HJGG0ZR7znMNFv/Uf7b2fFInpTHiHaSE=",
+        "lastModified": 1678872516,
+        "narHash": "sha256-/E1YwtMtFAu2KUQKV/1+KFuReYPANM2Rzehk84VxVoc=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "5eb98948b66de29f899c7fe27ae112a47964baf8",
+        "rev": "9b8e5abb18324c7fe9f07cb100c3cd4a29cda8b8",
        "type": "github"
      },
      "original": {
@ -361,11 +361,11 @@
    },
    "unstable": {
      "locked": {
-        "lastModified": 1678470307,
-        "narHash": "sha256-OEeMUr3ueLIXyW/OaFUX5jUdimyQwMg/7e+/Q0gC/QE=",
+        "lastModified": 1678898370,
+        "narHash": "sha256-xTICr1j+uat5hk9FyuPOFGxpWHdJRibwZC+ATi0RbtE=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "0c4800d579af4ed98ecc47d464a5e7b0870c4b1f",
+        "rev": "ac718d02867a84b42522a0ece52d841188208f2c",
        "type": "github"
      },
      "original": {
--- a/flake.nix
+++ b/flake.nix
@ -6,7 +6,6 @@
    stable.url = "github:NixOS/nixpkgs/nixos-22.11";
    #utilites --maybe flake-parts soon?
    flake-utils.url = "github:numtide/flake-utils";
-
    nixos-generators = {
      url = "github:nix-community/nixos-generators";
      inputs.nixpkgs.follows = "unstable";
--- a/misc/1-monitor.conf
+++ b/misc/1-monitor.conf
@ -0,0 +1,54 @@
+Section "InputClass"
+  Identifier "libinput mouse configuration"
+  MatchDriver "libinput"
+  MatchIsPointer "on"
+  Option "AccelProfile" "adaptive"
+  Option "LeftHanded" "off"
+  Option "MiddleEmulation" "on"
+  Option "NaturalScrolling" "off"
+  Option "ScrollMethod" "twofinger"
+  Option "HorizontalScrolling" "on"
+  Option "SendEventsMode" "enabled"
+  Option "Tapping" "on"
+  Option "TappingDragLock" "on"
+  Option "DisableWhileTyping" "off"
+  
+EndSection
+
+Section "ServerFlags"
+        Option "StandbyTime" "0"
+      Option "SuspendTime" "0"
+      Option "OffTime" "0"
+      Option "BlankTime" "0"
+EndSection
+
+Section "ServerLayout"
+	Identifier     "X.org Configured"
+	Screen      0  "Screen0" 0 0
+EndSection
+
+Section "Module"
+	Load  "glx"
+EndSection
+Section "Monitor"
+	Identifier   "Monitor0"
+	VendorName   "Monitor Vendor"
+	ModelName    "Monitor Model"
+  Option      "Primary" "true"
+  Modeline "1920x1080_144" 332.75 1920 1952 2016 2080 1080 1084 1089 1111 +HSync +VSync
+  Option "PreferredMode" "1920x1080_144"
+  Option "DPMS" "false"
+EndSection
+Section "Device"
+
+	Identifier  "Card0"
+	Driver      "amdgpu"
+	BusID       "PCI:15:0:0"
+EndSection
+Section "Screen"
+	Identifier "Screen0"
+	Device     "Card0"
+	Monitor    "Monitor0"
+  Option "TearFree" "true"
+EndSection
+
--- a/misc/2-monitor.conf
+++ b/misc/2-monitor.conf
@ -0,0 +1,61 @@
+Section "ServerFlags"
+  Option "AllowMouseOpenFail" "on"
+  Option "DontZap" "on"
+  
+EndSection
+
+Section "Monitor"
+  Identifier "Monitor[0]"
+  
+EndSection
+
+Section "InputClass"
+  Identifier "libinput mouse configuration"
+  MatchDriver "libinput"
+  MatchIsPointer "on"
+  Option "AccelProfile" "adaptive"
+  Option "LeftHanded" "off"
+  Option "MiddleEmulation" "on"
+  Option "NaturalScrolling" "off"
+  Option "ScrollMethod" "twofinger"
+  Option "HorizontalScrolling" "on"
+  Option "SendEventsMode" "enabled"
+  Option "Tapping" "on"
+  Option "TappingDragLock" "on"
+  Option "DisableWhileTyping" "off"
+EndSection
+
+
+Section "ServerLayout"
+  Identifier "Layout[all]"
+  Inactive "Device-amdgpu[0]"
+  Screen "Screen-nvidia[0]"
+EndSection
+
+
+Section "Device"
+  Identifier "Device-amdgpu[0]"
+  Driver "amdgpu"
+  BusID "PCI:15:0:0"
+EndSection
+
+
+Section "Device"
+  Identifier "Device-nvidia[0]"
+  Driver "nvidia"
+  BusID "PCI:1:0:0"
+EndSection
+
+Section "Screen"
+  Identifier "Screen-nvidia[0]"
+  Device "Device-nvidia[0]"
+  Option "RandRRotation" "on"
+  Option "AllowEmptyInitialConfiguration"
+EndSection
+
+
+
+
+
+
+
--- a/misc/alacritty.yml
+++ b/misc/alacritty.yml
@ -0,0 +1,71 @@
+window:
+  dimensions:
+    columns: 100
+    lines: 85
+  padding:
+    x: 0
+    y: 0
+    dynamic_padding: false
+  decorations: None
+  startup_mode: Windowed
+  opacity: 1.0
+scrolling:
+  history: 5000
+  multiplier: 5
+font:
+  size: 10.0
+  offset:
+    x: 0
+    y: 0
+  glyph_offset:
+    x: 0
+    y: 0
+custom_cursor_colors: true
+colors:
+  primary:
+    background:        '#080808'
+    foreground:        '#b2b2b2'
+    bright_foreground: '#eeeeee'
+
+  cursor:
+    text:   '#080808'
+    cursor: '#9e9e9e'
+
+  selection:
+    text:       '#080808'
+    background: '#b2ceee'
+
+  normal:
+    black:   '#323437'
+    red:     '#ff5454'
+    green:   '#8cc85f'
+    yellow:  '#e3c78a'
+    blue:    '#80a0ff'
+    magenta: '#cf87e8'
+    cyan:    '#79dac8'
+    white:   '#c6c6c6'
+
+  bright:
+    black:   '#949494'
+    red:     '#ff5189'
+    green:   '#36c692'
+    yellow:  '#c2c292'
+    blue:    '#74b2ff'
+    magenta: '#ae81ff'
+    cyan:    '#85dc85'
+    white:   '#e4e4e4'
+bell:
+  animation: EaseOutExpo
+  color: '#ffffff'
+  duration: 0
+selection:
+  semantic_escape_chars: ",│`|:\"' ()[]{}<>"
+  save_to_clipboard: false
+
+mouse:
+  double_click: { threshold: 300 }
+  triple_click: { threshold: 300 }
+  hide_when_typing: true
+cursor:
+  style: Beam
+
--- a/modules/DE/dwm.nix
+++ b/modules/DE/dwm.nix
@ -20,7 +20,7 @@ in {
      enable = true;
      displayManager = {
        sessionCommands = ''
-          ${pkgs.feh}/bin/feh --bg-scale ${self + /misc/recursion.png}
+          ${pkgs.feh}/bin/feh --bg-center ${self + /misc/recursion.png}
          ${pkgs.numlockx}/bin/numlockx
        '';
        defaultSession = "none+dwm";
--- a/modules/X11.nix
+++ b/modules/X11.nix
@ -15,6 +15,7 @@ in {
  config = mkMerge [
    {
      services.xserver = {
+        tty = lib.mkDefault 1;
        exportConfiguration = true;
        layout = "us";
        libinput.enable = true;
--- a/modules/packages.nix
+++ b/modules/packages.nix
@ -6,10 +6,10 @@ _: {pkgs, ...}: {
      pkgs.pciutils #lspci
      pkgs.alsa-utils #volume control
      pkgs.xclip #commandline clipboard access
-      pkgs.btrfs-progs #for external harddrive
      pkgs.bottom #view tasks
      pkgs.bc #terminal calculator
      pkgs.nix-tree #view packages
+      pkgs.nix-output-monitor
    ];
  };
 }
--- a/modules/theming.nix
+++ b/modules/theming.nix
@ -59,31 +59,6 @@ in {
        services.xserver.displayManager.sessionCommands = ''
          xrdb -load /etc/xdg/Xresources
        '';
-      }
-    )
-    (mkIf cfg.kmscon.enable {
-      services.kmscon = {
-        enable = true;
-        hwRender = true;
-        extraConfig = ''
-          font-size=10
-        '';
-        fonts = [
-          {
-            name = "OverpassMono Nerd Font";
-            package =
-              pkgs.nerdfonts.override
-              {
-                fonts = ["Overpass"];
-              };
-          }
-        ];
-      };
-      systemd.services = {
-        "autovt@tty1".enable = false;
-        "kmsconvt@tty1".enable = false;
-      };
-
        fonts = {
          fonts = [
            (pkgs.nerdfonts.override
@ -104,6 +79,30 @@ in {
            antialias = true;
          };
        };
+      }
+    )
+    (mkIf cfg.kmscon.enable {
+      services.kmscon = {
+        enable = true;
+        hwRender = false;
+        extraConfig = ''
+          font-size=10
+        '';
+        fonts = [
+          {
+            name = "OverpassMono Nerd Font";
+            package =
+              pkgs.nerdfonts.override
+              {
+                fonts = ["Overpass"];
+              };
+          }
+        ];
+      };
+      systemd.services = {
+        "autovt@tty1".enable = false;
+        "kmsconvt@tty1".enable = false;
+      };
    })
  ];
 }
--- a/systems/gerg-desktop/default.nix
+++ b/systems/gerg-desktop/default.nix
@ -6,7 +6,6 @@ inputs: {
  ...
 }: {
  imports = [
-    (import ./refreshrate.nix inputs)
    (import ./vfio.nix inputs)
    (import ./parrot.nix inputs)
    (import ./spicetify.nix inputs)
@ -33,7 +32,17 @@ inputs: {
      kmscon.enable = true;
    };
  };
-  services.xserver.videoDrivers = ["amdgpu"];
+  hardware.nvidia = {
+    package = config.boot.kernelPackages.nvidiaPackages.beta;
+    nvidiaPersistenced = false;
+    nvidiaSettings = false;
+    modesetting.enable = true;
+    open = true;
+  };
+  services.xserver = {
+    videoDrivers = ["nvidia" "amdgpu"];
+  };
+  nixpkgs.config.allowUnfree = true;

  environment.systemPackages = [
    pkgs.webcord # talk to people (gross)
@ -47,12 +56,10 @@ inputs: {
    pkgs.neovide #gui neovim
    pkgs.ripgrep
    inputs.suckless.packages.${pkgs.system}.st
+    pkgs.alacritty
  ];
-  #set webcord theme
-  systemd.tmpfiles.rules = let
-    theme = pkgs.writeText "black" (builtins.readFile "${self}/misc/black.theme.css");
-  in ["L+ /home/gerg/.config/WebCord/Themes/black - - - - ${theme}"];

+  environment.etc."xdg/alacritty/alacritty.yml".source = "${self}/misc/alacritty.yml";
  networking = {
    useDHCP = false;
    hostName = "gerg-desktop";
@ -117,8 +124,8 @@ inputs: {
    };
  };
  boot = {
-    initrd = {
    kernelModules = ["amdgpu"];
+    initrd = {
      availableKernelModules = ["nvme" "xhci_pci" "ahci" "usbhid" "sd_mod"];
      includeDefaultModules = false;
    };
--- a/systems/gerg-desktop/refreshrate.nix
+++ b/systems/gerg-desktop/refreshrate.nix
@ -1,24 +0,0 @@
-_: {
-  services.xserver = {
-    #fix nasty screen tearing
-    screenSection = ''
-      Option "TearFree" "true"
-    '';
-    #set monitor as primary
-    #set refreshrate to 144 instead of default(60)
-    #disable DPMS(screen turning off)
-    monitorSection = ''
-      Option      "Primary" "true"
-      Modeline "1920x1080_144" 332.75 1920 1952 2016 2080 1080 1084 1089 1111 +HSync +VSync
-      Option "PreferredMode" "1920x1080_144"
-      Option "DPMS" "false"
-    '';
-    #disable screen blanking in total
-    serverFlagsSection = ''
-      Option "StandbyTime" "0"
-      Option "SuspendTime" "0"
-      Option "OffTime" "0"
-      Option "BlankTime" "0"
-    '';
-  };
-}
--- a/systems/gerg-desktop/vfio.nix
+++ b/systems/gerg-desktop/vfio.nix
@ -2,15 +2,51 @@ _: {
  pkgs,
  settings,
  self,
+  config,
+  lib,
  ...
-}: {
+}: let
+  xcfg = config.services.xserver;
+  xserverbase = let
+    fontsForXServer =
+      config.fonts.fonts
+      ++ [
+        pkgs.xorg.fontadobe100dpi
+        pkgs.xorg.fontadobe75dpi
+      ];
+  in
+    pkgs.runCommand "xserverbase"
+    {
+      fontpath =
+        lib.optionalString (xcfg.fontPath != null)
+        ''FontPath "${xcfg.fontPath}"'';
+      inherit (xcfg) config;
+      preferLocalBuild = true;
+    }
+    ''
+      echo 'Section "Files"' >> $out
+      echo $fontpath >> $out
+      for i in ${toString fontsForXServer}; do
+        if test "''${i:0:''${#NIX_STORE}}" == "$NIX_STORE"; then
+          for j in $(find $i -name fonts.dir); do
+            echo "  FontPath \"$(dirname $j)\"" >> $out
+          done
+        fi
+      done
+      for i in $(find ${toString xcfg.modules} -type d); do
+        if test $(echo $i/*.so* | wc -w) -ne 0; then
+          echo "  ModulePath \"$i\"" >> $out
+        fi
+      done
+      echo '${xcfg.filesSection}' >> $out
+      echo 'EndSection' >> $out
+      echo >> $out
+    '';
+  oneMonitor = pkgs.writeText "1-monitor.conf" (lib.strings.concatStrings [(builtins.readFile xserverbase) (builtins.readFile "${self}/misc/1-monitor.conf")]);
+  twoMonitor = pkgs.writeText "2-monitor.conf" (lib.strings.concatStrings [(builtins.readFile xserverbase) (builtins.readFile "${self}/misc/2-monitor.conf")]);
+in {
  boot = {
    kernelParams = ["amd_iommu=on" "iommu=pt" "vfio_iommu_type1.allow_unsafe_interrupts=1" "kvm.ignore_msrs=1"];
-    kernelModules = ["kvm-amd" "vfio_virqfd" "vfio_pci" "vfio_iommu_type1" "vfio"];
-    initrd.kernelModules = ["vfio_virqfd" "vfio_pci" "vfio_iommu_type1" "vfio"];
-    extraModprobeConfig = ''
-      options vfio-pci ids=10de:228e,10de:2504
-    '';
  };
  virtualisation = {
    libvirtd = {
@ -27,7 +63,9 @@ _: {
    };
  };
  environment = {
-    systemPackages = [pkgs.virt-manager];
+    systemPackages = [
+      pkgs.virt-manager
+    ];
    shellAliases = {
      vm-start = "virsh start Windows";
      vm-stop = "virsh shutdown Windows";
@ -36,30 +74,54 @@ _: {

  users.users."${settings.username}".extraGroups = ["kvm" "libvirtd"];

+  services.xserver.displayManager.xserverArgs = lib.mkAfter ["-config /tmp/xorg.conf"];
+  services.xserver.displayManager.sessionCommands = lib.mkBefore ''
+    if ! (test -e "/tmp/ONE_MONITOR"); then
+        xrandr --setprovideroutputsource "AMD Radeon Graphics @ pci:0000:0f:00.0" NVIDIA-0
+        xrandr --auto
+        xrandr --output DP-0 --mode 3440x1440 --rate 120 --primary --left-of HDMI-A-1-0
+        xrandr --output HDMI-A-1-0 --mode 1920x1080 --rate 120 --set TearFree on
+        xset s off -dpms
+    fi
+  '';
+
  systemd.tmpfiles.rules = let
    xml = pkgs.writeText "Windows.xml" (builtins.readFile "${self}/misc/Windows.xml");
-    qemuHook = pkgs.writeScript "qemu-hook" ''
-      #!${pkgs.stdenv.shell}
+    qemuHook = pkgs.writeShellScript "qemu-hook" ''
      GUEST_NAME="$1"
      OPERATION="$2"
      SUB_OPERATION="$3"

      if [ "$GUEST_NAME" == "Windows" ]; then
-        if [ "$OPERATION" == "start" ]; then
+        if [ "$OPERATION" == "prepare" ]; then
+            systemctl stop display-manager.service
+            modprobe -r -a nvidia_uvm nvidia_drm nvidia nvidia_modeset
+            ${pkgs.libvirt}/bin/virsh nodedev-detach pci_0000_01_00_0
+            ${pkgs.libvirt}/bin/virsh nodedev-detach pci_0000_01_00_1
            systemctl set-property --runtime -- user.slice AllowedCPUs=8-15,24-31
            systemctl set-property --runtime -- system.slice AllowedCPUs=8-15,24-31
            systemctl set-property --runtime -- init.scope AllowedCPUs=8-15,24-31
+            ln -fs ${oneMonitor} /tmp/xorg.conf
+            touch /tmp/ONE_MONITOR
+            systemctl start display-manager.service
        fi
-
-        if [ "$OPERATION" == "stopped" ]; then
+        if [ "$OPERATION" == "release" ]; then
+          systemctl stop display-manager.service
          systemctl set-property --runtime -- user.slice AllowedCPUs=0-31
          systemctl set-property --runtime -- system.slice AllowedCPUs=0-31
          systemctl set-property --runtime -- init.scope AllowedCPUs=0-31
+          ${pkgs.libvirt}/bin/virsh nodedev-reattach pci_0000_01_00_0
+          ${pkgs.libvirt}/bin/virsh nodedev-reattach pci_0000_01_00_1
+          modprobe -a nvidia_uvm nvidia_drm nvidia nvidia_modeset
+          ln -fs ${twoMonitor} /tmp/xorg.conf
+          rm /tmp/ONE_MONITOR
+          systemctl start display-manager.service
        fi
      fi
    '';
  in [
-    "L+ /var/lib/libvirt/hooks/ - - - - ${qemuHook}"
+    "L  /tmp/xorg.conf - - - - ${twoMonitor}"
+    "L+ /var/lib/libvirt/hooks/qemu - - - - ${qemuHook}"
    "L+ /var/lib/libvirt/qemu/Windows.xml - - - - ${xml}"
  ];
 }