nixos/modules/system/hardware/gpu/nvidia.nix

{...}: {
  # Dedicated NVIDIA GPU (single discrete card, no hybrid/PRIME).
  # For Intel+NVIDIA laptops (e.g. the T480), use gpuNvidiaPrime instead.
  flake.nixosModules.gpuNvidia = {
    config,
    lib,
    pkgs,
    ...
  }: {
    services.xserver.videoDrivers = ["nvidia"];

    hardware.graphics = {
      enable = true;
      enable32Bit = true;
    };

    hardware.nvidia = {
      modesetting.enable = true;
      # Open kernel modules. Mandatory to set explicitly on driver >= 560 —
      # the option defaults to null there and nixpkgs asserts against null, so
      # this line cannot simply be dropped.
      #
      # true because every card this module targets is Turing or later (RTX 20xx
      # and newer), which is exactly what upstream recommends open modules for.
      # Set false only for a genuinely pre-Turing card, which would want the
      # gpuNvidia module pointed at a legacy package branch anyway.
      #
      # This is NOT cosmetic and NOT a version switch: `open` selects between two
      # real builds of the same release — nvidia-open (~12MB) vs the proprietary
      # nvidia-kernel-modules (~85MB). It has no effect on driver version, which
      # comes solely from `package` below.
      #
      # Kernel modules only change on reboot, so `cat /proc/driver/nvidia/version`
      # describes the RUNNING module and can disagree with this setting until the
      # next boot. Compare /run/booted-system against /run/current-system to see
      # a pending swap.
      #
      # mkDefault because "Turing or later" is an assumption about the hosts that
      # happen to import this module today, not a property of the module itself.
      # A future pre-Turing card can then override it at the host level instead of
      # having to edit this file (and note that without mkDefault, importing both
      # gpuNvidia and gpuNvidiaPrime would be an outright merge conflict rather
      # than something resolvable).
      open = lib.mkDefault true;
      nvidiaSettings = true;
      # Required for suspend/hibernate to come back with an intact framebuffer:
      # this sets NVreg_PreserveVideoMemoryAllocations=1, so VRAM contents are
      # spooled out across sleep instead of being discarded.
      #
      # The driver spools VRAM to NVreg_TemporaryFilePath, which nixpkgs leaves
      # unset so it defaults to /tmp. Keep /tmp on a real filesystem — setting
      # boot.tmp.useTmpfs would route a full VRAM dump into RAM, which on a
      # large card is enough to break hibernation outright.
      powerManagement.enable = true;
      # Which mechanism tells the driver about sleep transitions. Two options:
      #
      #   true  -> NVreg_UseKernelSuspendNotifiers=1; the driver hooks the
      #            in-kernel PM notifier chain and the systemd units are NOT
      #            generated.
      #   false -> nvidia-{suspend,hibernate,resume}.service call nvidia-sleep.sh
      #            around systemd-{suspend,hibernate}.service.
      #
      # Pinned to false deliberately. nixpkgs defaults this to
      # `useOpenModules && version >= 595`, which is silently true here (open =
      # true above, driver 595), and that path wedges the GPU on resume — the
      # notifier callback warns and then the card never comes back:
      #
      #   WARNING: nvidia/nv.c:4564 at nv_restore_user_channels+0x1d9/0x240
      #     nv_pm_notifier+0x35/0xa0 [nvidia]
      #   nvidia-modeset: ERROR: GPU:0: Failed detecting connected display devices
      #   NVRM: GPU0 _issueRpcAndWait: rpcSendMessage failed with status 0x11
      #
      # followed by an unrecoverable GSP RPC storm, i.e. black screens until a
      # hard power cycle. Revisit once a newer driver fixes the notifier path;
      # test by flipping this to true and confirming a hibernate round-trip
      # relights the displays before leaving it that way.
      powerManagement.kernelSuspendNotifier = false;
      package = config.boot.kernelPackages.nvidiaPackages.stable;
    };

    # The nvidia-{suspend,hibernate,resume}.service units nixpkgs generates are
    # wired only to systemd-{suspend,hibernate}.service. suspend-then-hibernate
    # runs as a single systemd-suspend-then-hibernate.service that performs both
    # transitions internally, so those units never fire for it and the GPU would
    # sleep with VRAM unspooled.
    #
    # NVIDIA ships a hook that handles exactly this, dispatching on
    # $SYSTEMD_SLEEP_ACTION and covering the suspend-after-failed-hibernate case
    # too. nixpkgs sets `systemd.packages = nvidia_x11.out`, but that only
    # collects lib/systemd/system/, never lib/systemd/system-sleep/, so upstream's
    # hook is silently discarded. Install it by hand.
    #
    # It must NOT be installed verbatim, though. Past its suspend-then-hibernate
    # branch the hook ends in an unconditional fallthrough:
    #
    #   case "$1" in post) nvidia-sleep.sh "resume" ;; esac
    #
    # For plain suspend/hibernate that duplicates nvidia-resume.service, and the
    # duplicate is destructive rather than merely redundant: nvidia-sleep.sh's
    # RestoreVT() reads /run/nvidia-sleep/Xorg.vt_number, *deletes* it, then
    # chvt's back. `chvt` lives in pkgs.kbd, which nixpkgs puts on
    # nvidia-resume.service's `path` but which is absent from the stripped PATH
    # systemd-sleep gives its hooks. So the hook consumes the VT file, fails to
    # chvt, and the real unit then finds nothing to restore — no VT switch at
    # all, i.e. the same black-screen-on-resume this module exists to avoid.
    #
    # Hence the first line of the wrapper: bail out unless systemd is actually
    # doing suspend-then-hibernate, the one transition nixpkgs' units do not
    # cover. Upstream's own comment notes that its suspend-then-hibernate path
    # deliberately writes /proc/driver/nvidia/suspend directly and leaves the VT
    # switch to nvidia-resume.service, so gating here loses nothing.
    #
    # The wrapper also has to PATCH A HOLE in upstream's dispatch. Its
    # suspend-then-hibernate branch is:
    #
    #   case "$1:$SYSTEMD_SLEEP_ACTION" in
    #       post:*)                             echo resume    > ... ;;
    #       pre:hibernate)                      echo hibernate > ... ;;
    #       pre:suspend-after-failed-hibernate) echo suspend   > ... ;;
    #   esac
    #
    # There is no `pre:suspend`. But that is exactly the FIRST leg of
    # suspend-then-hibernate: systemd invokes the hook with $1=pre and
    # SYSTEMD_SLEEP_ACTION=suspend before the S3 transition. It matches nothing,
    # the case falls through, and the driver never gets its VRAM-spool handshake
    # — so with NVreg_PreserveVideoMemoryAllocations=1 the GPU refuses to
    # suspend and the whole operation aborts:
    #
    #   nvidia 0000:2d:00.0: PM: failed to suspend async: error -5
    #   PM: Some devices failed to suspend, or early wake event detected
    #   systemd-sleep: Failed to put system to sleep. System resumed again:
    #     Input/output error
    #
    # Observed on this host: plain suspend and plain hibernate both worked (their
    # nvidia-*.service units did the handshake) while suspend-then-hibernate
    # failed at the S3 leg with no nvidia unit having run at all. So handle
    # pre:suspend here, the same way upstream handles pre:hibernate, then hand
    # everything else to upstream unchanged.
    #
    # Guarded on the same condition nixpkgs uses for the units: with the kernel
    # suspend notifier active the driver handles transitions in-kernel and this
    # hook must not also poke /proc/driver/nvidia/suspend.
    environment.etc."systemd/system-sleep/nvidia" = lib.mkIf (config.hardware.nvidia.powerManagement.enable
      && !config.hardware.nvidia.powerManagement.kernelSuspendNotifier) {
      source = pkgs.writeShellScript "nvidia-system-sleep-hook" ''
        [ "$2" = "suspend-then-hibernate" ] || exit 0

        # The gap upstream leaves: the S3 leg of suspend-then-hibernate.
        if [ "$1" = "pre" ] && [ "$SYSTEMD_SLEEP_ACTION" = "suspend" ]; then
          if [ -f /proc/driver/nvidia/suspend ]; then
            echo suspend > /proc/driver/nvidia/suspend
          fi
          exit 0
        fi

        exec ${config.hardware.nvidia.package.out}/lib/systemd/system-sleep/nvidia "$@"
      '';
    };

    environment.systemPackages = with pkgs; [
      vulkan-tools
    ];
  };
}