diff --git a/modules/system/hardware/gpu/nvidia.nix b/modules/system/hardware/gpu/nvidia.nix index c01461f..41f29d6 100644 --- a/modules/system/hardware/gpu/nvidia.nix +++ b/modules/system/hardware/gpu/nvidia.nix @@ -3,6 +3,7 @@ # For Intel+NVIDIA laptops (e.g. the T480), use gpuNvidiaPrime instead. flake.nixosModules.gpuNvidia = { config, + lib, pkgs, ... }: { @@ -15,15 +16,147 @@ hardware.nvidia = { modesetting.enable = true; - # Proprietary kernel modules. Set to true only for Turing+ (RTX 20xx and - # newer) where the open modules are well supported. - open = false; + # Open kernel modules. Mandatory to set explicitly on driver >= 560 — + # the option defaults to null there and nixpkgs asserts against null, so + # this line cannot simply be dropped. + # + # true because every card this module targets is Turing or later (RTX 20xx + # and newer), which is exactly what upstream recommends open modules for. + # Set false only for a genuinely pre-Turing card, which would want the + # gpuNvidia module pointed at a legacy package branch anyway. + # + # This is NOT cosmetic and NOT a version switch: `open` selects between two + # real builds of the same release — nvidia-open (~12MB) vs the proprietary + # nvidia-kernel-modules (~85MB). It has no effect on driver version, which + # comes solely from `package` below. + # + # Kernel modules only change on reboot, so `cat /proc/driver/nvidia/version` + # describes the RUNNING module and can disagree with this setting until the + # next boot. Compare /run/booted-system against /run/current-system to see + # a pending swap. + # + # mkDefault because "Turing or later" is an assumption about the hosts that + # happen to import this module today, not a property of the module itself. + # A future pre-Turing card can then override it at the host level instead of + # having to edit this file (and note that without mkDefault, importing both + # gpuNvidia and gpuNvidiaPrime would be an outright merge conflict rather + # than something resolvable). + open = lib.mkDefault true; nvidiaSettings = true; - # Fixes suspend/resume corruption on many setups. + # Required for suspend/hibernate to come back with an intact framebuffer: + # this sets NVreg_PreserveVideoMemoryAllocations=1, so VRAM contents are + # spooled out across sleep instead of being discarded. + # + # The driver spools VRAM to NVreg_TemporaryFilePath, which nixpkgs leaves + # unset so it defaults to /tmp. Keep /tmp on a real filesystem — setting + # boot.tmp.useTmpfs would route a full VRAM dump into RAM, which on a + # large card is enough to break hibernation outright. powerManagement.enable = true; + # Which mechanism tells the driver about sleep transitions. Two options: + # + # true -> NVreg_UseKernelSuspendNotifiers=1; the driver hooks the + # in-kernel PM notifier chain and the systemd units are NOT + # generated. + # false -> nvidia-{suspend,hibernate,resume}.service call nvidia-sleep.sh + # around systemd-{suspend,hibernate}.service. + # + # Pinned to false deliberately. nixpkgs defaults this to + # `useOpenModules && version >= 595`, which is silently true here (open = + # true above, driver 595), and that path wedges the GPU on resume — the + # notifier callback warns and then the card never comes back: + # + # WARNING: nvidia/nv.c:4564 at nv_restore_user_channels+0x1d9/0x240 + # nv_pm_notifier+0x35/0xa0 [nvidia] + # nvidia-modeset: ERROR: GPU:0: Failed detecting connected display devices + # NVRM: GPU0 _issueRpcAndWait: rpcSendMessage failed with status 0x11 + # + # followed by an unrecoverable GSP RPC storm, i.e. black screens until a + # hard power cycle. Revisit once a newer driver fixes the notifier path; + # test by flipping this to true and confirming a hibernate round-trip + # relights the displays before leaving it that way. + powerManagement.kernelSuspendNotifier = false; package = config.boot.kernelPackages.nvidiaPackages.stable; }; + # The nvidia-{suspend,hibernate,resume}.service units nixpkgs generates are + # wired only to systemd-{suspend,hibernate}.service. suspend-then-hibernate + # runs as a single systemd-suspend-then-hibernate.service that performs both + # transitions internally, so those units never fire for it and the GPU would + # sleep with VRAM unspooled. + # + # NVIDIA ships a hook that handles exactly this, dispatching on + # $SYSTEMD_SLEEP_ACTION and covering the suspend-after-failed-hibernate case + # too. nixpkgs sets `systemd.packages = nvidia_x11.out`, but that only + # collects lib/systemd/system/, never lib/systemd/system-sleep/, so upstream's + # hook is silently discarded. Install it by hand. + # + # It must NOT be installed verbatim, though. Past its suspend-then-hibernate + # branch the hook ends in an unconditional fallthrough: + # + # case "$1" in post) nvidia-sleep.sh "resume" ;; esac + # + # For plain suspend/hibernate that duplicates nvidia-resume.service, and the + # duplicate is destructive rather than merely redundant: nvidia-sleep.sh's + # RestoreVT() reads /run/nvidia-sleep/Xorg.vt_number, *deletes* it, then + # chvt's back. `chvt` lives in pkgs.kbd, which nixpkgs puts on + # nvidia-resume.service's `path` but which is absent from the stripped PATH + # systemd-sleep gives its hooks. So the hook consumes the VT file, fails to + # chvt, and the real unit then finds nothing to restore — no VT switch at + # all, i.e. the same black-screen-on-resume this module exists to avoid. + # + # Hence the first line of the wrapper: bail out unless systemd is actually + # doing suspend-then-hibernate, the one transition nixpkgs' units do not + # cover. Upstream's own comment notes that its suspend-then-hibernate path + # deliberately writes /proc/driver/nvidia/suspend directly and leaves the VT + # switch to nvidia-resume.service, so gating here loses nothing. + # + # The wrapper also has to PATCH A HOLE in upstream's dispatch. Its + # suspend-then-hibernate branch is: + # + # case "$1:$SYSTEMD_SLEEP_ACTION" in + # post:*) echo resume > ... ;; + # pre:hibernate) echo hibernate > ... ;; + # pre:suspend-after-failed-hibernate) echo suspend > ... ;; + # esac + # + # There is no `pre:suspend`. But that is exactly the FIRST leg of + # suspend-then-hibernate: systemd invokes the hook with $1=pre and + # SYSTEMD_SLEEP_ACTION=suspend before the S3 transition. It matches nothing, + # the case falls through, and the driver never gets its VRAM-spool handshake + # — so with NVreg_PreserveVideoMemoryAllocations=1 the GPU refuses to + # suspend and the whole operation aborts: + # + # nvidia 0000:2d:00.0: PM: failed to suspend async: error -5 + # PM: Some devices failed to suspend, or early wake event detected + # systemd-sleep: Failed to put system to sleep. System resumed again: + # Input/output error + # + # Observed on this host: plain suspend and plain hibernate both worked (their + # nvidia-*.service units did the handshake) while suspend-then-hibernate + # failed at the S3 leg with no nvidia unit having run at all. So handle + # pre:suspend here, the same way upstream handles pre:hibernate, then hand + # everything else to upstream unchanged. + # + # Guarded on the same condition nixpkgs uses for the units: with the kernel + # suspend notifier active the driver handles transitions in-kernel and this + # hook must not also poke /proc/driver/nvidia/suspend. + environment.etc."systemd/system-sleep/nvidia" = lib.mkIf (config.hardware.nvidia.powerManagement.enable + && !config.hardware.nvidia.powerManagement.kernelSuspendNotifier) { + source = pkgs.writeShellScript "nvidia-system-sleep-hook" '' + [ "$2" = "suspend-then-hibernate" ] || exit 0 + + # The gap upstream leaves: the S3 leg of suspend-then-hibernate. + if [ "$1" = "pre" ] && [ "$SYSTEMD_SLEEP_ACTION" = "suspend" ]; then + if [ -f /proc/driver/nvidia/suspend ]; then + echo suspend > /proc/driver/nvidia/suspend + fi + exit 0 + fi + + exec ${config.hardware.nvidia.package.out}/lib/systemd/system-sleep/nvidia "$@" + ''; + }; + environment.systemPackages = with pkgs; [ vulkan-tools ];