diff --git a/nixos/doc/manual/release-notes/rl-2103.xml b/nixos/doc/manual/release-notes/rl-2103.xml
index fda0f8ea074..3186eb7449f 100644
--- a/nixos/doc/manual/release-notes/rl-2103.xml
+++ b/nixos/doc/manual/release-notes/rl-2103.xml
@@ -106,6 +106,13 @@
        for more info.
      </para>
    </listitem>
+   <listitem>
+     <para>
+       Setting <option>services.openssh.authorizedKeysFiles</option> now also affects which keys <option>security.pam.enableSSHAgentAuth</option> will use.
+
+       WARNING: If you are using these options in combination do make sure that any key paths you use are present in <option>services.openssh.authorizedKeysFiles</option>!
+     </para>
+   </listitem>
    <listitem>
     <para>
      The option <option>fonts.enableFontDir</option> has been renamed to
diff --git a/nixos/modules/security/pam.nix b/nixos/modules/security/pam.nix
index a517f9e51ce..a428103eaa9 100644
--- a/nixos/modules/security/pam.nix
+++ b/nixos/modules/security/pam.nix
@@ -396,7 +396,7 @@ let
           ${optionalString cfg.logFailures
               "auth required pam_tally.so"}
           ${optionalString (config.security.pam.enableSSHAgentAuth && cfg.sshAgentAuth)
-              "auth sufficient ${pkgs.pam_ssh_agent_auth}/libexec/pam_ssh_agent_auth.so file=~/.ssh/authorized_keys:~/.ssh/authorized_keys2:/etc/ssh/authorized_keys.d/%u"}
+              "auth sufficient ${pkgs.pam_ssh_agent_auth}/libexec/pam_ssh_agent_auth.so file=${lib.concatStringsSep ":" config.services.openssh.authorizedKeysFiles}"}
           ${optionalString cfg.fprintAuth
               "auth sufficient ${pkgs.fprintd}/lib/security/pam_fprintd.so"}
           ${let p11 = config.security.pam.p11; in optionalString cfg.p11Auth
diff --git a/nixos/modules/services/networking/ssh/sshd.nix b/nixos/modules/services/networking/ssh/sshd.nix
index 1b745931c4f..3cc77e4cb93 100644
--- a/nixos/modules/services/networking/ssh/sshd.nix
+++ b/nixos/modules/services/networking/ssh/sshd.nix
@@ -477,7 +477,7 @@ in
     # https://github.com/NixOS/nixpkgs/pull/10155
     # https://github.com/NixOS/nixpkgs/pull/41745
     services.openssh.authorizedKeysFiles =
-      [ ".ssh/authorized_keys" ".ssh/authorized_keys2" "/etc/ssh/authorized_keys.d/%u" ];
+      [ "%h/.ssh/authorized_keys" "%h/.ssh/authorized_keys2" "/etc/ssh/authorized_keys.d/%u" ];
 
     services.openssh.extraConfig = mkOrder 0
       ''