Unify mutableUsers = { true, false }

With mutableUsers = true, we now ensure that all users and groups that
were created declaratively, are updated or removed
appropriately. Thus, adding a user to users.extraUsers and then
removing it now causes the acoount to be removed from
/etc/passwd. Thus user/group management is fully congruent except that
users and groups that were created imperatively (via useradd/groupadd)
are not touched. We distinguish between declarative and imperative
users/groups by tracking the former in
/var/lib/nixos/declarative-{groups,users}.

With mutableUsers = false, you are now no longer required to specify
UIDs/GIDs for all users. The handling of mutableUsers = true/false is
the same code path; the only difference is that the "false" mode
ignores the existing contents of /etc/{passwd,group}.

The attribute ‘createUser’ is gone. It doesn't really make sense to
specify users that shouldn't be created.

nixos/doc/manual/configuration.xml

@@ -1072,11 +1072,6 @@ users.extraGroups.students.gid = 1000;
 As with users, the group ID (gid) is optional and will be assigned
 automatically if it’s missing.</para>
 
-<warning><para>Currently declarative user management is not perfect:
-<command>nixos-rebuild</command> does not know how to realise certain
-configuration changes.  This includes removing a user or group, and
-removing group membership from a user.</para></warning>
-
 <para>In the imperative style, users and groups are managed by
 commands such as <command>useradd</command>,
 <command>groupmod</command> and so on.  For instance, to create a user

nixos/modules/config/update-users-groups.pl

@@ -0,0 +1,239 @@
+use strict;
+use File::Path qw(make_path);
+use File::Slurp;
+use JSON;
+
+make_path("/var/lib/nixos", { mode => 0755 });
+
+
+# Functions for allocating free GIDs/UIDs. FIXME: respect ID ranges in
+# /etc/login.defs.
+sub allocId {
+    my ($used, $idMin, $idMax, $up, $getid) = @_;
+    my $id = $up ? $idMin : $idMax;
+    while ($id >= $idMin && $id <= $idMax) {
+        if (!$used->{$id} && !defined &$getid($id)) {
+            $used->{$id} = 1;
+            return $id;
+        }
+        $used->{$id} = 1;
+        if ($up) { $id++; } else { $id--; }
+    }
+    die "$0: out of free UIDs or GIDs\n";
+}
+
+my (%gidsUsed, %uidsUsed);
+
+sub allocGid {
+    return allocId(\%gidsUsed, 400, 499, 0, sub { my ($gid) = @_; getgrgid($gid) });
+}
+
+sub allocUid {
+    my ($isSystemUser) = @_;
+    my ($min, $max, $up) = $isSystemUser ? (400, 499, 0) : (1000, 29999, 1);
+    return allocId(\%uidsUsed, $min, $max, $up, sub { my ($uid) = @_; getpwuid($uid) });
+}
+
+
+# Read the declared users/groups.
+my $spec = decode_json(read_file($ARGV[0]));
+
+# Don't allocate UIDs/GIDs that are already in use.
+foreach my $g (@{$spec->{groups}}) {
+    $gidsUsed{$g->{gid}} = 1 if defined $g->{gid};
+}
+
+foreach my $u (@{$spec->{groups}}) {
+    $uidsUsed{$u->{u}} = 1 if defined $u->{uid};
+}
+
+# Read the current /etc/group.
+sub parseGroup {
+    chomp;
+    my @f = split(':', $_, -4);
+    my $gid = $f[2] eq "" ? undef : int($f[2]);
+    $gidsUsed{$gid} = 1 if defined $gid;
+    return ($f[0], { name => $f[0], password => $f[1], gid => $gid, members => $f[3] });
+}
+
+my %groupsCur = -f "/etc/group" ? map { parseGroup } read_file("/etc/group") : ();
+
+# Read the current /etc/passwd.
+sub parseUser {
+    chomp;
+    my @f = split(':', $_, -7);
+    my $uid = $f[2] eq "" ? undef : int($f[2]);
+    $uidsUsed{$uid} = 1 if defined $uid;
+    return ($f[0], { name => $f[0], fakePassword => $f[1], uid => $uid,
+        gid => $f[3], description => $f[4], home => $f[5], shell => $f[6] });
+}
+
+my %usersCur = -f "/etc/passwd" ? map { parseUser } read_file("/etc/passwd") : ();
+
+# Read the groups that were created declaratively (i.e. not by groups)
+# in the past. These must be removed if they are no longer in the
+# current spec.
+my $declGroupsFile = "/var/lib/nixos/declarative-groups";
+my %declGroups;
+$declGroups{$_} = 1 foreach split / /, -e $declGroupsFile ? read_file($declGroupsFile) : "";
+
+# Idem for the users.
+my $declUsersFile = "/var/lib/nixos/declarative-users";
+my %declUsers;
+$declUsers{$_} = 1 foreach split / /, -e $declUsersFile ? read_file($declUsersFile) : "";
+
+
+# Generate a new /etc/group containing the declared groups.
+my %groupsOut;
+foreach my $g (@{$spec->{groups}}) {
+    my $name = $g->{name};
+    my $existing = $groupsCur{$name};
+
+    my %members = map { ($_, 1) } @{$g->{members}};
+
+    if (defined $existing) {
+        $g->{gid} = $existing->{gid} if !defined $g->{gid};
+        if ($g->{gid} != $existing->{gid}) {
+            warn "warning: not applying GID change of group ‘$name’\n";
+            $g->{gid} = $existing->{gid};
+        }
+        $g->{password} = $existing->{password}; # do we want this?
+        if ($spec->{mutableUsers}) {
+            # Merge in non-declarative group members.
+            foreach my $uname (split /,/, $existing->{members} // "") {
+                $members{$uname} = 1 if !defined $declUsers{$uname};
+            }
+        }
+    } else {
+        $g->{gid} = allocGid if !defined $g->{gid};
+        $g->{password} = "x";
+    }
+
+    $g->{members} = join ",", sort(keys(%members));
+    $groupsOut{$name} = $g;
+}
+
+# Update the persistent list of declarative groups.
+write_file($declGroupsFile, join(" ", sort(keys %groupsOut)));
+
+# Merge in the existing /etc/group.
+foreach my $name (keys %groupsCur) {
+    my $g = $groupsCur{$name};
+    next if defined $groupsOut{$name};
+    if (!$spec->{mutableUsers} || defined $declGroups{$name}) {
+        print STDERR "removing group ‘$name’\n";
+    } else {
+        $groupsOut{$name} = $g;
+    }
+}
+
+
+# Rewrite /etc/group. FIXME: acquire lock.
+my @lines = map { join(":", $_->{name}, $_->{password}, $_->{gid}, $_->{members}) . "\n" }
+    (sort { $a->{gid} <=> $b->{gid} } values(%groupsOut));
+write_file("/etc/group.tmp", @lines);
+rename("/etc/group.tmp", "/etc/group") or die;
+system("nscd --invalidate group");
+
+# Generate a new /etc/passwd containing the declared users.
+my %usersOut;
+foreach my $u (@{$spec->{users}}) {
+    my $name = $u->{name};
+
+    # Resolve the gid of the user.
+    if ($u->{group} =~ /^[0-9]$/) {
+        $u->{gid} = $u->{group};
+    } elsif (defined $groupsOut{$u->{group}}) {
+        $u->{gid} = $groupsOut{$u->{group}}->{gid} // die;
+    } else {
+        warn "warning: user ‘$name’ has unknown group ‘$u->{group}’\n";
+        $u->{gid} = 65534;
+    }
+
+    my $existing = $usersCur{$name};
+    if (defined $existing) {
+        $u->{uid} = $existing->{uid} if !defined $u->{uid};
+        if ($u->{uid} != $existing->{uid}) {
+            warn "warning: not applying UID change of user ‘$name’\n";
+            $u->{uid} = $existing->{uid};
+        }
+    } else {
+        $u->{uid} = allocUid($u->{isSystemUser}) if !defined $u->{uid};
+
+        # Create a home directory.
+        if ($u->{createHome}) {
+            make_path($u->{home}, { mode => 0700 }) if ! -e $u->{home};
+            chown $u->{uid}, $u->{gid}, $u->{home};
+        }
+    }
+
+    if (defined $u->{passwordFile}) {
+        if (-e $u->{passwordFile}) {
+            $u->{hashedPassword} = read_file($u->{passwordFile});
+            chomp $u->{hashedPassword};
+        } else {
+            warn "warning: password file ‘$u->{passwordFile}’ does not exist\n";
+        }
+    }
+
+    $u->{fakePassword} = $existing->{fakePassword} // "x";
+    $usersOut{$name} = $u;
+}
+
+# Update the persistent list of declarative users.
+write_file($declUsersFile, join(" ", sort(keys %usersOut)));
+
+# Merge in the existing /etc/passwd.
+foreach my $name (keys %usersCur) {
+    my $u = $usersCur{$name};
+    next if defined $usersOut{$name};
+    if (!$spec->{mutableUsers} || defined $declUsers{$name}) {
+        print STDERR "removing user ‘$name’\n";
+    } else {
+        $usersOut{$name} = $u;
+    }
+}
+
+# Rewrite /etc/passwd. FIXME: acquire lock.
+@lines = map { join(":", $_->{name}, $_->{fakePassword}, $_->{uid}, $_->{gid}, $_->{description}, $_->{home}, $_->{shell}) . "\n" }
+    (sort { $a->{uid} <=> $b->{uid} } (values %usersOut));
+write_file("/etc/passwd.tmp", @lines);
+rename("/etc/passwd.tmp", "/etc/passwd") or die;
+system("nscd --invalidate passwd");
+
+
+# Rewrite /etc/shadow to add new accounts or remove dead ones.
+my @shadowNew;
+my %shadowSeen;
+
+foreach my $line (-f "/etc/shadow" ? read_file("/etc/shadow") : ()) {
+    chomp $line;
+    my ($name, $password, @rest) = split(':', $line, -9);
+    my $u = $usersOut{$name};;
+    next if !defined $u;
+    $password = $u->{hashedPassword} if $u->{hashedPassword} && !$spec->{mutableUsers}; # FIXME
+    push @shadowNew, join(":", $name, $password, @rest) . "\n";
+    $shadowSeen{$name} = 1;
+}
+
+foreach my $u (values %usersOut) {
+    next if defined $shadowSeen{$u->{name}};
+    my $password = "!";
+    $password = $u->{hashedPassword} if $u->{hashedPassword};
+    # FIXME: set correct value for sp_lstchg.
+    push @shadowNew, join(":", $u->{name}, $password, "1::::::") . "\n";
+}
+
+write_file("/etc/shadow.tmp", { perms => 0600 }, @shadowNew);
+rename("/etc/shadow.tmp", "/etc/shadow") or die;
+
+
+# Call chpasswd to apply password. FIXME: generate the hashes directly
+# and merge into the /etc/shadow updating above.
+foreach my $u (@{$spec->{users}}) {
+    if (defined $u->{password}) {
+        my $pid = open(PW, "| chpasswd") or die;
+        print PW "$u->{name}:$u->{password}\n";
+        close PW or die "unable to change password of user ‘$u->{name}’: $?\n";
+    }
+}

nixos/modules/config/users-groups.nix

@@ -7,9 +7,6 @@ let
   ids = config.ids;
   cfg = config.users;
 
-  nonUidUsers = filterAttrs (n: u: u.createUser && u.uid == null) cfg.extraUsers;
-  nonGidGroups = filterAttrs (n: g: g.gid == null) cfg.extraGroups;
-
   passwordDescription = ''
     The options <literal>hashedPassword</literal>,
     <literal>password</literal> and <literal>passwordFile</literal>
@@ -55,10 +52,8 @@ let
         type = with types; nullOr int;
         default = null;
         description = ''
-          The account UID. If the <option>mutableUsers</option> option
+          The account UID. If the UID is null, a free UID is picked on
-          is false, the UID cannot be null. Otherwise, the UID might be
+          activation.
-          null, in which case a free UID is picked on activation (by the
-          useradd command).
         '';
       };
 
@@ -67,8 +62,7 @@ let
         default = false;
         description = ''
           Indicates if the user is a system user or not. This option
-          only has an effect if <option>mutableUsers</option> is
+          only has an effect if <option>uid</option> is
-          <literal>true</literal> and <option>uid</option> is
           <option>null</option>, in which case it determines whether
           the user's UID is allocated in the range for system users
           (below 500) or in the range for normal users (starting at
@@ -152,16 +146,6 @@ let
           ${passwordDescription}
         '';
       };
-
-      createUser = mkOption {
-        type = types.bool;
-        default = true;
-        description = ''
-          Indicates if the user should be created automatically as a local user.
-          Set this to false if the user for instance is an LDAP user. NixOS will
-          then not modify any of the basic properties for the user account.
-        '';
-      };
     };
 
     config = {
@@ -187,10 +171,8 @@ let
         type = with types; nullOr int;
         default = null;
         description = ''
-          The group GID. If the <literal>mutableUsers</literal> option
+          The group GID. If the GID is null, a free GID is picked on
-          is false, the GID cannot be null. Otherwise, the GID might be
+          activation.
-          null, in which case a free GID is picked on activation (by the
-          groupadd command).
         '';
       };
 
@@ -211,84 +193,6 @@ let
 
   };
 
-  getGroup = gname:
-    let
-      groups = mapAttrsToList (n: g: g) (
-        filterAttrs (n: g: g.name == gname) cfg.extraGroups
-      );
-    in
-      if length groups == 1 then head groups
-      else if groups == [] then throw "Group ${gname} not defined"
-      else throw "Group ${gname} has multiple definitions";
-
-  getUser = uname:
-    let
-      users = mapAttrsToList (n: u: u) (
-        filterAttrs (n: u: u.name == uname) cfg.extraUsers
-      );
-    in
-      if length users == 1 then head users
-      else if users == [] then throw "User ${uname} not defined"
-      else throw "User ${uname} has multiple definitions";
-
-  mkGroupEntry = gname:
-    let
-      g = getGroup gname;
-      users = mapAttrsToList (n: u: u.name) (
-        filterAttrs (n: u: elem g.name u.extraGroups) cfg.extraUsers
-      );
-    in concatStringsSep ":" [
-      g.name "x" (toString g.gid)
-      (concatStringsSep "," (users ++ (filter (u: !(elem u users)) g.members)))
-    ];
-
-  mkPasswdEntry = uname: let u = getUser uname; in
-    concatStringsSep ":" [
-      u.name "x" (toString u.uid)
-      (toString (getGroup u.group).gid)
-      u.description u.home u.shell
-    ];
-
-  sortOn = a: sort (as1: as2: lessThan (getAttr a as1) (getAttr a as2));
-
-  groupFile = pkgs.writeText "group" (
-    concatStringsSep "\n" (map (g: mkGroupEntry g.name) (
-      let f = g: g.gid != null; in
-        sortOn "gid" (filter f (attrValues cfg.extraGroups))
-    ))
-  );
-
-  passwdFile = pkgs.writeText "passwd" (
-    concatStringsSep "\n" (map (u: mkPasswdEntry u.name) (
-      let f = u: u.createUser && (u.uid != null); in
-        sortOn "uid" (filter f (attrValues cfg.extraUsers))
-    ))
-  );
-
-  # If mutableUsers is true, this script adds all users/groups defined in
-  # users.extra{Users,Groups} to /etc/{passwd,group} iff there isn't any
-  # existing user/group with the same name in those files.
-  # If mutableUsers is false, the /etc/{passwd,group} files will simply be
-  # replaced with the users/groups defined in the NixOS configuration.
-  # The merging procedure could certainly be improved, and instead of just
-  # keeping the lines as-is from /etc/{passwd,group} they could be combined
-  # in some way with the generated content from the NixOS configuration.
-  merger = src: pkgs.writeScript "merger" ''
-    #!${pkgs.bash}/bin/bash
-
-    PATH=${pkgs.gawk}/bin:${pkgs.gnugrep}/bin:$PATH
-
-    ${if !cfg.mutableUsers
-      then ''cp ${src} $1.tmp''
-      else ''awk -F: '{ print "^"$1":.*" }' $1 | egrep -vf - ${src} | cat $1 - > $1.tmp''
-    }
-
-    # set mtime to +1, otherwise change might go unnoticed (vipw/vigr only looks at mtime)
-    touch -m -t $(date -d @$(($(stat -c %Y $1)+1)) +%Y%m%d%H%M.%S) $1.tmp
-
-    mv -f $1.tmp $1
-  '';
-
   idsAreUnique = set: idAttr: !(fold (name: args@{ dup, acc }:
     let
       id = builtins.toString (builtins.getAttr idAttr (builtins.getAttr name set));
@@ -302,6 +206,21 @@ let
   uidsAreUnique = idsAreUnique (filterAttrs (n: u: u.uid != null) cfg.extraUsers) "uid";
   gidsAreUnique = idsAreUnique (filterAttrs (n: g: g.gid != null) cfg.extraGroups) "gid";
 
+  spec = builtins.toFile "users-groups.json" (builtins.toJSON {
+    inherit (cfg) mutableUsers;
+    users = mapAttrsToList (n: u:
+      { inherit (u)
+          name uid group description home shell createHome isSystemUser
+          password passwordFile hashedPassword;
+      }) cfg.extraUsers;
+    groups = mapAttrsToList (n: g:
+      { inherit (g) name gid;
+        members = mapAttrsToList (n: u: u.name) (
+          filterAttrs (n: u: elem g.name u.extraGroups) cfg.extraUsers
+        );
+      }) cfg.extraGroups;
+  });
+
 in {
 
   ###### interface
@@ -438,67 +357,12 @@ in {
       grsecurity.gid = ids.gids.grsecurity;
     };
 
-    system.activationScripts.users =
+    system.activationScripts.users = stringAfter [ "etc" ]
-      let
-        mkhomeUsers = filterAttrs (n: u: u.createHome) cfg.extraUsers;
-        setpwUsers = filterAttrs (n: u: u.createUser) cfg.extraUsers;
-        pwFile = u: if !(isNull u.hashedPassword)
-          then pkgs.writeTextFile { name = "password-file"; text = u.hashedPassword; }
-          else if !(isNull u.password)
-          then pkgs.runCommand "password-file" { pw = u.password; } ''
-            echo -n "$pw" | ${pkgs.mkpasswd}/bin/mkpasswd -s > $out
-          '' else u.passwordFile;
-        setpw = n: u: ''
-          setpw=yes
-          ${optionalString cfg.mutableUsers ''
-            test "$(getent shadow '${u.name}' | cut -d: -f2)" != "x" && setpw=no
-          ''}
-          if [ "$setpw" == "yes" ]; then
-            ${if !(isNull (pwFile u))
-              then ''
-                echo -n "${u.name}:" | cat - "${pwFile u}" | \
-                  ${pkgs.shadow}/sbin/chpasswd -e
       ''
-              else "passwd -l '${u.name}' &>/dev/null"
+        ${pkgs.perl}/bin/perl -w \
-            }
+          -I${pkgs.perlPackages.FileSlurp}/lib/perl5/site_perl \
-          fi
+          -I${pkgs.perlPackages.JSON}/lib/perl5/site_perl \
-        '';
+          ${./update-users-groups.pl} ${spec}
-        mkhome = n: u: ''
-          uid="$(id -u ${u.name})"
-          gid="$(id -g ${u.name})"
-          h="${u.home}"
-          test -a "$h" || mkdir -p "$h" || true
-          test "$(stat -c %u "$h")" = $uid || chown $uid "$h" || true
-          test "$(stat -c %g "$h")" = $gid || chgrp $gid "$h" || true
-        '';
-        groupadd = n: g: ''
-          if [ -z "$(getent group "${g.name}")" ]; then
-            ${pkgs.shadow}/sbin/groupadd "${g.name}"
-          fi
-        '';
-        useradd = n: u: ''
-          if ! id "${u.name}" &>/dev/null; then
-            ${pkgs.shadow}/sbin/useradd \
-              -g "${u.group}" \
-              -G "${concatStringsSep "," u.extraGroups}" \
-              -s "${u.shell}" \
-              -d "${u.home}" \
-              ${optionalString u.isSystemUser "--system"} \
-              "${u.name}"
-            echo "${u.name}:x" | ${pkgs.shadow}/sbin/chpasswd -e
-          fi
-        '';
-      in stringAfter [ "etc" ] ''
-        touch /etc/group
-        touch /etc/passwd
-        VISUAL=${merger groupFile} ${pkgs.shadow}/sbin/vigr &>/dev/null
-        VISUAL=${merger passwdFile} ${pkgs.shadow}/sbin/vipw &>/dev/null
-        ${pkgs.shadow}/sbin/grpconv
-        ${pkgs.shadow}/sbin/pwconv
-        ${concatStrings (mapAttrsToList groupadd nonGidGroups)}
-        ${concatStrings (mapAttrsToList useradd nonUidUsers)}
-        ${concatStrings (mapAttrsToList mkhome mkhomeUsers)}
-        ${concatStrings (mapAttrsToList setpw setpwUsers)}
       '';
 
     # for backwards compatibility
@@ -506,13 +370,7 @@ in {
 
     assertions = [
       { assertion = !cfg.enforceIdUniqueness || (uidsAreUnique && gidsAreUnique);
-        message = "uids and gids must be unique!";
+        message = "UIDs and GIDs must be unique!";
-      }
-      { assertion = cfg.mutableUsers || (nonUidUsers == {});
-        message = "When mutableUsers is false, no uid can be null: ${toString (attrNames nonUidUsers)}";
-      }
-      { assertion = cfg.mutableUsers || (nonGidGroups == {});
-        message = "When mutableUsers is false, no gid can be null";
       }
     ];