Merge pull request #114362 from talyz/gitlab-backups

nixos/gitlab: Introduce automatic backups

nixos/doc/manual/release-notes/rl-2105.xml

@@ -883,6 +883,14 @@ environment.systemPackages = [
      Please test your setup and container images with containerd prior to upgrading.
     </para>
    </listitem>
+   <listitem>
+    <para>
+     The GitLab module now has support for automatic backups. A
+     schedule can be set with the
+     <link linkend="opt-services.gitlab.backup.startAt">services.gitlab.backup.startAt</link>
+     option.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 </section>

nixos/modules/services/misc/gitlab.nix

@@ -116,7 +116,11 @@ let
       omniauth.enabled = false;
       shared.path = "${cfg.statePath}/shared";
       gitaly.client_path = "${cfg.packages.gitaly}/bin";
-      backup.path = "${cfg.backupPath}";
+      backup = {
+        path = cfg.backup.path;
+        keep_time = cfg.backup.keepTime;
+        upload = cfg.backup.uploadOptions;
+      };
       gitlab_shell = {
         path = "${cfg.packages.gitlab-shell}";
         hooks_path = "${cfg.statePath}/shell/hooks";
@@ -207,6 +211,7 @@ in {
 
   imports = [
     (mkRenamedOptionModule [ "services" "gitlab" "stateDir" ] [ "services" "gitlab" "statePath" ])
+    (mkRenamedOptionModule [ "services" "gitlab" "backupPath" ] [ "services" "gitlab" "backup" "path" ])
     (mkRemovedOptionModule [ "services" "gitlab" "satelliteDir" ] "")
   ];
 
@@ -260,7 +265,7 @@ in {
         type = types.str;
         default = "/var/gitlab/state";
         description = ''
-          Gitlab state directory. Configuration, repositories and
+          GitLab state directory. Configuration, repositories and
           logs, among other things, are stored here.
 
           The directory will be created automatically if it doesn't
@@ -270,17 +275,108 @@ in {
         '';
       };
 
-      backupPath = mkOption {
+      backup.startAt = mkOption {
+        type = with types; either str (listOf str);
+        default = [];
+        example = "03:00";
+        description = ''
+          The time(s) to run automatic backup of GitLab
+          state. Specified in systemd's time format; see
+          <citerefentry><refentrytitle>systemd.time</refentrytitle>
+          <manvolnum>7</manvolnum></citerefentry>.
+        '';
+      };
+
+      backup.path = mkOption {
         type = types.str;
         default = cfg.statePath + "/backup";
-        description = "Gitlab path for backups.";
+        description = "GitLab path for backups.";
+      };
+
+      backup.keepTime = mkOption {
+        type = types.int;
+        default = 0;
+        example = 48;
+        apply = x: x * 60 * 60;
+        description = ''
+          How long to keep the backups around, in
+          hours. <literal>0</literal> means <quote>keep
+          forever</quote>.
+        '';
+      };
+
+      backup.skip = mkOption {
+        type = with types;
+          let value = enum [
+                "db"
+                "uploads"
+                "builds"
+                "artifacts"
+                "lfs"
+                "registry"
+                "pages"
+                "repositories"
+                "tar"
+              ];
+          in
+            either value (listOf value);
+        default = [];
+        example = [ "artifacts" "lfs" ];
+        apply = x: if isString x then x else concatStringsSep "," x;
+        description = ''
+          Directories to exclude from the backup. The example excludes
+          CI artifacts and LFS objects from the backups. The
+          <literal>tar</literal> option skips the creation of a tar
+          file.
+
+          Refer to <link xlink:href="https://docs.gitlab.com/ee/raketasks/backup_restore.html#excluding-specific-directories-from-the-backup"/>
+          for more information.
+        '';
+      };
+
+      backup.uploadOptions = mkOption {
+        type = types.attrs;
+        default = {};
+        example = literalExample ''
+          {
+            # Fog storage connection settings, see http://fog.io/storage/
+            connection = {
+              provider = "AWS";
+              region = "eu-north-1";
+              aws_access_key_id = "AKIAXXXXXXXXXXXXXXXX";
+              aws_secret_access_key = { _secret = config.deployment.keys.aws_access_key.path; };
+            };
+
+            # The remote 'directory' to store your backups in.
+            # For S3, this would be the bucket name.
+            remote_directory = "my-gitlab-backups";
+
+            # Use multipart uploads when file size reaches 100MB, see
+            # http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html
+            multipart_chunk_size = 104857600;
+
+            # Turns on AWS Server-Side Encryption with Amazon S3-Managed Keys for backups, this is optional
+            encryption = "AES256";
+
+            # Specifies Amazon S3 storage class to use for backups, this is optional
+            storage_class = "STANDARD";
+          };
+        '';
+        description = ''
+          GitLab automatic upload specification. Tells GitLab to
+          upload the backup to a remote location when done.
+
+          Attributes specified here are added under
+          <literal>production -> backup -> upload</literal> in
+          <filename>config/gitlab.yml</filename>.
+        '';
       };
 
       databaseHost = mkOption {
         type = types.str;
         default = "";
         description = ''
-          Gitlab database hostname. An empty string means <quote>use
+          GitLab database hostname. An empty string means <quote>use
           local unix socket connection</quote>.
         '';
       };
@@ -289,7 +385,7 @@ in {
         type = with types; nullOr path;
         default = null;
         description = ''
-          File containing the Gitlab database user password.
+          File containing the GitLab database user password.
 
           This should be a string, not a nix path, since nix paths are
           copied into the world-readable nix store.
@@ -310,13 +406,13 @@ in {
       databaseName = mkOption {
         type = types.str;
         default = "gitlab";
-        description = "Gitlab database name.";
+        description = "GitLab database name.";
       };
 
       databaseUsername = mkOption {
         type = types.str;
         default = "gitlab";
-        description = "Gitlab database user.";
+        description = "GitLab database user.";
       };
 
       databasePool = mkOption {
@@ -360,14 +456,14 @@ in {
       host = mkOption {
         type = types.str;
         default = config.networking.hostName;
-        description = "Gitlab host name. Used e.g. for copy-paste URLs.";
+        description = "GitLab host name. Used e.g. for copy-paste URLs.";
       };
 
       port = mkOption {
         type = types.int;
         default = 8080;
         description = ''
-          Gitlab server port for copy-paste URLs, e.g. 80 or 443 if you're
+          GitLab server port for copy-paste URLs, e.g. 80 or 443 if you're
           service over https.
         '';
       };
@@ -420,26 +516,26 @@ in {
         address = mkOption {
           type = types.str;
           default = "localhost";
-          description = "Address of the SMTP server for Gitlab.";
+          description = "Address of the SMTP server for GitLab.";
         };
 
         port = mkOption {
           type = types.int;
           default = 25;
-          description = "Port of the SMTP server for Gitlab.";
+          description = "Port of the SMTP server for GitLab.";
         };
 
         username = mkOption {
           type = with types; nullOr str;
           default = null;
-          description = "Username of the SMTP server for Gitlab.";
+          description = "Username of the SMTP server for GitLab.";
         };
 
         passwordFile = mkOption {
           type = types.nullOr types.path;
           default = null;
           description = ''
-            File containing the password of the SMTP server for Gitlab.
+            File containing the password of the SMTP server for GitLab.
 
             This should be a string, not a nix path, since nix paths
             are copied into the world-readable nix store.
@@ -720,7 +816,7 @@ in {
       "d /run/gitlab 0755 ${cfg.user} ${cfg.group} -"
       "d ${gitlabEnv.HOME} 0750 ${cfg.user} ${cfg.group} -"
       "z ${gitlabEnv.HOME}/.ssh/authorized_keys 0600 ${cfg.user} ${cfg.group} -"
-      "d ${cfg.backupPath} 0750 ${cfg.user} ${cfg.group} -"
+      "d ${cfg.backup.path} 0750 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath} 0750 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath}/builds 0750 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath}/config 0750 ${cfg.user} ${cfg.group} -"
@@ -1053,6 +1149,23 @@ in {
 
     };
 
+    systemd.services.gitlab-backup = {
+      after = [ "gitlab.service" ];
+      bindsTo = [ "gitlab.service" ];
+      startAt = cfg.backup.startAt;
+      environment = {
+        RAILS_ENV = "production";
+        CRON = "1";
+      } // optionalAttrs (stringLength cfg.backup.skip > 0) {
+        SKIP = cfg.backup.skip;
+      };
+      serviceConfig = {
+        User = cfg.user;
+        Group = cfg.group;
+        ExecStart = "${gitlab-rake}/bin/gitlab-rake gitlab:backup:create";
+      };
+    };
+
   };
 
   meta.doc = ./gitlab.xml;

nixos/modules/services/misc/gitlab.xml

@@ -3,15 +3,15 @@
          xmlns:xi="http://www.w3.org/2001/XInclude"
          version="5.0"
          xml:id="module-services-gitlab">
- <title>Gitlab</title>
+ <title>GitLab</title>
  <para>
-  Gitlab is a feature-rich git hosting service.
+  GitLab is a feature-rich git hosting service.
  </para>
  <section xml:id="module-services-gitlab-prerequisites">
   <title>Prerequisites</title>
 
   <para>
-   The gitlab service exposes only an Unix socket at
+   The <literal>gitlab</literal> service exposes only an Unix socket at
    <literal>/run/gitlab/gitlab-workhorse.socket</literal>. You need to
    configure a webserver to proxy HTTP requests to the socket.
   </para>
@@ -39,7 +39,7 @@
   <title>Configuring</title>
 
   <para>
-   Gitlab depends on both PostgreSQL and Redis and will automatically enable
+   GitLab depends on both PostgreSQL and Redis and will automatically enable
    both services. In the case of PostgreSQL, a database and a role will be
    created.
   </para>
@@ -85,20 +85,20 @@ services.gitlab = {
   </para>
 
   <para>
-   If you're setting up a new Gitlab instance, generate new
+   If you're setting up a new GitLab instance, generate new
    secrets. You for instance use <literal>tr -dc A-Za-z0-9 &lt;
    /dev/urandom | head -c 128 &gt; /var/keys/gitlab/db</literal> to
    generate a new db secret. Make sure the files can be read by, and
    only by, the user specified by <link
-   linkend="opt-services.gitlab.user">services.gitlab.user</link>. Gitlab
+   linkend="opt-services.gitlab.user">services.gitlab.user</link>. GitLab
    encrypts sensitive data stored in the database. If you're restoring
-   an existing Gitlab instance, you must specify the secrets secret
+   an existing GitLab instance, you must specify the secrets secret
-   from <literal>config/secrets.yml</literal> located in your Gitlab
+   from <literal>config/secrets.yml</literal> located in your GitLab
    state folder.
   </para>
 
   <para>
-    When <literal>icoming_mail.enabled</literal> is set to <literal>true</literal>
+    When <literal>incoming_mail.enabled</literal> is set to <literal>true</literal>
     in <link linkend="opt-services.gitlab.extraConfig">extraConfig</link> an additional
     service called <literal>gitlab-mailroom</literal> is enabled for fetching incoming mail.
   </para>
@@ -112,21 +112,40 @@ services.gitlab = {
  <section xml:id="module-services-gitlab-maintenance">
   <title>Maintenance</title>
 
+  <section xml:id="module-services-gitlab-maintenance-backups">
+   <title>Backups</title>
    <para>
-   You can run Gitlab's rake tasks with <literal>gitlab-rake</literal> which
+     Backups can be configured with the options in <link
-   will be available on the system when gitlab is enabled. You will have to run
+     linkend="opt-services.gitlab.backup.keepTime">services.gitlab.backup</link>. Use
-   the command as the user that you configured to run gitlab with.
+     the <link
+     linkend="opt-services.gitlab.backup.startAt">services.gitlab.backup.startAt</link>
+     option to configure regular backups.
    </para>
 
    <para>
-   For example, to backup a Gitlab instance:
+     To run a manual backup, start the <literal>gitlab-backup</literal> service:
 <screen>
-<prompt>$ </prompt>sudo -u git -H gitlab-rake gitlab:backup:create
+<prompt>$ </prompt>systemctl start gitlab-backup.service
 </screen>
+   </para>
+  </section>
+
+  <section xml:id="module-services-gitlab-maintenance-rake">
+   <title>Rake tasks</title>
+
+   <para>
+    You can run GitLab's rake tasks with <literal>gitlab-rake</literal>
+    which will be available on the system when GitLab is enabled. You
+    will have to run the command as the user that you configured to run
+    GitLab with.
+   </para>
+
+   <para>
     A list of all availabe rake tasks can be obtained by running:
 <screen>
 <prompt>$ </prompt>sudo -u git -H gitlab-rake -T
 </screen>
    </para>
   </section>
+ </section>
 </chapter>

nixos/tests/gitlab.nix

@@ -34,6 +34,8 @@ import ./make-test-python.nix ({ pkgs, lib, ...} : with lib; {
         enableImap = true;
       };
 
+      systemd.services.gitlab-backup.environment.BACKUP = "dump";
+
       services.gitlab = {
         enable = true;
         databasePasswordFile = pkgs.writeText "dbPassword" "xo0daiF4";
@@ -64,7 +66,7 @@ import ./make-test-python.nix ({ pkgs, lib, ...} : with lib; {
     };
   };
 
-  testScript =
+  testScript = { nodes, ... }:
     let
       auth = pkgs.writeText "auth.json" (builtins.toJSON {
         grant_type = "password";
@@ -83,19 +85,22 @@ import ./make-test-python.nix ({ pkgs, lib, ...} : with lib; {
         content = "some content";
         commit_message = "create a new file";
       });
-  in
-  ''
-    gitlab.start()
 
+      # Wait for all GitLab services to be fully started.
+      waitForServices = ''
         gitlab.wait_for_unit("gitaly.service")
         gitlab.wait_for_unit("gitlab-workhorse.service")
         gitlab.wait_for_unit("gitlab-pages.service")
         gitlab.wait_for_unit("gitlab-mailroom.service")
         gitlab.wait_for_unit("gitlab.service")
         gitlab.wait_for_unit("gitlab-sidekiq.service")
-    gitlab.wait_for_file("/var/gitlab/state/tmp/sockets/gitlab.socket")
+        gitlab.wait_for_file("${nodes.gitlab.config.services.gitlab.statePath}/tmp/sockets/gitlab.socket")
         gitlab.wait_until_succeeds("curl -sSf http://gitlab/users/sign_in")
+      '';
 
+      # The actual test of GitLab. Only push data to GitLab if
+      # `doSetup` is is true.
+      test = doSetup: ''
         gitlab.succeed(
             "curl -isSf http://gitlab | grep -i location | grep -q http://gitlab/users/sign_in"
         )
@@ -105,12 +110,14 @@ import ./make-test-python.nix ({ pkgs, lib, ...} : with lib; {
         gitlab.succeed(
             "echo \"Authorization: Bearer \$(curl -X POST -H 'Content-Type: application/json' -d @${auth} http://gitlab/oauth/token | ${pkgs.jq}/bin/jq -r '.access_token')\" >/tmp/headers"
         )
+      '' + optionalString doSetup ''
         gitlab.succeed(
             "curl -X POST -H 'Content-Type: application/json' -H @/tmp/headers -d @${createProject} http://gitlab/api/v4/projects"
         )
         gitlab.succeed(
             "curl -X POST -H 'Content-Type: application/json' -H @/tmp/headers -d @${putFile} http://gitlab/api/v4/projects/1/repository/files/some-file.txt"
         )
+      '' + ''
         gitlab.succeed(
             "curl -H @/tmp/headers http://gitlab/api/v4/projects/1/repository/archive.tar.gz > /tmp/archive.tar.gz"
         )
@@ -120,4 +127,28 @@ import ./make-test-python.nix ({ pkgs, lib, ...} : with lib; {
         gitlab.succeed("test -s /tmp/archive.tar.gz")
         gitlab.succeed("test -s /tmp/archive.tar.bz2")
       '';
+
+  in ''
+      gitlab.start()
+    ''
+    + waitForServices
+    + test true
+    + ''
+      gitlab.systemctl("start gitlab-backup.service")
+      gitlab.wait_for_unit("gitlab-backup.service")
+      gitlab.wait_for_file("${nodes.gitlab.config.services.gitlab.statePath}/backup/dump_gitlab_backup.tar")
+      gitlab.systemctl("stop postgresql.service gitlab.target")
+      gitlab.succeed(
+          "find ${nodes.gitlab.config.services.gitlab.statePath} -mindepth 1 -maxdepth 1 -not -name backup -execdir rm -r {} +"
+      )
+      gitlab.succeed("systemd-tmpfiles --create")
+      gitlab.succeed("rm -rf ${nodes.gitlab.config.services.postgresql.dataDir}")
+      gitlab.systemctl("start gitlab-config.service gitlab-postgresql.service")
+      gitlab.succeed(
+          "sudo -u gitlab -H gitlab-rake gitlab:backup:restore RAILS_ENV=production BACKUP=dump force=yes"
+      )
+      gitlab.systemctl("start gitlab.target")
+    ''
+    + waitForServices
+    + test false;
 })