From 3fb04ca505b19831c78d84b63213dc208cda997c Mon Sep 17 00:00:00 2001 From: talyz Date: Tue, 23 Feb 2021 18:51:48 +0100 Subject: [PATCH 1/4] nixos/gitlab: Introduce automatic backup support Add support for automatically backing up GitLab state, both locally and to remote locations. --- nixos/modules/services/misc/gitlab.nix | 121 ++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 4 deletions(-) diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix index c9dd10ec557..0af57bd4e89 100644 --- a/nixos/modules/services/misc/gitlab.nix +++ b/nixos/modules/services/misc/gitlab.nix @@ -116,7 +116,11 @@ let omniauth.enabled = false; shared.path = "${cfg.statePath}/shared"; gitaly.client_path = "${cfg.packages.gitaly}/bin"; - backup.path = "${cfg.backupPath}"; + backup = { + path = cfg.backup.path; + keep_time = cfg.backup.keepTime; + upload = cfg.backup.uploadOptions; + }; gitlab_shell = { path = "${cfg.packages.gitlab-shell}"; hooks_path = "${cfg.statePath}/shell/hooks"; @@ -207,6 +211,7 @@ in { imports = [ (mkRenamedOptionModule [ "services" "gitlab" "stateDir" ] [ "services" "gitlab" "statePath" ]) + (mkRenamedOptionModule [ "services" "gitlab" "backupPath" ] [ "services" "gitlab" "backup" "path" ]) (mkRemovedOptionModule [ "services" "gitlab" "satelliteDir" ] "") ]; @@ -270,10 +275,101 @@ in { ''; }; - backupPath = mkOption { + backup.startAt = mkOption { + type = with types; either str (listOf str); + default = []; + example = "03:00"; + description = '' + The time(s) to run automatic backup of GitLab + state. Specified in systemd's time format; see + systemd.time + 7. + ''; + }; + + backup.path = mkOption { type = types.str; default = cfg.statePath + "/backup"; - description = "Gitlab path for backups."; + description = "GitLab path for backups."; + }; + + backup.keepTime = mkOption { + type = types.int; + default = 0; + example = 48; + apply = x: x * 60 * 60; + description = '' + How long to keep the backups around, in + hours. 0 means keep + forever. + ''; + }; + + backup.skip = mkOption { + type = with types; + let value = enum [ + "db" + "uploads" + "builds" + "artifacts" + "lfs" + "registry" + "pages" + "repositories" + "tar" + ]; + in + either value (listOf value); + default = []; + example = [ "artifacts" "lfs" ]; + apply = x: if isString x then x else concatStringsSep "," x; + description = '' + Directories to exclude from the backup. The example excludes + CI artifacts and LFS objects from the backups. The + tar option skips the creation of a tar + file. + + Refer to + for more information. + ''; + }; + + backup.uploadOptions = mkOption { + type = types.attrs; + default = {}; + example = literalExample '' + { + # Fog storage connection settings, see http://fog.io/storage/ + connection = { + provider = "AWS"; + region = "eu-north-1"; + aws_access_key_id = "AKIAXXXXXXXXXXXXXXXX"; + aws_secret_access_key = { _secret = config.deployment.keys.aws_access_key.path; }; + }; + + # The remote 'directory' to store your backups in. + # For S3, this would be the bucket name. + remote_directory = "my-gitlab-backups"; + + # Use multipart uploads when file size reaches 100MB, see + # http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html + multipart_chunk_size = 104857600; + + # Turns on AWS Server-Side Encryption with Amazon S3-Managed Keys for backups, this is optional + encryption = "AES256"; + + # Specifies Amazon S3 storage class to use for backups, this is optional + storage_class = "STANDARD"; + }; + ''; + description = '' + GitLab automatic upload specification. Tells GitLab to + upload the backup to a remote location when done. + + Attributes specified here are added under + production -> backup -> upload in + config/gitlab.yml. + ''; }; databaseHost = mkOption { @@ -720,7 +816,7 @@ in { "d /run/gitlab 0755 ${cfg.user} ${cfg.group} -" "d ${gitlabEnv.HOME} 0750 ${cfg.user} ${cfg.group} -" "z ${gitlabEnv.HOME}/.ssh/authorized_keys 0600 ${cfg.user} ${cfg.group} -" - "d ${cfg.backupPath} 0750 ${cfg.user} ${cfg.group} -" + "d ${cfg.backup.path} 0750 ${cfg.user} ${cfg.group} -" "d ${cfg.statePath} 0750 ${cfg.user} ${cfg.group} -" "d ${cfg.statePath}/builds 0750 ${cfg.user} ${cfg.group} -" "d ${cfg.statePath}/config 0750 ${cfg.user} ${cfg.group} -" @@ -1053,6 +1149,23 @@ in { }; + systemd.services.gitlab-backup = { + after = [ "gitlab.service" ]; + bindsTo = [ "gitlab.service" ]; + startAt = cfg.backup.startAt; + environment = { + RAILS_ENV = "production"; + CRON = "1"; + } // optionalAttrs (stringLength cfg.backup.skip > 0) { + SKIP = cfg.backup.skip; + }; + serviceConfig = { + User = cfg.user; + Group = cfg.group; + ExecStart = "${gitlab-rake}/bin/gitlab-rake gitlab:backup:create"; + }; + }; + }; meta.doc = ./gitlab.xml; From 35582c5af7be083e0e7aaacccec5642ecccf83fb Mon Sep 17 00:00:00 2001 From: talyz Date: Thu, 25 Feb 2021 11:40:47 +0100 Subject: [PATCH 2/4] gitlab.tests: Test backup and restore Test the automatic backup and restore functionality by backing up the instance after running the initial tests, stopping GitLab and removing all state, running the restore rake task, then running the tests again, but without pushing data. --- nixos/tests/gitlab.nix | 135 +++++++++++++++++++++++++---------------- 1 file changed, 83 insertions(+), 52 deletions(-) diff --git a/nixos/tests/gitlab.nix b/nixos/tests/gitlab.nix index baad675229c..582f5faf9bc 100644 --- a/nixos/tests/gitlab.nix +++ b/nixos/tests/gitlab.nix @@ -34,6 +34,8 @@ import ./make-test-python.nix ({ pkgs, lib, ...} : with lib; { enableImap = true; }; + systemd.services.gitlab-backup.environment.BACKUP = "dump"; + services.gitlab = { enable = true; databasePasswordFile = pkgs.writeText "dbPassword" "xo0daiF4"; @@ -64,60 +66,89 @@ import ./make-test-python.nix ({ pkgs, lib, ...} : with lib; { }; }; - testScript = - let - auth = pkgs.writeText "auth.json" (builtins.toJSON { - grant_type = "password"; - username = "root"; - password = initialRootPassword; - }); + testScript = { nodes, ... }: + let + auth = pkgs.writeText "auth.json" (builtins.toJSON { + grant_type = "password"; + username = "root"; + password = initialRootPassword; + }); - createProject = pkgs.writeText "create-project.json" (builtins.toJSON { - name = "test"; - }); + createProject = pkgs.writeText "create-project.json" (builtins.toJSON { + name = "test"; + }); - putFile = pkgs.writeText "put-file.json" (builtins.toJSON { - branch = "master"; - author_email = "author@example.com"; - author_name = "Firstname Lastname"; - content = "some content"; - commit_message = "create a new file"; - }); - in - '' - gitlab.start() + putFile = pkgs.writeText "put-file.json" (builtins.toJSON { + branch = "master"; + author_email = "author@example.com"; + author_name = "Firstname Lastname"; + content = "some content"; + commit_message = "create a new file"; + }); - gitlab.wait_for_unit("gitaly.service") - gitlab.wait_for_unit("gitlab-workhorse.service") - gitlab.wait_for_unit("gitlab-pages.service") - gitlab.wait_for_unit("gitlab-mailroom.service") - gitlab.wait_for_unit("gitlab.service") - gitlab.wait_for_unit("gitlab-sidekiq.service") - gitlab.wait_for_file("/var/gitlab/state/tmp/sockets/gitlab.socket") - gitlab.wait_until_succeeds("curl -sSf http://gitlab/users/sign_in") + # Wait for all GitLab services to be fully started. + waitForServices = '' + gitlab.wait_for_unit("gitaly.service") + gitlab.wait_for_unit("gitlab-workhorse.service") + gitlab.wait_for_unit("gitlab-pages.service") + gitlab.wait_for_unit("gitlab-mailroom.service") + gitlab.wait_for_unit("gitlab.service") + gitlab.wait_for_unit("gitlab-sidekiq.service") + gitlab.wait_for_file("${nodes.gitlab.config.services.gitlab.statePath}/tmp/sockets/gitlab.socket") + gitlab.wait_until_succeeds("curl -sSf http://gitlab/users/sign_in") + ''; - gitlab.succeed( - "curl -isSf http://gitlab | grep -i location | grep -q http://gitlab/users/sign_in" - ) - gitlab.succeed( - "${pkgs.sudo}/bin/sudo -u gitlab -H gitlab-rake gitlab:check 1>&2" - ) - gitlab.succeed( - "echo \"Authorization: Bearer \$(curl -X POST -H 'Content-Type: application/json' -d @${auth} http://gitlab/oauth/token | ${pkgs.jq}/bin/jq -r '.access_token')\" >/tmp/headers" - ) - gitlab.succeed( - "curl -X POST -H 'Content-Type: application/json' -H @/tmp/headers -d @${createProject} http://gitlab/api/v4/projects" - ) - gitlab.succeed( - "curl -X POST -H 'Content-Type: application/json' -H @/tmp/headers -d @${putFile} http://gitlab/api/v4/projects/1/repository/files/some-file.txt" - ) - gitlab.succeed( - "curl -H @/tmp/headers http://gitlab/api/v4/projects/1/repository/archive.tar.gz > /tmp/archive.tar.gz" - ) - gitlab.succeed( - "curl -H @/tmp/headers http://gitlab/api/v4/projects/1/repository/archive.tar.bz2 > /tmp/archive.tar.bz2" - ) - gitlab.succeed("test -s /tmp/archive.tar.gz") - gitlab.succeed("test -s /tmp/archive.tar.bz2") - ''; + # The actual test of GitLab. Only push data to GitLab if + # `doSetup` is is true. + test = doSetup: '' + gitlab.succeed( + "curl -isSf http://gitlab | grep -i location | grep -q http://gitlab/users/sign_in" + ) + gitlab.succeed( + "${pkgs.sudo}/bin/sudo -u gitlab -H gitlab-rake gitlab:check 1>&2" + ) + gitlab.succeed( + "echo \"Authorization: Bearer \$(curl -X POST -H 'Content-Type: application/json' -d @${auth} http://gitlab/oauth/token | ${pkgs.jq}/bin/jq -r '.access_token')\" >/tmp/headers" + ) + '' + optionalString doSetup '' + gitlab.succeed( + "curl -X POST -H 'Content-Type: application/json' -H @/tmp/headers -d @${createProject} http://gitlab/api/v4/projects" + ) + gitlab.succeed( + "curl -X POST -H 'Content-Type: application/json' -H @/tmp/headers -d @${putFile} http://gitlab/api/v4/projects/1/repository/files/some-file.txt" + ) + '' + '' + gitlab.succeed( + "curl -H @/tmp/headers http://gitlab/api/v4/projects/1/repository/archive.tar.gz > /tmp/archive.tar.gz" + ) + gitlab.succeed( + "curl -H @/tmp/headers http://gitlab/api/v4/projects/1/repository/archive.tar.bz2 > /tmp/archive.tar.bz2" + ) + gitlab.succeed("test -s /tmp/archive.tar.gz") + gitlab.succeed("test -s /tmp/archive.tar.bz2") + ''; + + in '' + gitlab.start() + '' + + waitForServices + + test true + + '' + gitlab.systemctl("start gitlab-backup.service") + gitlab.wait_for_unit("gitlab-backup.service") + gitlab.wait_for_file("${nodes.gitlab.config.services.gitlab.statePath}/backup/dump_gitlab_backup.tar") + gitlab.systemctl("stop postgresql.service gitlab.target") + gitlab.succeed( + "find ${nodes.gitlab.config.services.gitlab.statePath} -mindepth 1 -maxdepth 1 -not -name backup -execdir rm -r {} +" + ) + gitlab.succeed("systemd-tmpfiles --create") + gitlab.succeed("rm -rf ${nodes.gitlab.config.services.postgresql.dataDir}") + gitlab.systemctl("start gitlab-config.service gitlab-postgresql.service") + gitlab.succeed( + "sudo -u gitlab -H gitlab-rake gitlab:backup:restore RAILS_ENV=production BACKUP=dump force=yes" + ) + gitlab.systemctl("start gitlab.target") + '' + + waitForServices + + test false; }) From 7b5cbde81ff5d59541deb64e12557ce964f65299 Mon Sep 17 00:00:00 2001 From: talyz Date: Thu, 25 Feb 2021 12:02:55 +0100 Subject: [PATCH 3/4] nixos/gitlab: Gitlab -> GitLab --- nixos/modules/services/misc/gitlab.nix | 22 +++++++++++----------- nixos/modules/services/misc/gitlab.xml | 22 +++++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix index 0af57bd4e89..b8bb4059dcc 100644 --- a/nixos/modules/services/misc/gitlab.nix +++ b/nixos/modules/services/misc/gitlab.nix @@ -265,7 +265,7 @@ in { type = types.str; default = "/var/gitlab/state"; description = '' - Gitlab state directory. Configuration, repositories and + GitLab state directory. Configuration, repositories and logs, among other things, are stored here. The directory will be created automatically if it doesn't @@ -376,7 +376,7 @@ in { type = types.str; default = ""; description = '' - Gitlab database hostname. An empty string means use + GitLab database hostname. An empty string means use local unix socket connection. ''; }; @@ -385,7 +385,7 @@ in { type = with types; nullOr path; default = null; description = '' - File containing the Gitlab database user password. + File containing the GitLab database user password. This should be a string, not a nix path, since nix paths are copied into the world-readable nix store. @@ -406,13 +406,13 @@ in { databaseName = mkOption { type = types.str; default = "gitlab"; - description = "Gitlab database name."; + description = "GitLab database name."; }; databaseUsername = mkOption { type = types.str; default = "gitlab"; - description = "Gitlab database user."; + description = "GitLab database user."; }; databasePool = mkOption { @@ -456,14 +456,14 @@ in { host = mkOption { type = types.str; default = config.networking.hostName; - description = "Gitlab host name. Used e.g. for copy-paste URLs."; + description = "GitLab host name. Used e.g. for copy-paste URLs."; }; port = mkOption { type = types.int; default = 8080; description = '' - Gitlab server port for copy-paste URLs, e.g. 80 or 443 if you're + GitLab server port for copy-paste URLs, e.g. 80 or 443 if you're service over https. ''; }; @@ -516,26 +516,26 @@ in { address = mkOption { type = types.str; default = "localhost"; - description = "Address of the SMTP server for Gitlab."; + description = "Address of the SMTP server for GitLab."; }; port = mkOption { type = types.int; default = 25; - description = "Port of the SMTP server for Gitlab."; + description = "Port of the SMTP server for GitLab."; }; username = mkOption { type = with types; nullOr str; default = null; - description = "Username of the SMTP server for Gitlab."; + description = "Username of the SMTP server for GitLab."; }; passwordFile = mkOption { type = types.nullOr types.path; default = null; description = '' - File containing the password of the SMTP server for Gitlab. + File containing the password of the SMTP server for GitLab. This should be a string, not a nix path, since nix paths are copied into the world-readable nix store. diff --git a/nixos/modules/services/misc/gitlab.xml b/nixos/modules/services/misc/gitlab.xml index 19a3df0a5f6..8ddc54794b2 100644 --- a/nixos/modules/services/misc/gitlab.xml +++ b/nixos/modules/services/misc/gitlab.xml @@ -3,15 +3,15 @@ xmlns:xi="http://www.w3.org/2001/XInclude" version="5.0" xml:id="module-services-gitlab"> - Gitlab + GitLab - Gitlab is a feature-rich git hosting service. + GitLab is a feature-rich git hosting service.
Prerequisites - The gitlab service exposes only an Unix socket at + The gitlab service exposes only an Unix socket at /run/gitlab/gitlab-workhorse.socket. You need to configure a webserver to proxy HTTP requests to the socket. @@ -39,7 +39,7 @@ Configuring - Gitlab depends on both PostgreSQL and Redis and will automatically enable + GitLab depends on both PostgreSQL and Redis and will automatically enable both services. In the case of PostgreSQL, a database and a role will be created. @@ -85,20 +85,20 @@ services.gitlab = { - If you're setting up a new Gitlab instance, generate new + If you're setting up a new GitLab instance, generate new secrets. You for instance use tr -dc A-Za-z0-9 < /dev/urandom | head -c 128 > /var/keys/gitlab/db to generate a new db secret. Make sure the files can be read by, and only by, the user specified by services.gitlab.user. Gitlab + linkend="opt-services.gitlab.user">services.gitlab.user. GitLab encrypts sensitive data stored in the database. If you're restoring - an existing Gitlab instance, you must specify the secrets secret - from config/secrets.yml located in your Gitlab + an existing GitLab instance, you must specify the secrets secret + from config/secrets.yml located in your GitLab state folder. - When icoming_mail.enabled is set to true + When incoming_mail.enabled is set to true in extraConfig an additional service called gitlab-mailroom is enabled for fetching incoming mail. @@ -113,13 +113,13 @@ services.gitlab = { Maintenance - You can run Gitlab's rake tasks with gitlab-rake which + You can run GitLab's rake tasks with gitlab-rake which will be available on the system when gitlab is enabled. You will have to run the command as the user that you configured to run gitlab with. - For example, to backup a Gitlab instance: + For example, to backup a GitLab instance: $ sudo -u git -H gitlab-rake gitlab:backup:create From abba76a3b96c75872010e31c216b84cac7fe3a6e Mon Sep 17 00:00:00 2001 From: talyz Date: Thu, 25 Feb 2021 13:48:33 +0100 Subject: [PATCH 4/4] nixos/gitlab: Document automatic backups --- nixos/doc/manual/release-notes/rl-2105.xml | 8 +++++ nixos/modules/services/misc/gitlab.xml | 39 ++++++++++++++++------ 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/nixos/doc/manual/release-notes/rl-2105.xml b/nixos/doc/manual/release-notes/rl-2105.xml index b7947293c01..73deab7f539 100644 --- a/nixos/doc/manual/release-notes/rl-2105.xml +++ b/nixos/doc/manual/release-notes/rl-2105.xml @@ -883,6 +883,14 @@ environment.systemPackages = [ Please test your setup and container images with containerd prior to upgrading. + + + The GitLab module now has support for automatic backups. A + schedule can be set with the + services.gitlab.backup.startAt + option. + +
diff --git a/nixos/modules/services/misc/gitlab.xml b/nixos/modules/services/misc/gitlab.xml index 8ddc54794b2..40424c5039a 100644 --- a/nixos/modules/services/misc/gitlab.xml +++ b/nixos/modules/services/misc/gitlab.xml @@ -112,21 +112,40 @@ services.gitlab = {
Maintenance - - You can run GitLab's rake tasks with gitlab-rake which - will be available on the system when gitlab is enabled. You will have to run - the command as the user that you configured to run gitlab with. - +
+ Backups + + Backups can be configured with the options in services.gitlab.backup. Use + the services.gitlab.backup.startAt + option to configure regular backups. + - - For example, to backup a GitLab instance: + + To run a manual backup, start the gitlab-backup service: -$ sudo -u git -H gitlab-rake gitlab:backup:create +$ systemctl start gitlab-backup.service - A list of all availabe rake tasks can be obtained by running: + +
+ +
+ Rake tasks + + + You can run GitLab's rake tasks with gitlab-rake + which will be available on the system when GitLab is enabled. You + will have to run the command as the user that you configured to run + GitLab with. + + + + A list of all availabe rake tasks can be obtained by running: $ sudo -u git -H gitlab-rake -T - + +