gitlab: 8.0.5 -> 8.5.0, service improvements

Updates gitlab to the current stable version and fixes a lot of features that
were broken, at least with the current version and our configuration.

Quite a lot of sweat and tears has gone into testing nearly all features and
reading/patching the Gitlab source as we're about to deploy gitlab for our
whole company.

Things to note:

 * The gitlab config is now written as a nix attribute set and will be
   converted to JSON. Gitlab uses YAML but JSON is a subset of YAML.
   The `extraConfig` opition is also an attribute set that will be merged
   with the default config. This way *all* Gitlab options are supported.

 * Some paths like uploads and configs are hardcoded in rails  (at least
   after my study of the Gitlab source). This is why they are linked from
   the Gitlab root to /run/gitlab and then linked to the  configurable
   `statePath`.

 * Backup & restore should work out of the box from another Gitlab instance.

 * gitlab-git-http-server has been replaced by gitlab-workhorse upstream.
   Push & pull over HTTPS works perfectly. Communication to gitlab is done
   over unix sockets. An HTTP server is required to proxy requests to
   gitlab-workhorse over another unix socket at
   `/run/gitlab/gitlab-workhorse.socket`.

 * The user & group running gitlab are now configurable. These can even be
   changed for live instances.

 * The initial email address & password of the root user can be configured.

Fixes #8598.
This commit is contained in:
Franz Pletz
2016-01-30 14:47:04 +01:00
parent 30891166be
commit bcfa59bf82
14 changed files with 3741 additions and 3578 deletions

View File

@@ -28,6 +28,9 @@ with lib;
(mkRenamedOptionModule [ "services" "subsonic" "host" ] [ "services" "subsonic" "listenAddress" ])
(mkRenamedOptionModule [ "jobs" ] [ "systemd" "services" ])
(mkRenamedOptionModule [ "services" "gitlab" "stateDir" ] [ "services" "gitlab" "statePath" ])
(mkRemovedOptionModule [ "services" "gitlab" "satelliteDir" ])
# Old Grub-related options.
(mkRenamedOptionModule [ "boot" "initrd" "extraKernelModules" ] [ "boot" "initrd" "kernelModules" ])
(mkRenamedOptionModule [ "boot" "extraKernelParams" ] [ "boot" "kernelParams" ])

View File

@@ -21,14 +21,15 @@ let
username: ${cfg.databaseUsername}
encoding: utf8
'';
gitlabShellYml = ''
user: gitlab
gitlab_url: "http://${cfg.host}:${toString cfg.port}/"
user: ${cfg.user}
gitlab_url: "http://localhost:8080/"
http_settings:
self_signed_cert: false
repos_path: "${cfg.stateDir}/repositories"
secret_file: "${cfg.stateDir}/config/gitlab_shell_secret"
log_file: "${cfg.stateDir}/log/gitlab-shell.log"
repos_path: "${cfg.statePath}/repositories"
secret_file: "${cfg.statePath}/config/gitlab_shell_secret"
log_file: "${cfg.statePath}/log/gitlab-shell.log"
redis:
bin: ${pkgs.redis}/bin/redis-cli
host: 127.0.0.1
@@ -37,33 +38,101 @@ let
namespace: resque:gitlab
'';
gitlabConfig = {
production = flip recursiveUpdate cfg.extraConfig {
gitlab = {
host = cfg.host;
port = cfg.port;
https = cfg.https;
user = cfg.user;
email_enabled = true;
email_display_name = "GitLab";
email_reply_to = "noreply@localhost";
default_theme = 2;
default_projects_features = {
issues = true;
merge_requests = true;
wiki = false;
snippets = false;
builds = true;
};
};
artifacts = {
enabled = true;
};
lfs = {
enabled = true;
};
gravatar = {
enabled = true;
};
cron_jobs = {
stuck_ci_builds_worker = {
cron = "0 0 * * *";
};
};
gitlab_ci = {
builds_path = "${cfg.statePath}/builds";
};
ldap = {
enabled = false;
};
omniauth = {
enabled = false;
};
shared = {
path = "${cfg.statePath}/shared";
};
backup = {
path = "${cfg.backupPath}";
};
gitlab_shell = {
path = "${pkgs.gitlab-shell}";
repos_path = "${cfg.statePath}/repositories";
hooks_path = "${cfg.statePath}/shell/hooks";
secret_file = "${cfg.statePath}/config/gitlab_shell_secret";
upload_pack = true;
receive_pack = true;
};
git = {
bin_path = "git";
max_size = 20971520; # 20MB
timeout = 10;
};
extra = {};
};
};
gitlabEnv = {
HOME = "${cfg.statePath}/home";
GEM_HOME = gemHome;
BUNDLE_GEMFILE = "${pkgs.gitlab}/share/gitlab/Gemfile";
UNICORN_PATH = "${cfg.statePath}/";
GITLAB_PATH = "${pkgs.gitlab}/share/gitlab/";
GITLAB_STATE_PATH = "${cfg.statePath}";
GITLAB_UPLOADS_PATH = "${cfg.statePath}/uploads";
GITLAB_LOG_PATH = "${cfg.statePath}/log";
GITLAB_SHELL_PATH = "${pkgs.gitlab-shell}";
GITLAB_SHELL_CONFIG_PATH = "${cfg.statePath}/shell/config.yml";
GITLAB_SHELL_SECRET_PATH = "${cfg.statePath}/config/gitlab_shell_secret";
GITLAB_SHELL_HOOKS_PATH = "${cfg.statePath}/shell/hooks";
RAILS_ENV = "production";
};
unicornConfig = builtins.readFile ./defaultUnicornConfig.rb;
gitlab-runner = pkgs.stdenv.mkDerivation rec {
name = "gitlab-runner";
buildInputs = [ pkgs.gitlab pkgs.bundler pkgs.makeWrapper ];
buildInputs = with pkgs; [ gitlab bundler makeWrapper ];
phases = "installPhase fixupPhase";
buildPhase = "";
installPhase = ''
mkdir -p $out/bin
makeWrapper ${bundler}/bin/bundle $out/bin/gitlab-runner\
--set RAKEOPT '"-f ${pkgs.gitlab}/share/gitlab/Rakefile"'\
--set GEM_HOME '${gemHome}'\
--set UNICORN_PATH "${cfg.stateDir}/"\
--set GITLAB_PATH "${pkgs.gitlab}/share/gitlab/"\
--set GITLAB_APPLICATION_LOG_PATH "${cfg.stateDir}/log/application.log"\
--set GITLAB_SATELLITES_PATH "${cfg.stateDir}/satellites"\
--set GITLAB_SHELL_PATH "${pkgs.gitlab-shell}"\
--set GITLAB_REPOSITORIES_PATH "${cfg.stateDir}/repositories"\
--set GITLAB_SHELL_HOOKS_PATH "${cfg.stateDir}/shell/hooks"\
--set BUNDLE_GEMFILE "${pkgs.gitlab}/share/gitlab/Gemfile"\
--set GITLAB_EMAIL_FROM "${cfg.emailFrom}"\
--set GITLAB_SHELL_CONFIG_PATH "${cfg.stateDir}/shell/config.yml"\
--set GITLAB_SHELL_SECRET_PATH "${cfg.stateDir}/config/gitlab_shell_secret"\
--set GITLAB_HOST "${cfg.host}"\
--set GITLAB_PORT "${toString cfg.port}"\
--set GITLAB_BACKUP_PATH "${cfg.backupPath}"\
--set RAILS_ENV "production"
makeWrapper ${bundler}/bin/bundle $out/bin/gitlab-runner \
${concatStrings (mapAttrsToList (name: value: "--set ${name} '\"${value}\"' ") gitlabEnv)} \
--set GITLAB_CONFIG_PATH '"${cfg.statePath}/config"' \
--set PATH '"${pkgs.nodejs}/bin:${pkgs.gzip}/bin:${config.services.postgresql.package}/bin:$PATH"' \
--set RAKEOPT '"-f ${pkgs.gitlab}/share/gitlab/Rakefile"'
'';
};
@@ -79,13 +148,7 @@ in {
'';
};
satelliteDir = mkOption {
type = types.str;
default = "/var/gitlab/git-satellites";
description = "Gitlab directory to store checked out git trees requires for operation.";
};
stateDir = mkOption {
statePath = mkOption {
type = types.str;
default = "/var/gitlab/state";
description = "Gitlab state directory, logs are stored here.";
@@ -93,7 +156,7 @@ in {
backupPath = mkOption {
type = types.str;
default = cfg.stateDir + "/backup";
default = cfg.statePath + "/backup";
description = "Gitlab path for backups.";
};
@@ -136,7 +199,60 @@ in {
port = mkOption {
type = types.int;
default = 8080;
description = "Gitlab server listening port.";
description = ''
Gitlab server port for copy-paste URLs, e.g. 80 or 443 if you're
service over https.
'';
};
https = mkOption {
type = types.bool;
default = false;
description = "Whether gitlab prints URLs with https as scheme.";
};
user = mkOption {
type = types.str;
default = "gitlab";
description = "User to run gitlab and all related services.";
};
group = mkOption {
type = types.str;
default = "gitlab";
description = "Group to run gitlab and all related services.";
};
initialRootEmail = mkOption {
type = types.str;
default = "admin@local.host";
description = ''
Initial email address of the root account if this is a new install.
'';
};
initialRootPassword = mkOption {
type = types.str;
default = "UseNixOS!";
description = ''
Initial password of the root account if this is a new install.
'';
};
extraConfig = mkOption {
type = types.attrs;
default = "";
example = {
gitlab = {
default_projects_features = {
builds = false;
};
};
};
description = ''
Extra options to be merged into config/gitlab.yml as nix
attribute set.
'';
};
};
};
@@ -159,39 +275,24 @@ in {
services.postfix.enable = mkDefault true;
users.extraUsers = [
{ name = "gitlab";
group = "gitlab";
home = "${cfg.stateDir}/home";
{ name = cfg.user;
group = cfg.group;
home = "${cfg.statePath}/home";
shell = "${pkgs.bash}/bin/bash";
uid = config.ids.uids.gitlab;
} ];
}
];
users.extraGroups = [
{ name = "gitlab";
{ name = cfg.group;
gid = config.ids.gids.gitlab;
} ];
}
];
systemd.services.gitlab-sidekiq = {
after = [ "network.target" "redis.service" ];
wantedBy = [ "multi-user.target" ];
environment.HOME = "${cfg.stateDir}/home";
environment.GEM_HOME = gemHome;
environment.UNICORN_PATH = "${cfg.stateDir}/";
environment.GITLAB_PATH = "${pkgs.gitlab}/share/gitlab/";
environment.GITLAB_APPLICATION_LOG_PATH = "${cfg.stateDir}/log/application.log";
environment.GITLAB_SATELLITES_PATH = "${cfg.stateDir}/satellites";
environment.GITLAB_SHELL_PATH = "${pkgs.gitlab-shell}";
environment.GITLAB_REPOSITORIES_PATH = "${cfg.stateDir}/repositories";
environment.GITLAB_SHELL_HOOKS_PATH = "${cfg.stateDir}/shell/hooks";
environment.BUNDLE_GEMFILE = "${pkgs.gitlab}/share/gitlab/Gemfile";
environment.GITLAB_EMAIL_FROM = "${cfg.emailFrom}";
environment.GITLAB_SHELL_CONFIG_PATH = "${cfg.stateDir}/shell/config.yml";
environment.GITLAB_SHELL_SECRET_PATH = "${cfg.stateDir}/config/gitlab_shell_secret";
environment.GITLAB_HOST = "${cfg.host}";
environment.GITLAB_PORT = "${toString cfg.port}";
environment.GITLAB_DATABASE_HOST = "${cfg.databaseHost}";
environment.GITLAB_DATABASE_PASSWORD = "${cfg.databasePassword}";
environment.RAILS_ENV = "production";
environment = gitlabEnv;
path = with pkgs; [
config.services.postgresql.package
gitAndTools.git
@@ -201,116 +302,132 @@ in {
];
serviceConfig = {
Type = "simple";
User = "gitlab";
Group = "gitlab";
User = cfg.user;
Group = cfg.group;
TimeoutSec = "300";
WorkingDirectory = "${pkgs.gitlab}/share/gitlab";
ExecStart="${bundler}/bin/bundle exec \"sidekiq -q post_receive -q mailer -q system_hook -q project_web_hook -q gitlab_shell -q common -q default -e production -P ${cfg.stateDir}/tmp/sidekiq.pid\"";
ExecStart="${bundler}/bin/bundle exec \"sidekiq -q post_receive -q mailer -q system_hook -q project_web_hook -q gitlab_shell -q common -q default -e production -P ${cfg.statePath}/tmp/sidekiq.pid\"";
};
};
systemd.services.gitlab-git-http-server = {
systemd.services.gitlab-workhorse = {
after = [ "network.target" "gitlab.service" ];
wantedBy = [ "multi-user.target" ];
environment.HOME = "${cfg.stateDir}/home";
environment.HOME = gitlabEnv.HOME;
environment.GITLAB_SHELL_CONFIG_PATH = gitlabEnv.GITLAB_SHELL_CONFIG_PATH;
path = with pkgs; [
gitAndTools.git
openssh
];
preStart = ''
mkdir -p /run/gitlab
chown ${cfg.user}:${cfg.group} /run/gitlab
'';
serviceConfig = {
PermissionsStartOnly = true; # preStart must be run as root
Type = "simple";
User = "gitlab";
Group = "gitlab";
User = cfg.user;
Group = cfg.group;
TimeoutSec = "300";
ExecStart = "${pkgs.gitlab-git-http-server}/bin/gitlab-git-http-server -listenUmask 0 -listenNetwork unix -listenAddr ${cfg.stateDir}/tmp/sockets/gitlab-git-http-server.socket -authBackend http://localhost:8080 ${cfg.stateDir}/repositories";
ExecStart =
"${pkgs.gitlab-workhorse}/bin/gitlab-workhorse "
+ "-listenUmask 0 "
+ "-listenNetwork unix "
+ "-listenAddr /run/gitlab/gitlab-workhorse.socket "
+ "-authSocket ${cfg.statePath}/tmp/sockets/gitlab.socket "
+ "-documentRoot ${pkgs.gitlab}/share/gitlab/public";
};
};
systemd.services.gitlab = {
after = [ "network.target" "postgresql.service" "redis.service" ];
wantedBy = [ "multi-user.target" ];
environment.HOME = "${cfg.stateDir}/home";
environment.GEM_HOME = gemHome;
environment.UNICORN_PATH = "${cfg.stateDir}/";
environment.GITLAB_PATH = "${pkgs.gitlab}/share/gitlab/";
environment.GITLAB_APPLICATION_LOG_PATH = "${cfg.stateDir}/log/application.log";
environment.GITLAB_SATELLITES_PATH = "${cfg.stateDir}/satellites";
environment.GITLAB_SHELL_PATH = "${pkgs.gitlab-shell}";
environment.GITLAB_SHELL_CONFIG_PATH = "${cfg.stateDir}/shell/config.yml";
environment.GITLAB_SHELL_SECRET_PATH = "${cfg.stateDir}/config/gitlab_shell_secret";
environment.GITLAB_REPOSITORIES_PATH = "${cfg.stateDir}/repositories";
environment.GITLAB_SHELL_HOOKS_PATH = "${cfg.stateDir}/shell/hooks";
environment.BUNDLE_GEMFILE = "${pkgs.gitlab}/share/gitlab/Gemfile";
environment.GITLAB_EMAIL_FROM = "${cfg.emailFrom}";
environment.GITLAB_HOST = "${cfg.host}";
environment.GITLAB_PORT = "${toString cfg.port}";
environment.GITLAB_DATABASE_HOST = "${cfg.databaseHost}";
environment.GITLAB_DATABASE_PASSWORD = "${cfg.databasePassword}";
environment.RAILS_ENV = "production";
environment = gitlabEnv;
path = with pkgs; [
config.services.postgresql.package
gitAndTools.git
ruby
openssh
nodejs
sudo
];
preStart = ''
# TODO: use env vars
mkdir -p ${cfg.stateDir}
mkdir -p ${cfg.stateDir}/log
mkdir -p ${cfg.stateDir}/satellites
mkdir -p ${cfg.stateDir}/repositories
mkdir -p ${cfg.stateDir}/shell/hooks
mkdir -p ${cfg.stateDir}/tmp/pids
mkdir -p ${cfg.stateDir}/tmp/sockets
rm -rf ${cfg.stateDir}/config
mkdir -p ${cfg.stateDir}/config
mkdir -p ${cfg.backupPath}
mkdir -p ${cfg.statePath}/builds
mkdir -p ${cfg.statePath}/repositories
mkdir -p ${gitlabConfig.production.shared.path}/artifacts
mkdir -p ${gitlabConfig.production.shared.path}/lfs-objects
mkdir -p ${cfg.statePath}/log
mkdir -p ${cfg.statePath}/shell
mkdir -p ${cfg.statePath}/tmp/pids
mkdir -p ${cfg.statePath}/tmp/sockets
rm -rf ${cfg.statePath}/config ${cfg.statePath}/shell/hooks
mkdir -p ${cfg.statePath}/config ${cfg.statePath}/shell
# TODO: What exactly is gitlab-shell doing with the secret?
tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 20 > ${cfg.stateDir}/config/gitlab_shell_secret
mkdir -p ${cfg.stateDir}/home/.ssh
touch ${cfg.stateDir}/home/.ssh/authorized_keys
tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 20 > ${cfg.statePath}/config/gitlab_shell_secret
cp -rf ${pkgs.gitlab}/share/gitlab/config ${cfg.stateDir}/
cp ${pkgs.gitlab}/share/gitlab/VERSION ${cfg.stateDir}/VERSION
# The uploads directory is hardcoded somewhere deep in rails. It is
# symlinked in the gitlab package to /run/gitlab/uploads to make it
# configurable
mkdir -p /run/gitlab
mkdir -p ${cfg.statePath}/uploads
ln -sf ${cfg.statePath}/uploads /run/gitlab/uploads
chown -R ${cfg.user}:${cfg.group} /run/gitlab
ln -fs ${pkgs.writeText "database.yml" databaseYml} ${cfg.stateDir}/config/database.yml
ln -fs ${pkgs.writeText "unicorn.rb" unicornConfig} ${cfg.stateDir}/config/unicorn.rb
# Prepare home directory
mkdir -p ${gitlabEnv.HOME}/.ssh
touch ${gitlabEnv.HOME}/.ssh/authorized_keys
chown -R ${cfg.user}:${cfg.group} ${gitlabEnv.HOME}/
chmod -R u+rwX,go-rwx+X ${gitlabEnv.HOME}/
chown -R gitlab:gitlab ${cfg.stateDir}/
chmod -R 755 ${cfg.stateDir}/
cp -rf ${pkgs.gitlab}/share/gitlab/config.dist/* ${cfg.statePath}/config
ln -sf ${cfg.statePath}/config /run/gitlab/config
cp ${pkgs.gitlab}/share/gitlab/VERSION ${cfg.statePath}/VERSION
# JSON is a subset of YAML
ln -fs ${pkgs.writeText "gitlab.yml" (builtins.toJSON gitlabConfig)} ${cfg.statePath}/config/gitlab.yml
ln -fs ${pkgs.writeText "database.yml" databaseYml} ${cfg.statePath}/config/database.yml
ln -fs ${pkgs.writeText "unicorn.rb" unicornConfig} ${cfg.statePath}/config/unicorn.rb
chown -R ${cfg.user}:${cfg.group} ${cfg.statePath}/
chmod -R ug+rwX,o-rwx+X ${cfg.statePath}/
# Install the shell required to push repositories
ln -fs ${pkgs.writeText "config.yml" gitlabShellYml} "$GITLAB_SHELL_CONFIG_PATH"
ln -fs ${pkgs.gitlab-shell}/hooks "$GITLAB_SHELL_HOOKS_PATH"
${pkgs.gitlab-shell}/bin/install
if [ "${cfg.databaseHost}" = "127.0.0.1" ]; then
if ! test -e "${cfg.stateDir}/db-created"; then
psql postgres -c "CREATE ROLE gitlab WITH LOGIN NOCREATEDB NOCREATEROLE NOCREATEUSER ENCRYPTED PASSWORD '${cfg.databasePassword}'"
if ! test -e "${cfg.statePath}/db-created"; then
psql postgres -c "CREATE ROLE gitlab WITH LOGIN CREATEDB NOCREATEROLE NOCREATEUSER ENCRYPTED PASSWORD '${cfg.databasePassword}'"
${config.services.postgresql.package}/bin/createdb --owner gitlab gitlab || true
touch "${cfg.stateDir}/db-created"
touch "${cfg.statePath}/db-created"
# force=yes disables the manual-interaction yes/no prompt
# which breaks without an stdin.
force=yes ${bundler}/bin/bundle exec rake -f ${pkgs.gitlab}/share/gitlab/Rakefile gitlab:setup RAILS_ENV=production
# The gitlab:setup task is horribly broken somehow, these two tasks will do the same for setting up the initial database
${gitlab-runner}/bin/gitlab-runner exec rake db:migrate RAILS_ENV=production
${gitlab-runner}/bin/gitlab-runner exec rake db:seed_fu RAILS_ENV=production \
GITLAB_ROOT_PASSWORD="${cfg.initialRootPassword}" GITLAB_ROOT_EMAIL="${cfg.initialRootEmail}";
fi
fi
${bundler}/bin/bundle exec rake -f ${pkgs.gitlab}/share/gitlab/Rakefile db:migrate RAILS_ENV=production
# Install the shell required to push repositories
ln -fs ${pkgs.writeText "config.yml" gitlabShellYml} ${cfg.stateDir}/shell/config.yml
export GITLAB_SHELL_CONFIG_PATH=""${cfg.stateDir}/shell/config.yml
${pkgs.gitlab-shell}/bin/install
# Always do the db migrations just to be sure the database is up-to-date
${gitlab-runner}/bin/gitlab-runner exec rake db:migrate RAILS_ENV=production
# Change permissions in the last step because some of the
# intermediary scripts like to create directories as root.
chown -R gitlab:gitlab ${cfg.stateDir}/
chmod -R 755 ${cfg.stateDir}/
# Change permissions in the last step because some of the
# intermediary scripts like to create directories as root.
chown -R ${cfg.user}:${cfg.group} ${cfg.statePath}
chmod -R u+rwX,go-rwx+X ${cfg.statePath}
'';
serviceConfig = {
PermissionsStartOnly = true; # preStart must be run as root
Type = "simple";
User = "gitlab";
Group = "gitlab";
User = cfg.user;
Group = cfg.group;
TimeoutSec = "300";
WorkingDirectory = "${pkgs.gitlab}/share/gitlab";
ExecStart="${bundler}/bin/bundle exec \"unicorn -c ${cfg.stateDir}/config/unicorn.rb -E production\"";
ExecStart="${bundler}/bin/bundle exec \"unicorn -c ${cfg.statePath}/config/unicorn.rb -E production\"";
};
};