hydra: 2020-02-06 -> 2020-03-{24,27}
Upgrades Hydra to the latest master/flake branch. To perform this upgrade, it's needed to do a non-trivial db-migration which provides a massive performance-improvement[1]. The basic ideas behind multi-step upgrades of services between NixOS versions have been gathered already[2]. For further context it's recommended to read this first. Basically, the following steps are needed: * Upgrade to a non-breaking version of Hydra with the db-changes (columns are still nullable here). If `system.stateVersion` is set to something older than 20.03, the package will be selected automatically, otherwise `pkgs.hydra-migration` needs to be used. * Run `hydra-backfill-ids` on the server. * Deploy either `pkgs.hydra-unstable` (for Hydra master) or `pkgs.hydra-flakes` (for flakes-support) to activate the optimization. The steps are also documented in the release-notes and in the module using `warnings`. `pkgs.hydra` has been removed as latest Hydra doesn't compile with `pkgs.nixStable` and to ensure a graceful migration using the newly introduced packages. To verify the approach, a simple vm-test has been added which verifies the migration steps. [1] https://github.com/NixOS/hydra/pull/711 [2] https://github.com/NixOS/nixpkgs/pull/82353#issuecomment-598269471
This commit is contained in:
Maximilian Bosch
135
pkgs/development/tools/misc/hydra/common.nix
Normal file
135
pkgs/development/tools/misc/hydra/common.nix
Normal file
@@ -0,0 +1,135 @@
|
||||
{ stdenv, nix, perlPackages, buildEnv, fetchFromGitHub
|
||||
, makeWrapper, autoconf, automake, libtool, unzip, pkgconfig, sqlite, libpqxx
|
||||
, gitAndTools, mercurial, darcs, subversion, bazaar, openssl, bzip2, libxslt
|
||||
, guile, perl, postgresql, nukeReferences, git, boehmgc, nlohmann_json
|
||||
, docbook_xsl, openssh, gnused, coreutils, findutils, gzip, lzma, gnutar
|
||||
, rpm, dpkg, cdrkit, pixz, lib, boost, autoreconfHook, src ? null, version ? null
|
||||
, migration ? false
|
||||
}:
|
||||
|
||||
with stdenv;
|
||||
|
||||
if lib.versions.major nix.version == "1"
|
||||
then throw "This Hydra version doesn't support Nix 1.x"
|
||||
else
|
||||
|
||||
let
|
||||
perlDeps = buildEnv {
|
||||
name = "hydra-perl-deps";
|
||||
paths = with perlPackages; lib.closePropagation
|
||||
[ ModulePluggable
|
||||
CatalystActionREST
|
||||
CatalystAuthenticationStoreDBIxClass
|
||||
CatalystDevel
|
||||
CatalystDispatchTypeRegex
|
||||
CatalystPluginAccessLog
|
||||
CatalystPluginAuthorizationRoles
|
||||
CatalystPluginCaptcha
|
||||
CatalystPluginSessionStateCookie
|
||||
CatalystPluginSessionStoreFastMmap
|
||||
CatalystPluginStackTrace
|
||||
CatalystPluginUnicodeEncoding
|
||||
CatalystTraitForRequestProxyBase
|
||||
CatalystViewDownload
|
||||
CatalystViewJSON
|
||||
CatalystViewTT
|
||||
CatalystXScriptServerStarman
|
||||
CatalystXRoleApplicator
|
||||
CryptRandPasswd
|
||||
DBDPg
|
||||
DBDSQLite
|
||||
DataDump
|
||||
DateTime
|
||||
DigestSHA1
|
||||
EmailMIME
|
||||
EmailSender
|
||||
FileSlurp
|
||||
IOCompress
|
||||
IPCRun
|
||||
JSON
|
||||
JSONAny
|
||||
JSONXS
|
||||
LWP
|
||||
LWPProtocolHttps
|
||||
NetAmazonS3
|
||||
NetPrometheus
|
||||
NetStatsd
|
||||
PadWalker
|
||||
Readonly
|
||||
SQLSplitStatement
|
||||
SetScalar
|
||||
Starman
|
||||
SysHostnameLong
|
||||
TermSizeAny
|
||||
TestMore
|
||||
TextDiff
|
||||
TextTable
|
||||
XMLSimple
|
||||
nix
|
||||
nix.perl-bindings
|
||||
git
|
||||
boehmgc
|
||||
];
|
||||
};
|
||||
in stdenv.mkDerivation rec {
|
||||
pname = "hydra";
|
||||
|
||||
inherit stdenv src version;
|
||||
|
||||
buildInputs =
|
||||
[ makeWrapper autoconf automake libtool unzip nukeReferences sqlite libpqxx
|
||||
gitAndTools.top-git mercurial /*darcs*/ subversion bazaar openssl bzip2 libxslt
|
||||
perlDeps perl nix
|
||||
postgresql # for running the tests
|
||||
nlohmann_json
|
||||
boost
|
||||
];
|
||||
|
||||
hydraPath = lib.makeBinPath (
|
||||
[ sqlite subversion openssh nix coreutils findutils pixz
|
||||
gzip bzip2 lzma gnutar unzip git gitAndTools.top-git mercurial /*darcs*/ gnused bazaar
|
||||
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] );
|
||||
|
||||
nativeBuildInputs = [ autoreconfHook pkgconfig ];
|
||||
|
||||
configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ];
|
||||
|
||||
NIX_CFLAGS_COMPILE = "-pthread";
|
||||
|
||||
shellHook = ''
|
||||
PATH=$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$(pwd)/src/hydra-evaluator:$PATH
|
||||
PERL5LIB=$(pwd)/src/lib:$PERL5LIB;
|
||||
'';
|
||||
|
||||
enableParallelBuilding = true;
|
||||
|
||||
preCheck = ''
|
||||
patchShebangs .
|
||||
export LOGNAME=''${LOGNAME:-foo}
|
||||
'';
|
||||
|
||||
postInstall = ''
|
||||
mkdir -p $out/nix-support
|
||||
for i in $out/bin/*; do
|
||||
read -n 4 chars < $i
|
||||
if [[ $chars =~ ELF ]]; then continue; fi
|
||||
wrapProgram $i \
|
||||
--prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
|
||||
--prefix PATH ':' $out/bin:$hydraPath \
|
||||
--set HYDRA_RELEASE ${version} \
|
||||
--set HYDRA_HOME $out/libexec/hydra \
|
||||
--set NIX_RELEASE ${nix.name or "unknown"}
|
||||
done
|
||||
''; # */
|
||||
|
||||
dontStrip = true;
|
||||
|
||||
passthru = { inherit perlDeps migration; };
|
||||
|
||||
meta = with stdenv.lib; {
|
||||
description = "Nix-based continuous build system";
|
||||
license = licenses.gpl3;
|
||||
platforms = platforms.linux;
|
||||
maintainers = with maintainers; [ ma27 ];
|
||||
};
|
||||
}
|
||||
@@ -1,143 +1,42 @@
|
||||
{ stdenv, nix, perlPackages, buildEnv, fetchFromGitHub
|
||||
, makeWrapper, autoconf, automake, libtool, unzip, pkgconfig, sqlite, libpqxx
|
||||
, gitAndTools, mercurial, darcs, subversion, bazaar, openssl, bzip2, libxslt
|
||||
, guile, perl, postgresql, nukeReferences, git, boehmgc, nlohmann_json
|
||||
, docbook_xsl, openssh, gnused, coreutils, findutils, gzip, lzma, gnutar
|
||||
, rpm, dpkg, cdrkit, pixz, lib, boost, autoreconfHook
|
||||
}:
|
||||
{ fetchFromGitHub, nixStable, nixUnstable, callPackage, nixFlakes }:
|
||||
|
||||
with stdenv;
|
||||
|
||||
if lib.versions.major nix.version == "1"
|
||||
then throw "This Hydra version doesn't support Nix 1.x"
|
||||
else
|
||||
|
||||
let
|
||||
perlDeps = buildEnv {
|
||||
name = "hydra-perl-deps";
|
||||
paths = with perlPackages; lib.closePropagation
|
||||
[ ModulePluggable
|
||||
CatalystActionREST
|
||||
CatalystAuthenticationStoreDBIxClass
|
||||
CatalystDevel
|
||||
CatalystDispatchTypeRegex
|
||||
CatalystPluginAccessLog
|
||||
CatalystPluginAuthorizationRoles
|
||||
CatalystPluginCaptcha
|
||||
CatalystPluginSessionStateCookie
|
||||
CatalystPluginSessionStoreFastMmap
|
||||
CatalystPluginStackTrace
|
||||
CatalystPluginUnicodeEncoding
|
||||
CatalystTraitForRequestProxyBase
|
||||
CatalystViewDownload
|
||||
CatalystViewJSON
|
||||
CatalystViewTT
|
||||
CatalystXScriptServerStarman
|
||||
CatalystXRoleApplicator
|
||||
CryptRandPasswd
|
||||
DBDPg
|
||||
DBDSQLite
|
||||
DataDump
|
||||
DateTime
|
||||
DigestSHA1
|
||||
EmailMIME
|
||||
EmailSender
|
||||
FileSlurp
|
||||
IOCompress
|
||||
IPCRun
|
||||
JSON
|
||||
JSONAny
|
||||
JSONXS
|
||||
LWP
|
||||
LWPProtocolHttps
|
||||
NetAmazonS3
|
||||
NetPrometheus
|
||||
NetStatsd
|
||||
PadWalker
|
||||
Readonly
|
||||
SQLSplitStatement
|
||||
SetScalar
|
||||
Starman
|
||||
SysHostnameLong
|
||||
TermSizeAny
|
||||
TestMore
|
||||
TextDiff
|
||||
TextTable
|
||||
XMLSimple
|
||||
nix
|
||||
nix.perl-bindings
|
||||
git
|
||||
boehmgc
|
||||
];
|
||||
};
|
||||
in stdenv.mkDerivation rec {
|
||||
pname = "hydra";
|
||||
version = "2020-02-06";
|
||||
|
||||
inherit stdenv;
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "NixOS";
|
||||
repo = pname;
|
||||
rev = "2b4f14963b16b21ebfcd6b6bfa7832842e9b2afc";
|
||||
sha256 = "16q0cffcsfx5pqd91n9k19850c1nbh4vvbd9h8yi64ihn7v8bick";
|
||||
{
|
||||
# Package for phase-1 of the db migration for Hydra.
|
||||
# https://github.com/NixOS/hydra/pull/711
|
||||
hydra-migration = callPackage ./common.nix {
|
||||
version = "2020-02-10";
|
||||
src = fetchFromGitHub {
|
||||
owner = "NixOS";
|
||||
repo = "hydra";
|
||||
rev = "add4f610ce6f206fb44702b5a894d877b3a30e3a";
|
||||
sha256 = "1d8hdgjx2ys0zmixi2ydmimdq7ml20h1ji4amwawcyw59kssh6l3";
|
||||
};
|
||||
nix = nixStable;
|
||||
migration = true;
|
||||
};
|
||||
|
||||
buildInputs =
|
||||
[ makeWrapper autoconf automake libtool unzip nukeReferences sqlite libpqxx
|
||||
gitAndTools.top-git mercurial darcs subversion bazaar openssl bzip2 libxslt
|
||||
guile # optional, for Guile + Guix support
|
||||
perlDeps perl nix
|
||||
postgresql # for running the tests
|
||||
nlohmann_json
|
||||
boost
|
||||
];
|
||||
# Hydra from latest master (or flakes) branch. Contains breaking changes,
|
||||
# so when having an older version, `pkgs.hydra-migration` should be deployed first.
|
||||
|
||||
hydraPath = lib.makeBinPath (
|
||||
[ sqlite subversion openssh nix coreutils findutils pixz
|
||||
gzip bzip2 lzma gnutar unzip git gitAndTools.top-git mercurial darcs gnused bazaar
|
||||
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] );
|
||||
hydra-unstable = callPackage ./common.nix {
|
||||
version = "2020-03-24";
|
||||
src = fetchFromGitHub {
|
||||
owner = "NixOS";
|
||||
repo = "hydra";
|
||||
rev = "12cc46cdb36321acd4c982429a86eb0f8f3cc969";
|
||||
sha256 = "10ipxzdxr47c8w5jg69mbax2ykc7lb5fs9bbdd3iai9wzyfz17ln";
|
||||
};
|
||||
nix = nixUnstable;
|
||||
};
|
||||
|
||||
nativeBuildInputs = [ autoreconfHook pkgconfig ];
|
||||
|
||||
configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ];
|
||||
|
||||
NIX_CFLAGS_COMPILE = "-pthread";
|
||||
|
||||
shellHook = ''
|
||||
PATH=$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$(pwd)/src/hydra-evaluator:$PATH
|
||||
PERL5LIB=$(pwd)/src/lib:$PERL5LIB;
|
||||
'';
|
||||
|
||||
enableParallelBuilding = true;
|
||||
|
||||
preCheck = ''
|
||||
patchShebangs .
|
||||
export LOGNAME=''${LOGNAME:-foo}
|
||||
'';
|
||||
|
||||
postInstall = ''
|
||||
mkdir -p $out/nix-support
|
||||
for i in $out/bin/*; do
|
||||
read -n 4 chars < $i
|
||||
if [[ $chars =~ ELF ]]; then continue; fi
|
||||
wrapProgram $i \
|
||||
--prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
|
||||
--prefix PATH ':' $out/bin:$hydraPath \
|
||||
--set HYDRA_RELEASE ${version} \
|
||||
--set HYDRA_HOME $out/libexec/hydra \
|
||||
--set NIX_RELEASE ${nix.name or "unknown"}
|
||||
done
|
||||
''; # */
|
||||
|
||||
dontStrip = true;
|
||||
|
||||
passthru.perlDeps = perlDeps;
|
||||
|
||||
meta = with stdenv.lib; {
|
||||
description = "Nix-based continuous build system";
|
||||
license = licenses.gpl3;
|
||||
platforms = platforms.linux;
|
||||
maintainers = with maintainers; [ ma27 ];
|
||||
hydra-flakes = callPackage ./common.nix {
|
||||
version = "2020-03-27";
|
||||
src = fetchFromGitHub {
|
||||
owner = "NixOS";
|
||||
repo = "hydra";
|
||||
rev = "a7540b141d085a7e78c21fda8e8c05907c659b34";
|
||||
sha256 = "08fs7593w5zs8vh4c66gvrxk6s840pp6hj8nwf51wsa27kg5a943";
|
||||
};
|
||||
nix = nixFlakes;
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user