buildbot: 0.9.0rc4 -> 0.9.0.post1

- updates buildbot to version 9 release
- adds nixos configuration module
- fixes buildbot-www package deps
- re-hardcode path to tail
- builbot configuration via module vars

fixes #19759

nixos/modules/module-list.nix

@@ -137,6 +137,7 @@
   ./services/computing/torque/server.nix
   ./services/computing/torque/mom.nix
   ./services/computing/slurm/slurm.nix
+  ./services/continuous-integration/buildbot/master.nix
   ./services/continuous-integration/buildkite-agent.nix
   ./services/continuous-integration/hydra/default.nix
   ./services/continuous-integration/gitlab-runner.nix

nixos/modules/services/continuous-integration/buildbot/master.nix

@@ -0,0 +1,250 @@
+# NixOS module for Buildbot continous integration server.
+
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.buildbot-master;
+  escapeStr = s: escape ["'"] s;
+  masterCfg = pkgs.writeText "master.cfg" ''
+    from buildbot.plugins import *
+    factory = util.BuildFactory()
+    c = BuildmasterConfig = dict(
+     workers       = [${concatStringsSep "," cfg.workers}],
+     protocols     = { 'pb': {'port': ${cfg.bpPort} } },
+     title         = '${escapeStr cfg.title}',
+     titleURL      = '${escapeStr cfg.titleUrl}',
+     buildbotURL   = '${escapeStr cfg.buildbotUrl}',
+     db            = dict(db_url='${escapeStr cfg.dbUrl}'),
+     www           = dict(port=${toString cfg.port}),
+     change_source = [ ${concatStringsSep "," cfg.changeSource} ],
+     schedulers    = [ ${concatStringsSep "," cfg.schedulers} ],
+     builders      = [ ${concatStringsSep "," cfg.builders} ],
+     status        = [ ${concatStringsSep "," cfg.status} ],
+    )
+    for step in [ ${concatStringsSep "," cfg.factorySteps} ]:
+      factory.addStep(step)
+
+    ${cfg.extraConfig}
+  '';
+
+  configFile = if cfg.masterCfg == null then masterCfg else cfg.masterCfg;
+
+in {
+  options = {
+    services.buildbot-master = {
+
+      factorySteps = mkOption {
+        type = types.listOf types.str;
+        description = "Factory Steps";
+        default = [];
+        example = [
+          "steps.Git(repourl='git://github.com/buildbot/pyflakes.git', mode='incremental')"
+          "steps.ShellCommand(command=['trial', 'pyflakes'])"
+        ];
+      };
+
+      changeSource = mkOption {
+        type = types.listOf types.str;
+        description = "List of Change Sources.";
+        default = [];
+        example = [
+          "changes.GitPoller('git://github.com/buildbot/pyflakes.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)"
+        ];
+      };
+
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = "Whether to enable the Buildbot continuous integration server.";
+      };
+
+      extraConfig = mkOption {
+        type = types.str;
+        description = "Extra configuration to append to master.cfg";
+        default = "";
+      };
+
+      masterCfg = mkOption {
+        type = with types; nullOr path;
+        description = ''
+          Optionally pass path to raw master.cfg file.
+          Other options in this configuration will be ignored.
+        '';
+        default = null;
+        example = literalExample ''
+          pkgs.writeText "master.cfg" "BuildmasterConfig = c = {}"
+        '';
+      };
+
+      schedulers = mkOption {
+        type = types.listOf types.str;
+        description = "List of Schedulers.";
+        default = [
+          "schedulers.SingleBranchScheduler(name='all', change_filter=util.ChangeFilter(branch='master'), treeStableTimer=None, builderNames=['runtests'])"
+          "schedulers.ForceScheduler(name='force',builderNames=['runtests'])"
+        ];
+      };
+
+      builders = mkOption {
+        type = types.listOf types.str;
+        description = "List of Builders.";
+        default = [
+          "util.BuilderConfig(name='runtests',workernames=['default-worker'],factory=factory)"
+        ];
+      };
+
+      workers = mkOption {
+        type = types.listOf types.str;
+        description = "List of Workers.";
+        default = [
+          "worker.Worker('default-worker', 'password')"
+        ];
+        example = [ "worker.LocalWorker('default-worker')" ];
+      };
+
+      status = mkOption {
+        default = [];
+        type = types.listOf types.str;
+        description = "List of status notification endpoints.";
+      };
+
+      user = mkOption {
+        default = "buildbot";
+        type = types.str;
+        description = "User the buildbot server should execute under.";
+      };
+
+      group = mkOption {
+        default = "buildbot";
+        type = types.str;
+        description = "Primary group of buildbot user.";
+      };
+
+      extraGroups = mkOption {
+        type = types.listOf types.str;
+        default = [ "nixbld" ];
+        description = "List of extra groups that the buildbot user should be a part of.";
+      };
+
+      home = mkOption {
+        default = "/home/buildbot";
+        type = types.path;
+        description = "Buildbot home directory.";
+      };
+
+      buildbotDir = mkOption {
+        default = "${cfg.home}/master";
+        type = types.path;
+        description = "Specifies the Buildbot directory.";
+      };
+
+      bpPort = mkOption {
+        default = "9989";
+        type = types.string;
+        example = "tcp:10000:interface=127.0.0.1";
+        description = "Port where the master will listen to Buildbot Worker.";
+      };
+
+      listenAddress = mkOption {
+        default = "0.0.0.0";
+        type = types.str;
+        description = "Specifies the bind address on which the buildbot HTTP interface listens.";
+      };
+
+      buildbotUrl = mkOption {
+        default = "http://localhost:8010/";
+        type = types.str;
+        description = "Specifies the Buildbot URL.";
+      };
+
+      title = mkOption {
+        default = "Buildbot";
+        type = types.str;
+        description = "Specifies the Buildbot Title.";
+      };
+
+      titleUrl = mkOption {
+        default = "Buildbot";
+        type = types.str;
+        description = "Specifies the Buildbot TitleURL.";
+      };
+
+      dbUrl = mkOption {
+        default = "sqlite:///state.sqlite";
+        type = types.str;
+        description = "Specifies the database connection string.";
+      };
+
+      port = mkOption {
+        default = 8010;
+        type = types.int;
+        description = "Specifies port number on which the buildbot HTTP interface listens.";
+      };
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.buildbot-ui;
+        description = ''
+          Package to use for buildbot.
+          <literal>buildbot-full</literal> is required in order to use local workers.
+        '';
+        example = pkgs.buildbot-full;
+      };
+
+      packages = mkOption {
+        default = [ ];
+        example = [ pkgs.git ];
+        type = types.listOf types.package;
+        description = "Packages to add to PATH for the buildbot process.";
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    users.extraGroups = optional (cfg.group == "buildbot") {
+      name = "buildbot";
+    };
+
+    users.extraUsers = optional (cfg.user == "buildbot") {
+      name = "buildbot";
+      description = "buildbot user";
+      isNormalUser = true;
+      createHome = true;
+      home = cfg.home;
+      group = cfg.group;
+      extraGroups = cfg.extraGroups;
+      useDefaultShell = true;
+    };
+
+    systemd.services.buildbot-master = {
+      description = "Buildbot Continuous Integration Server";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      path = cfg.packages;
+
+      serviceConfig = {
+        Type = "forking";
+        User = cfg.user;
+        Group = cfg.group;
+        WorkingDirectory = cfg.home;
+        ExecStart = "${cfg.package}/bin/buildbot start ${cfg.buildbotDir}";
+      };
+
+      preStart = ''
+        mkdir -vp ${cfg.buildbotDir}
+        chown -c ${cfg.user}:${cfg.group} ${cfg.buildbotDir}
+        ln -sf ${configFile} ${cfg.buildbotDir}/master.cfg
+        ${cfg.package}/bin/buildbot create-master ${cfg.buildbotDir}
+      '';
+
+      postStart = ''
+        until [[ $(${pkgs.curl}/bin/curl -s --head -w '\n%{http_code}' http://localhost:${toString cfg.port} | tail -n1) =~ ^(200|403)$ ]]; do
+          sleep 1
+        done
+      '';
+    };
+  };
+
+}

pkgs/development/tools/build-managers/buildbot/default.nix

@@ -1,12 +1,21 @@
-{ stdenv, pythonPackages, fetchurl, coreutils, plugins ? [] }:
+{ stdenv,
+  lib,
+  pythonPackages,
+  fetchurl,
+  coreutils,
+  openssh,
+  buildbot-worker,
+  plugins ? [],
+  enableLocalWorker ? false
+}:
 
 pythonPackages.buildPythonApplication (rec {
   name = "${pname}-${version}";
   pname = "buildbot";
-  version = "0.9.0rc4";
+  version = "0.9.0.post1";
   src = fetchurl {
     url = "mirror://pypi/b/${pname}/${name}.tar.gz";
-    sha256 = "16bnrr5qkfpnby9sw9azcagnw0ybi7d8bpdlga2a4c61jg2d5dnc";
+    sha256 = "18rnsp691cnmbymlch6czx3mrcmifmf6dk97h9nslgfkkyf25n5g";
   };
 
   buildInputs = with pythonPackages; [
@@ -22,7 +31,7 @@ pythonPackages.buildPythonApplication (rec {
     pylint
     astroid
     pyflakes
-  ];
+  ] ++ lib.optionals (enableLocalWorker) [openssh];
 
   propagatedBuildInputs = with pythonPackages; [
 
@@ -52,14 +61,17 @@ pythonPackages.buildPythonApplication (rec {
     ramlfications
     sphinx-jinja
 
-  ] ++ plugins;
+  ] ++ plugins ++
+  lib.optionals (enableLocalWorker) [buildbot-worker];
 
   preInstall = ''
     # writes out a file that can't be read properly
     sed -i.bak -e '69,84d' buildbot/test/unit/test_www_config.py
+  '';
 
+  postPatch = ''
     # re-hardcode path to tail
-    sed -i.bak 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot/scripts/logwatcher.py
+    sed -i 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot/scripts/logwatcher.py
   '';
 
   postFixup = ''

pkgs/development/tools/build-managers/buildbot/plugins.nix

@@ -4,11 +4,11 @@ let
   buildbot-pkg = pythonPackages.buildPythonPackage rec {
     name = "${pname}-${version}";
     pname = "buildbot-pkg";
-    version = "0.9.0rc4";
+    version = "0.9.0.post1";
 
     src = fetchurl {
       url = "mirror://pypi/b/${pname}/${name}.tar.gz";
-      sha256 = "0dfdyc3x0926dynzdl9w7z0p84w287l362mxdl3r6wl87gkisr10";
+      sha256 = "0frmnc73dsyc9mjnrnpm4vdrwb7c63gc6maq6xvlp486v7sdhjbi";
     };
 
     propagatedBuildInputs = with pythonPackages; [ setuptools ];
@@ -23,22 +23,19 @@ let
   };
 
 in {
-
   www = pythonPackages.buildPythonPackage rec {
     name = "${pname}-${version}";
     pname = "buildbot_www";
-    version = "0.9.0rc4";
+    version = "0.9.0.post1";
 
     # NOTE: wheel is used due to buildbot circular dependency
     format = "wheel";
 
     src = fetchurl {
-      url = "https://pypi.python.org/packages/78/45/b43bd85695cd0178f8bac9c3b394062e9eb46f489b655c11e950e54278a2/${name}-py2-none-any.whl";
+      url = "https://pypi.python.org/packages/02/d0/fc56ee27a09498638a47dcc5637ee5412ab7a67bfb4b3ff47e041f3d7b66/${name}-py2-none-any.whl";
-      sha256 = "0ixi0y0jhbql55swsvy0jin1v6xf4q4mw9p5n9sll2h10lyp9h0p";
+      sha256 = "14ghch67k6090736n89l401swz7r9hnk2zlmdb59niq8lg7dyg9q";
     };
 
-    propagatedBuildInputs = [ buildbot-pkg ];
-
     meta = with stdenv.lib; {
       homepage = http://buildbot.net/;
       description = "Buildbot UI";
@@ -51,11 +48,11 @@ in {
   console-view = pythonPackages.buildPythonPackage rec {
     name = "${pname}-${version}";
     pname = "buildbot-console-view";
-    version = "0.9.0rc4";
+    version = "0.9.0.post1";
 
     src = fetchurl {
       url = "mirror://pypi/b/${pname}/${name}.tar.gz";
-      sha256 = "1fig635yg5dgn239g9wzfpw9wc3p91lcl9nnig9k7fijz85pwrva";
+      sha256 = "0dc7rb7mrpva5gj7l57i96a78d6yj28pkkj9hfim1955z9dgn58l";
     };
 
     propagatedBuildInputs = [ buildbot-pkg ];
@@ -72,11 +69,11 @@ in {
   waterfall-view = pythonPackages.buildPythonPackage rec {
     name = "${pname}-${version}";
     pname = "buildbot-waterfall-view";
-    version = "0.9.0rc4";
+    version = "0.9.0.post1";
 
     src = fetchurl {
       url = "mirror://pypi/b/${pname}/${name}.tar.gz";
-      sha256 = "08kh966grj9b4mif337vv7bqy5ixz8xz31ml63wysjb65djnjbk8";
+      sha256 = "0x9vvw15zzgj4w3qcxh8r10rb36ni0qh1215y7wbawh5lggnjm0g";
     };
 
     propagatedBuildInputs = [ buildbot-pkg ];

pkgs/development/tools/build-managers/buildbot/worker.nix

@@ -3,11 +3,11 @@
 pythonPackages.buildPythonApplication (rec {
   name = "${pname}-${version}";
   pname = "buildbot-worker";
-  version = "0.9.0rc4";
+  version = "0.9.0.post1";
 
   src = fetchurl {
     url = "mirror://pypi/b/${pname}/${name}.tar.gz";
-    sha256 = "1fv40pki1awv5f2z9vd7phjk7dlsy1cp4blsy2vdhqwbc7112a8c";
+    sha256 = "1f8ij3y62r9z7qv92x21rg9h9whhakkwv59rgniq09j64ggjz8lx";
   };
 
   buildInputs = with pythonPackages; [ setuptoolsTrial mock ];

pkgs/top-level/all-packages.nix

@@ -5952,6 +5952,7 @@ in
   };
   buildbot-full = self.buildbot.override {
     plugins = with self.buildbot-plugins; [ www console-view waterfall-view ];
+    enableLocalWorker = true;
   };
 
   buildkite-agent = callPackage ../development/tools/continuous-integration/buildkite-agent { };