From 924016f45f0fbe61b6a0db97ce9f2b41b8645dda Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 9 Aug 2018 15:33:22 +0200 Subject: [PATCH] dd-agent: Simplify inclusion of additional Datadog core integrations Refactors the process used to build the Datadog core integrations to be more easily extensible with integrations other than the ones built and installed by default. Documentation has been added in relevant parts of the module to describe how the process works. As a high-level overview: The `datadog-integrations-core` attribute in the top-level package set now accepts an extra parameter. This parameter is an attribute set where each key is the name of a Datadog integration as it appears in Datadog's integrations-core repository[1], and the value is a function that receives the Python package set and returns the required dependencies of this integration. For example: datadog-integrations-core { ntp = (ps: [ ps.ntplib ]); }; This would build the default integrations and, additionally, the `ntp` integration. To support passing the modified Python environment to the datadog-agent itself, the `python` key has been moved inside of the derivation which means that it will be made overridable. This relates to NixOS/nixpkgs#40399. [1]: https://github.com/DataDog/integrations-core --- pkgs/tools/networking/dd-agent/6.nix | 5 +- .../networking/dd-agent/integrations-core.nix | 101 +++++++++++------- pkgs/top-level/all-packages.nix | 5 +- 3 files changed, 69 insertions(+), 42 deletions(-) diff --git a/pkgs/tools/networking/dd-agent/6.nix b/pkgs/tools/networking/dd-agent/6.nix index be5f016c187..56a71595cea 100644 --- a/pkgs/tools/networking/dd-agent/6.nix +++ b/pkgs/tools/networking/dd-agent/6.nix @@ -1,8 +1,6 @@ { stdenv, fetchFromGitHub, buildGoPackage, makeWrapper, pythonPackages, pkgconfig }: let - inherit (pythonPackages) python; - # keep this in sync with github.com/DataDog/agent-payload dependency payloadVersion = "4.7"; @@ -27,6 +25,9 @@ in buildGoPackage rec { goDeps = ./deps.nix; goPackagePath = "github.com/${owner}/${repo}"; + # Explicitly set this here to allow it to be overridden. + python = pythonPackages.python; + nativeBuildInputs = [ pkgconfig makeWrapper ]; PKG_CONFIG_PATH = "${python}/lib/pkgconfig"; diff --git a/pkgs/tools/networking/dd-agent/integrations-core.nix b/pkgs/tools/networking/dd-agent/integrations-core.nix index 2efc8247369..9212209e775 100644 --- a/pkgs/tools/networking/dd-agent/integrations-core.nix +++ b/pkgs/tools/networking/dd-agent/integrations-core.nix @@ -1,7 +1,39 @@ -{ pkgs -, python -, overrides ? (self: super: {}) -}: +# The declarations in this file build the Datadog agent's core +# integrations. These integrations are tracked in a separate +# repository[1] outside of the agent's primary repository and provide +# checks for various kinds of services. +# +# Not all services are relevant for all users, however. As some of +# them depend on various tools and Python packages it is nonsensical +# to build *all* integrations by default. +# +# A set of default integrations is defined and built either way. +# Additional integrations can be specified by overriding +# `extraIntegrations` in datadog-integrations-core. +# +# In practice the syntax for using this with additional integrations +# is not the most beautiful, but it works. For example to use +# datadog-agent from the top-level with the `ntp`-integration +# included, one could say: +# +# let +# integrationsWithNtp = datadog-integrations-core { +# # Extra integrations map from the integration name (as in the +# # integrations-core repository) to a function that receives the +# # Python package set and returns the required dependencies.g +# ntp = (ps: [ ps.ntplib ]); +# }; +# +# in ddAgentWithNtp = datadog-agent.overrideAttrs(_ : { +# python = integrationsWithNtp.python; +# }); +# +# The NixOS module 'datadog-agent' provides a simplified interface to +# this. Please see the module itself for more information. +# +# [1]: https://github.com/DataDog/integrations-core + +{ pkgs, python, extraIntegrations ? {} }: with pkgs.lib; @@ -14,6 +46,7 @@ let }; version = "git-2018-05-27"; + # Build helper to build a single datadog integration package. buildIntegration = { pname, ... }@args: python.pkgs.buildPythonPackage (args // { inherit src version; name = "datadog-integration-${pname}-${version}"; @@ -27,40 +60,32 @@ let doCheck = false; }); - packages = (self: { - python = python.withPackages (ps: with self; [ disk network postgres nginx mongo ]); + # Base package depended on by all other integrations. + datadog_checks_base = buildIntegration { + pname = "checks-base"; + sourceRoot = "datadog_checks_base"; + propagatedBuildInputs = with python.pkgs; [ + requests protobuf prometheus_client uuid simplejson uptime + ]; + }; - datadog_checks_base = buildIntegration { - pname = "checks-base"; - sourceRoot = "datadog_checks_base"; - propagatedBuildInputs = with self; with python.pkgs; [ requests protobuf prometheus_client uuid simplejson uptime ]; - }; + # Default integrations that should be built: + defaultIntegrations = { + disk = (ps: [ ps.psutil ]); + mongo = (ps: [ ps.pymongo ]); + network = (ps: [ ps.psutil ]); + nginx = (ps: []); + postgres = (ps: with ps; [ pg8000 psycopg2 ]); + }; - disk = buildIntegration { - pname = "disk"; - propagatedBuildInputs = with self; with python.pkgs; [ datadog_checks_base psutil ]; - }; + # All integrations (default + extra): + integrations = defaultIntegrations // extraIntegrations; + builtIntegrations = mapAttrs (pname: fdeps: buildIntegration { + inherit pname; + propagatedBuildInputs = (fdeps python.pkgs) ++ [ datadog_checks_base ]; + }) integrations; - network = buildIntegration { - pname = "network"; - propagatedBuildInputs = with self; with python.pkgs; [ datadog_checks_base psutil ]; - }; - - postgres = buildIntegration { - pname = "postgres"; - propagatedBuildInputs = with self; with python.pkgs; [ datadog_checks_base pg8000 psycopg2 ]; - }; - - nginx = buildIntegration { - pname = "nginx"; - propagatedBuildInputs = with self; with python.pkgs; [ datadog_checks_base ]; - }; - - mongo = buildIntegration { - pname = "mongo"; - propagatedBuildInputs = with self; with python.pkgs; [ datadog_checks_base pymongo ]; - }; - - }); - -in fix' (extends overrides packages) +in builtIntegrations // { + inherit datadog_checks_base; + python = python.withPackages (_: (attrValues builtIntegrations)); +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 323a5daab3c..3d50a280ac5 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -15674,10 +15674,11 @@ with pkgs; dd-agent = callPackage ../tools/networking/dd-agent/5.nix { }; datadog-agent = callPackage ../tools/networking/dd-agent/6.nix { - pythonPackages = datadog-integrations-core; + pythonPackages = datadog-integrations-core {}; }; - datadog-integrations-core = callPackage ../tools/networking/dd-agent/integrations-core.nix { + datadog-integrations-core = extras: callPackage ../tools/networking/dd-agent/integrations-core.nix { python = python27; + extraIntegrations = extras; }; ddgr = callPackage ../applications/misc/ddgr { };