From d16e0f8dc3b600a5d501d31bb5e87bf000a51ae3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 26 Jan 2019 10:06:25 -0500 Subject: [PATCH 1/7] all-packages: move fetch* to pkgs/build-support/ --- pkgs/build-support/fetchbitbucket/default.nix | 10 +++ pkgs/build-support/fetchgithub/default.nix | 33 +++++++++ pkgs/build-support/fetchgitlab/default.nix | 10 +++ pkgs/build-support/fetchrepoorcz/default.nix | 10 +++ pkgs/build-support/fetchsavannah/default.nix | 10 +++ pkgs/top-level/all-packages.nix | 73 ++----------------- 6 files changed, 78 insertions(+), 68 deletions(-) create mode 100644 pkgs/build-support/fetchbitbucket/default.nix create mode 100644 pkgs/build-support/fetchgithub/default.nix create mode 100644 pkgs/build-support/fetchgitlab/default.nix create mode 100644 pkgs/build-support/fetchrepoorcz/default.nix create mode 100644 pkgs/build-support/fetchsavannah/default.nix diff --git a/pkgs/build-support/fetchbitbucket/default.nix b/pkgs/build-support/fetchbitbucket/default.nix new file mode 100644 index 00000000000..a99f72e9eaa --- /dev/null +++ b/pkgs/build-support/fetchbitbucket/default.nix @@ -0,0 +1,10 @@ +{ fetchzip }: + +{ owner, repo, rev, name ? "source" +, ... # For hash agility +}@args: fetchzip ({ + inherit name; + url = "https://bitbucket.org/${owner}/${repo}/get/${rev}.tar.gz"; + meta.homepage = "https://bitbucket.org/${owner}/${repo}/"; + extraPostFetch = ''rm -f "$out"/.hg_archival.txt''; # impure file; see #12002 +} // removeAttrs args [ "owner" "repo" "rev" ]) // { inherit rev; } diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix new file mode 100644 index 00000000000..66671dd0a6a --- /dev/null +++ b/pkgs/build-support/fetchgithub/default.nix @@ -0,0 +1,33 @@ +{ lib, fetchgit, fetchzip }: + +{ owner, repo, rev, name ? "source" +, fetchSubmodules ? false, private ? false +, githubBase ? "github.com", varPrefix ? null +, ... # For hash agility +}@args: assert private -> !fetchSubmodules; +let + baseUrl = "https://${githubBase}/${owner}/${repo}"; + passthruAttrs = removeAttrs args [ "owner" "repo" "rev" "fetchSubmodules" "private" "githubBase" "varPrefix" ]; + varBase = "NIX${if varPrefix == null then "" else "_${varPrefix}"}_GITHUB_PRIVATE_"; + # We prefer fetchzip in cases we don't need submodules as the hash + # is more stable in that case. + fetcher = if fetchSubmodules then fetchgit else fetchzip; + privateAttrs = lib.optionalAttrs private { + netrcPhase = '' + if [ -z "''$${varBase}USERNAME" -o -z "''$${varBase}PASSWORD" ]; then + echo "Error: Private fetchFromGitHub requires the nix building process (nix-daemon in multi user mode) to have the ${varBase}USERNAME and ${varBase}PASSWORD env vars set." >&2 + exit 1 + fi + cat > netrc < !fetchSubmodules; - let - baseUrl = "https://${githubBase}/${owner}/${repo}"; - passthruAttrs = removeAttrs args [ "owner" "repo" "rev" "fetchSubmodules" "private" "githubBase" "varPrefix" ]; - varBase = "NIX${if varPrefix == null then "" else "_${varPrefix}"}_GITHUB_PRIVATE_"; - # We prefer fetchzip in cases we don't need submodules as the hash - # is more stable in that case. - fetcher = if fetchSubmodules then fetchgit else fetchzip; - privateAttrs = lib.optionalAttrs private { - netrcPhase = '' - if [ -z "''$${varBase}USERNAME" -o -z "''$${varBase}PASSWORD" ]; then - echo "Error: Private fetchFromGitHub requires the nix building process (nix-daemon in multi user mode) to have the ${varBase}USERNAME and ${varBase}PASSWORD env vars set." >&2 - exit 1 - fi - cat > netrc < Date: Sat, 26 Jan 2019 10:24:58 -0500 Subject: [PATCH 2/7] nixos/manual: use default bs value Apparently this is a little slower but much safer & not prone to potential argument errors. --- nixos/doc/manual/installation/installing-usb.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/doc/manual/installation/installing-usb.xml b/nixos/doc/manual/installation/installing-usb.xml index 3a81b3a2040..c0372e8ebd9 100644 --- a/nixos/doc/manual/installation/installing-usb.xml +++ b/nixos/doc/manual/installation/installing-usb.xml @@ -23,7 +23,7 @@ $ diskutil list [..] $ diskutil unmountDisk diskN Unmount of all volumes on diskN was successful -$ sudo dd bs=1000000 if=nix.iso of=/dev/rdiskN +$ sudo dd if=nix.iso of=/dev/rdiskN Using the 'raw' rdiskN device instead of diskN completes in minutes instead of hours. After From 17ec7f3a16a473e3ff3de6328c98cb36db6b112e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 26 Jan 2019 21:42:16 -0500 Subject: [PATCH 3/7] nixpkgs/manual: document fetcher functions Fixes #32439. --- doc/functions.xml | 1 + doc/functions/fetchers.xml | 198 +++++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+) create mode 100644 doc/functions/fetchers.xml diff --git a/doc/functions.xml b/doc/functions.xml index e6d59ebde97..0dc32bbc5bd 100644 --- a/doc/functions.xml +++ b/doc/functions.xml @@ -11,6 +11,7 @@ + diff --git a/doc/functions/fetchers.xml b/doc/functions/fetchers.xml new file mode 100644 index 00000000000..96937ca7182 --- /dev/null +++ b/doc/functions/fetchers.xml @@ -0,0 +1,198 @@ +
+ Fetcher functions + + + When using Nix, you will frequently need to download source code + and other file from the internet. Nixpkgs comes with a few helper + functions that allow you to fetch fixed-output derivations in + structured way. + + + + The two fetcher primitives are fetchurl and + fetchzip. Both of these have two required + arguments, a URL and a hash. The hash is typically + sha256, although many more hash algorithms are + supported. Nixpkgs contributors are currently recommended to use + sha256. This hash will be used by Nix to + identify your source. A typical usage of fetchurl is provided + below. + + + + + + The main difference between fetchurl and + fetchzip is in how they store the contents. + fetchurl will store the unaltered contents of + the URL within the Nix store. fetchzip on the + other hand will decompress the archive for you, making files and + directories directly accessible in the future. + fetchzip can only be used with archives. + Despite the name, fetchzip is not limited to + .zip files and can also be used with any tarball. + + + + fetchpatch works very similarly to + fetchurl with the same arguments expected. + + + + Other fetcher functions allow you to add source code directly from + a VCS such as subversion or git. These are mostly straightforward + names based on the name of the command used with the VCS system. + Because they give you a working repository, they act most like + fetchzip. + + + + + + fetchsvn + + + + Used with Subversion. Expects url to a + Subversion directory, rev, and + sha256. + + + + + + fetchgit + + + + Used with Git. Expects url to a Git repo, + rev, and sha256. + + + + + + fetchfossil + + + + Used with Fossil. Expects url to a Fossil + archive, rev, and sha256. + + + + + + fetchcvs + + + + Used with CVS. Expects cvsRoot, + tag, and sha256. + + + + + + fetchhg + + + + Used with Mercurial. Expects url, + rev, and sha256. + + + + + + + A number of fetcher functions wrap part of + fetchurl and fetchzip. + They are mainly convenience functions intended for commonly used + destinations of source code in Nixpkgs. These wrapper fetchers are + listed below. + + + + + + fetchFromGitHub + + + + fetchFromGitHub expects four arguments. + owner is a string corresponding to the + GitHub user or organization that controls this repository. + repo corresponds to the name of the + software repository. These are located at the top of every + GitHub HTML page as + owner/repo. + rev corresponds to the Git commit hash or + tag that will be downloaded from Git. Finally, + sha256. Again, other hash algorithms are + also available but sha256 is currently + preferred. + + + + + + fetchFromGitLab + + + + This is used with GitLab repositories. The arguments expected + are very similar to fetchFromGitHub above. + + + + + + fetchFromBitbucket + + + + This is used with BitBucket repositories. The arguments expected + are very similar to fetchFromGitHub above. + + + + + + fetchFromSavannah + + + + This is used with Savannah repositories. The arguments expected + are very similar to fetchFromGitHub above. + + + + + + fetchFromRepoOrCz + + + + This is used with repo.or.cz repositories. The arguments + expected are very similar to fetchFromGitHub above. + + + + + + +
From adb717a153e9fda88d9bf9ac183d64fdf4887c40 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 26 Jan 2019 22:20:23 -0500 Subject: [PATCH 4/7] nixpkgs/manual: document default setup hooks Fixes #34857. --- doc/stdenv.xml | 135 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 131 insertions(+), 4 deletions(-) diff --git a/doc/stdenv.xml b/doc/stdenv.xml index ac0d84b90f9..21667252ad0 100644 --- a/doc/stdenv.xml +++ b/doc/stdenv.xml @@ -2192,10 +2192,128 @@ addEnvHooks "$hostOffset" myBashFunction - Here are some packages that provide a setup hook. Since the mechanism is - modular, this probably isn't an exhaustive list. Then again, since the - mechanism is only to be used as a last resort, it might be. - + First, let’s cover some setup hooks that are part of Nixpkgs + default stdenv. This means that they are run for every package + built using stdenv.mkDerivation. Some of + these are platform specific, so they may run on Linux but not + Darwin or vice-versa. + + + + move-docs.sh + + + + This setup hook moves any installed documentation to the + /share subdirectory directory. This includes + the man, doc and info directories. This is needed for legacy + programs that do not know use the share subdirectory. + + + + + + compress-man-pages.sh + + + + This setup hook compresses any man pages that have been + installed. The compression is done using the gzip program. This + helps to reduce installed size of packages. + + + + + + strip.sh + + + + This runs the strip command on installed binaries and + libraries. This removed things like debug symbols when they are + not needed. This also helps to reduce installed size of + packages. + + + + + + patch-shebangs.sh + + + + This setup hook patches installed scripts to use the full path + to the shebang interpreter. A shebang interpreter is the first + commented line of a script telling the operating system + what to use to run this script. In Nix, we want an exact path + to that interpreter to be used. This often replcaes + /bin/sh with a path in the Nix store. + + + + + + audit-tmpdir.sh + + + + This verifies that no references are left from the install + binaries to the directory used to build those binaries. This + ensures that the binaries do not need things outside the Nix + store. This currently Linux only. + + + + + + multiple-outputs.sh + + + + This setup hook adds configure flags that tell packages to + install files into any one of the proper outputs listed in + outputs. This behavior can be turned off by setting + setOutputFlags to false in the derivation + environment. See for + more information. + + + + + + move-sbin.sh + + + + This setup hook moves any binaries installed in the sbin + subdirectory into bin. In addition, a link is provided from + sbin to bin for compatibility. + + + + + + move-lib64.sh + + + + This setup hook moves any libraries installed in the lib64 + subdirectory into lib. In addition, a link is provided from + lib64 to lib for compatibility. + + + + + + set-source-date-epoch-to-latest.sh + + + + This sets SOURCE_DATE_EPOCH to the + modification time of the most recent file. + + + Bintools Wrapper @@ -2302,6 +2420,15 @@ addEnvHooks "$hostOffset" myBashFunction + + + + + Here are some more packages that provide a setup hook. Since the + mechanism is modular, this probably isn't an exhaustive list. Then + again, since the mechanism is only to be used as a last resort, it + might be. + Perl From 498a242bf4b4ad8aaf5624bd19602b7676766af8 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 26 Jan 2019 22:34:06 -0500 Subject: [PATCH 5/7] nixpkgs/manual: add trivial builders section Fixes #25507. --- doc/functions.xml | 1 + doc/functions/trivial-builders.xml | 84 ++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 doc/functions/trivial-builders.xml diff --git a/doc/functions.xml b/doc/functions.xml index 0dc32bbc5bd..0d6e2770e6e 100644 --- a/doc/functions.xml +++ b/doc/functions.xml @@ -12,6 +12,7 @@ + diff --git a/doc/functions/trivial-builders.xml b/doc/functions/trivial-builders.xml new file mode 100644 index 00000000000..4fbe8883610 --- /dev/null +++ b/doc/functions/trivial-builders.xml @@ -0,0 +1,84 @@ +
+ Trivial builders + + + There are a couple of functions provide in Nixpkgs that help with + building derivations. The most important one, + stdenv.mkDerivation, has already been + documented above. These wrap + stdenv.mkDerivation, making it easier to use + in certain cases. + + + + + + runCommand + + + + This takes three arguments, name, + env, and buildCommand. + name is just the name that Nix will use to + refer to the derivation. env is an attribute + set specifying environment variables that will be set for this + derivation. buildCommand specifies the + commands that will be run to create this derivation. Note that + you will need to create $out for Nix to + register the command as successful. + + + + + + runCommandCC + + + + This works just like runCommand. The only + difference is that it also provides a C compiler for your use. + To minimize your dependencies, you should only use this if you + are sure you will need a C compiler as part of running your command. + + + + + + writeTextFile + + + + This writes text to the Nix store. This is + useful for creating scripts from Nix expressions. This takes an + attribute set and expects two arguments, + name and text. + name corresponds to the name used in the Nix + store path. text will be the contents of the + file. You can also set executable to true to + make this file have the executable bit set. + + + + + + symlinkJoin + + + + This can be used to put many derivations into the same directory + structure. It works by creating a new derivation and adding + symlinks to each of the paths listed. It expects two arguments, + name, and paths. + name is the name used in the Nix store path + for the created derivation. paths is a list of + paths that will be symlinked. These paths can be to Nix store + derivations or any other directory. + + + + + +
From d7b62cb601b48d0dd8a70fdc4b5169d9492e4ce2 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sun, 27 Jan 2019 11:57:36 -0500 Subject: [PATCH 6/7] nixpkgs/manual: address review comments Mostly taken from requested changes exactly as recommended. --- doc/functions/fetchers.xml | 20 ++++--- doc/functions/trivial-builders.xml | 83 ++++++++++++++++++++++-------- doc/stdenv.xml | 28 +++++----- 3 files changed, 90 insertions(+), 41 deletions(-) diff --git a/doc/functions/fetchers.xml b/doc/functions/fetchers.xml index 96937ca7182..b3bd2fe0f45 100644 --- a/doc/functions/fetchers.xml +++ b/doc/functions/fetchers.xml @@ -6,8 +6,8 @@ When using Nix, you will frequently need to download source code - and other file from the internet. Nixpkgs comes with a few helper - functions that allow you to fetch fixed-output derivations in + and other files from the internet. Nixpkgs comes with a few helper + functions that allow you to fetch fixed-output derivations in a structured way. @@ -48,7 +48,11 @@ stdenv.mkDerivation { fetchpatch works very similarly to - fetchurl with the same arguments expected. + fetchurl with the same arguments expected. It + expects patch files as a source and and performs normalization on + them before computing the checksum. For example it will remove + comments or other unstable parts that are sometimes added by + version control systems and can change over time. @@ -80,6 +84,9 @@ stdenv.mkDerivation { Used with Git. Expects url to a Git repo, rev, and sha256. + rev in this case can be full the git commit + id (SHA1 hash) or a tag name like + refs/tags/v1.0. @@ -141,9 +148,10 @@ stdenv.mkDerivation { GitHub HTML page as owner/repo. rev corresponds to the Git commit hash or - tag that will be downloaded from Git. Finally, - sha256. Again, other hash algorithms are - also available but sha256 is currently + tag (e.g v1.0) that will be downloaded from + Git. Finally, sha256 corresponds to the + hash of the extracted directory. Again, other hash algorithms + are also available but sha256 is currently preferred. diff --git a/doc/functions/trivial-builders.xml b/doc/functions/trivial-builders.xml index 4fbe8883610..c5972d3ea90 100644 --- a/doc/functions/trivial-builders.xml +++ b/doc/functions/trivial-builders.xml @@ -5,10 +5,10 @@ Trivial builders - There are a couple of functions provide in Nixpkgs that help with - building derivations. The most important one, + Nixpkgs provides a couple of functions that help with building + derivations. The most important one, stdenv.mkDerivation, has already been - documented above. These wrap + documented above. The following functions wrap stdenv.mkDerivation, making it easier to use in certain cases. @@ -22,14 +22,42 @@ This takes three arguments, name, env, and buildCommand. - name is just the name that Nix will use to - refer to the derivation. env is an attribute - set specifying environment variables that will be set for this - derivation. buildCommand specifies the - commands that will be run to create this derivation. Note that - you will need to create $out for Nix to - register the command as successful. - + name is just the name that Nix will append + to the store path in the same way that + stdenv.mkDerivation uses its + name attribute. env is an + attribute set specifying environment variables that will be set + for this derivation. These attributes are then passed to the + wrapped stdenv.mkDerivation. + buildCommand specifies the commands that + will be run to create this derivation. Note that you will need + to create $out for Nix to register the + command as successful. + + + An example of using runCommand is provided + below. + + + (import <nixpkgs> {}).runCommand "my-example" {} '' + echo My example command is running + + mkdir $out + + echo I can write data to the Nix store > $out/message + + echo I can also run basic commands like: + + echo ls + ls + + echo whoami + whoami + + echo date + date + '' + @@ -47,19 +75,30 @@ - writeTextFile + writeTextFile, writeText, + writeTextDir, writeScript, + writeScriptBin - This writes text to the Nix store. This is - useful for creating scripts from Nix expressions. This takes an - attribute set and expects two arguments, - name and text. - name corresponds to the name used in the Nix - store path. text will be the contents of the - file. You can also set executable to true to - make this file have the executable bit set. - + These functions write text to the Nix store. + This is useful for creating scripts from Nix expressions. + writeTextFile takes an attribute set and + expects two arguments, name and + text. name corresponds to + the name used in the Nix store path. text + will be the contents of the file. You can also set + executable to true to make this file have + the executable bit set. + + + Many more commands wrap writeTextFile + including writeText, + writeTextDir, + writeScript, and + writeScriptBin. These are convenience + functions over writeTextFile. + @@ -75,7 +114,7 @@ name is the name used in the Nix store path for the created derivation. paths is a list of paths that will be symlinked. These paths can be to Nix store - derivations or any other directory. + derivations or any other subdirectory contained within. diff --git a/doc/stdenv.xml b/doc/stdenv.xml index 21667252ad0..3a51907eb8a 100644 --- a/doc/stdenv.xml +++ b/doc/stdenv.xml @@ -2207,7 +2207,8 @@ addEnvHooks "$hostOffset" myBashFunction This setup hook moves any installed documentation to the /share subdirectory directory. This includes the man, doc and info directories. This is needed for legacy - programs that do not know use the share subdirectory. + programs that do not know how to use the + share subdirectory. @@ -2219,7 +2220,7 @@ addEnvHooks "$hostOffset" myBashFunction This setup hook compresses any man pages that have been installed. The compression is done using the gzip program. This - helps to reduce installed size of packages. + helps to reduce the installed size of packages. @@ -2230,9 +2231,9 @@ addEnvHooks "$hostOffset" myBashFunction This runs the strip command on installed binaries and - libraries. This removed things like debug symbols when they are - not needed. This also helps to reduce installed size of - packages. + libraries. This removes unnecessary information like debug + symbols when they are not needed. This also helps to reduce the + installed size of packages. @@ -2244,10 +2245,11 @@ addEnvHooks "$hostOffset" myBashFunction This setup hook patches installed scripts to use the full path to the shebang interpreter. A shebang interpreter is the first - commented line of a script telling the operating system - what to use to run this script. In Nix, we want an exact path - to that interpreter to be used. This often replcaes - /bin/sh with a path in the Nix store. + commented line of a script telling the operating system which + program will run the script (e.g #!/bin/bash). In + Nix, we want an exact path to that interpreter to be used. This + often replaces /bin/sh with a path in the + Nix store. @@ -2260,7 +2262,7 @@ addEnvHooks "$hostOffset" myBashFunction This verifies that no references are left from the install binaries to the directory used to build those binaries. This ensures that the binaries do not need things outside the Nix - store. This currently Linux only. + store. This is currently supported in Linux only. @@ -2425,9 +2427,9 @@ addEnvHooks "$hostOffset" myBashFunction Here are some more packages that provide a setup hook. Since the - mechanism is modular, this probably isn't an exhaustive list. Then - again, since the mechanism is only to be used as a last resort, it - might be. + list of hooks is extensible, this is not an exhaustive list the + mechanism is only to be used as a last resort, it might cover most + uses. From a376d624410b91fed47a17b7b4db39278a9ddb6b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sun, 27 Jan 2019 12:01:23 -0500 Subject: [PATCH 7/7] nixpkgs/manual: add one more fix for a missed review --- doc/functions/trivial-builders.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/functions/trivial-builders.xml b/doc/functions/trivial-builders.xml index c5972d3ea90..92a07aedb5b 100644 --- a/doc/functions/trivial-builders.xml +++ b/doc/functions/trivial-builders.xml @@ -67,9 +67,10 @@ This works just like runCommand. The only - difference is that it also provides a C compiler for your use. - To minimize your dependencies, you should only use this if you - are sure you will need a C compiler as part of running your command. + difference is that it also provides a C compiler in + buildCommand’s environment. To minimize your + dependencies, you should only use this if you are sure you will + need a C compiler as part of running your command.