Merge pull request #89453 from c00w/vend

Add vend support to go-modules

doc/languages-frameworks/go.xml

@@ -64,6 +64,11 @@ pet = buildGoModule rec {
       <varname>subPackages</varname> limits the builder from building child packages that have not been listed. If <varname>subPackages</varname> is not specified, all child packages will be built.
      </para>
     </callout>
+    <callout arearefs='ex-buildGoModule-3'>
+     <para>
+         <varname>runVend</varname> runs the vend command to generate the vendor directory. This is useful if your code depends on c code and go mod tidy does not include the needed sources to build.
+     </para>
+    </callout>
    </calloutlist>
   </para>
 

pkgs/applications/blockchains/go-ethereum.nix

@@ -11,21 +11,8 @@ buildGoModule rec {
     sha256 = "0nkzwmrzk0m9662cr18h5i54v07mw8v3fh0csvqx8n50z5fcvb7b";
   };
 
-  usb = fetchFromGitHub {
-    owner = "karalabe";
-    repo = "usb";
-    rev = "911d15fe12a9c411cf5d0dd5635231c759399bed";
-    sha256 = "0asd5fz2rhzkjmd8wjgmla5qmqyz4jaa6qf0n2ycia16jsck6wc2";
-  };
-
-  vendorSha256 = "13wh6r9zi5qw72xkbzy3mcgn7lv9l981x4lniypjbnkwhq2dj5iz";
-
-  overrideModAttrs = (_: {
-      postBuild = ''
-      cp -r --reflink=auto ${usb}/libusb vendor/github.com/karalabe/usb
-      cp -r --reflink=auto ${usb}/hidapi vendor/github.com/karalabe/usb
-      '';
-    });
+  runVend = true;
+  vendorSha256 = "1744df059bjksvih4653nnvb4kb1xvzdhypd0nnz36m1wrihqssv";
 
   subPackages = [
     "cmd/abidump"

pkgs/applications/misc/hugo/default.nix

@@ -11,21 +11,8 @@ buildGoModule rec {
     sha256 = "0rikr4yrjvmrv8smvr8jdbcjqwf61y369wn875iywrj63pyr74r9";
   };
 
-  golibsass = fetchFromGitHub {
-    owner = "bep";
-    repo = "golibsass";
-    rev = "8a04397f0baba474190a9f58019ff499ec43057a";
-    sha256 = "0xk3m2ynbydzx87dz573ihwc4ryq0r545vz937szz175ivgfrhh3";
-  };
-
-  overrideModAttrs = (_: {
-      postBuild = ''
-      rm -rf vendor/github.com/bep/golibsass/
-      cp -r --reflink=auto ${golibsass} vendor/github.com/bep/golibsass
-      '';
-    });
-
-  vendorSha256 = "031k8bvca1pb1naw922vg5h95gnwp76dii1cjcs0b1qj93isdibk";
+  vendorSha256 = "17xn6bdy942g6nx5xky41ixmd5kaz68chj3rb02ibpyraamx04nm";
+  runVend = true;
 
   buildFlags = [ "-tags" "extended" ];
 

pkgs/applications/networking/mailreaders/aerc/default.nix

@@ -13,21 +13,8 @@ buildGoModule rec {
     sha256 = "05qy14k9wmyhsg1hiv4njfx1zn1m9lz4d1p50kc36v7pq0n4csfk";
   };
 
-  libvterm = fetchFromGitHub {
-    owner = "ddevault";
-    repo = "go-libvterm";
-    rev = "b7d861da381071e5d3701e428528d1bfe276e78f";
-    sha256 = "06vv4pgx0i6hjdjcar4ch18hp9g6q6687mbgkvs8ymmbacyhp7s6";
-  };
-
-  vendorSha256 = "1rqn36510m0yb7k4bvq2hgirr3z8a2h5xa7cq5mb84xsmhvf0g69";
-
-  overrideModAttrs = (_: {
-      postBuild = ''
-      cp -r --reflink=auto ${libvterm}/libvterm vendor/github.com/ddevault/go-libvterm
-      cp -r --reflink=auto ${libvterm}/encoding vendor/github.com/ddevault/go-libvterm
-      '';
-    });
+  runVend = true;
+  vendorSha256 = "0avdvbhv1jlisiicpi5vshz28a2p2fgnlrag9zngzglcrbhdd1rn";
 
   nativeBuildInputs = [
     scdoc

pkgs/development/go-modules/generic/default.nix

@@ -1,4 +1,4 @@
-{ go, cacert, git, lib, removeReferencesTo, stdenv }:
+{ go, cacert, git, lib, removeReferencesTo, stdenv, vend }:
 
 { name ? "${args'.pname}-${args'.version}"
 , src
@@ -20,6 +20,9 @@
 , vendorSha256
 # Whether to delete the vendor folder supplied with the source.
 , deleteVendor ? false
+# Whether to run the vend tool to regenerate the vendor directory.
+# This is useful if any dependency contain C files.
+, runVend ? false
 
 , modSha256 ? null
 
@@ -48,6 +51,8 @@ let
 
   deleteFlag = if deleteVendor then "true" else "false";
 
+  vendCommand = if runVend then "${vend}/bin/vend" else "false";
+
   go-modules = if vendorSha256 != null then go.stdenv.mkDerivation (let modArgs = {
 
     name = "${name}-go-modules";
@@ -87,7 +92,13 @@ let
         echo "vendor folder exists, please set 'vendorSha256=null;' or 'deleteVendor=true;' in your expression"
         exit 10
       fi
-      go mod vendor
+
+      if [ ${vendCommand} != "false" ]; then
+        echo running vend to rewrite vendor folder
+        ${vendCommand}
+      else
+        go mod vendor
+      fi
       mkdir -p vendor
 
       runHook postBuild

pkgs/development/tools/vend/default.nix

@@ -0,0 +1,26 @@
+{ stdenv, buildGoModule, fetchFromGitHub }:
+
+buildGoModule {
+  pname = "vend";
+  version = "unstable-2020-06-04";
+
+  patches = [./remove_tidy.patch];
+
+  # A permanent fork from master is maintained to avoid non deterministic go tidy
+  src = fetchFromGitHub {
+    owner = "c00w";
+    repo = "vend";
+    rev = "24fdebfdb2c3cc0516321a9cf33a3fd81c209c04";
+    sha256 = "112p9dz9by2h2m3jha2bv1bvzn2a86bpg1wphgmf9gksjpwy835l";
+  };
+
+  vendorSha256 = null;
+
+  meta = with stdenv.lib; {
+    homepage = "https://github.com/c00w/vend";
+    description = "A utility which vendors go code including c dependencies";
+    maintainers = with maintainers; [ c00w ];
+    license = licenses.mit;
+    platforms = platforms.all;
+  };
+}

pkgs/development/tools/vend/remove_tidy.patch

@@ -0,0 +1,13 @@
+diff --git a/cli/cmd.go b/cli/cmd.go
+index c766559..3a133fd 100644
+--- a/cli/cmd.go
++++ b/cli/cmd.go
+@@ -12,7 +12,7 @@ import (
+ // UpdateModule makes sure the module is updated ready to vendor the
+ // dependencies.
+ func UpdateModule() {
+-	var commands = []string{"tidy", "download", "vendor"}
++	var commands = []string{"download", "vendor"}
+ 
+ 	for _, command := range commands {
+ 		cmd := exec.Command("go", "mod", command)

pkgs/servers/blockbook/default.nix

@@ -23,7 +23,8 @@ buildGoModule rec {
     sha256 = "0da1kav5x2xcmwvdgfk1q70l1k0sqqj3njgx2xx885d40m6qbnrs";
   };
 
-  vendorSha256 = "1qjlvhizl8cy06cgf4phia70bgbm4lj57z5z2gyr8aglx98bnpdn";
+  runVend = true;
+  vendorSha256 = "0p7vyw61nwvmaz7gz2bdh9fi6wp62i2vnzw6iz2r8cims4sbz53b";
 
   nativeBuildInputs = [ packr pkg-config ];
 
@@ -36,20 +37,6 @@ buildGoModule rec {
        -X github.com/trezor/blockbook/common.buildDate=unknown
   '';
 
-  goethereum = fetchFromGitHub {
-    owner = "ethereum";
-    repo = "go-ethereum";
-    rev = "v1.8.20";
-    sha256 = "0m2q1nz6f39pyr2rk6vflkwi4ykganzwr7wndpwr9rliw0x8jgi0";
-  };
-
-  overrideModAttrs = (_: {
-    postBuild = ''
-      rm -r vendor/github.com/ethereum/go-ethereum
-      cp -r --reflink=auto ${goethereum} vendor/github.com/ethereum/go-ethereum
-    '';
-  });
-
   preBuild = stdenv.lib.optionalString stdenv.isDarwin ''
     ulimit -n 8192
   '' + ''

pkgs/servers/mautrix-whatsapp/default.nix

@@ -13,19 +13,8 @@ buildGoModule rec {
 
   buildInputs = [ olm ];
 
-  vendorSha256 = "0ixfawfavv5r1d01d4gmj87vf5vv6p3f7kv4rkhfv48ys0j0437a";
-
-  overrideModAttrs = _: {
-    postBuild = ''
-      rm -r vendor/github.com/chai2010/webp
-      cp -r --reflink=auto ${fetchFromGitHub {
-        owner = "chai2010";
-        repo = "webp";
-        rev = "3da79ec3d682694d42bfd211db18fc1343c07cd7";
-        sha256 = "0gh3g52vz8na153mjmxkl80g3dvrcjw77xpjs1c02vagpj9jyw46";
-      }} vendor/github.com/chai2010/webp
-    '';
-  };
+  vendorSha256 = "05cqwprd1rcciw27wyz7lj1s3zmz2vq093vw1cx3kkjyf6lq8sk6";
+  runVend = true;
 
   meta = with stdenv.lib; {
     homepage = "https://github.com/tulir/mautrix-whatsapp";

pkgs/tools/security/saml2aws/default.nix

@@ -11,20 +11,8 @@ buildGoModule rec {
     sha256 = "0y5gvdrdr6i9spdwsxvzs1bxs32icxpkqxnglp1bf4gglc580d87";
   };
 
-  hid = fetchFromGitHub {
-    owner = "karalabe";
-    repo = "hid";
-    rev = "9c14560f9ee858c43f40b5cd01392b167aacf4e8";
-    sha256 = "0xc7b8mwha64j7l2fr2g5zy8pz7cqi0vrxx60gii52b6ii31xncx";
-  };
-
-  vendorSha256 = "0f81nrg8v3xh2hcx7g77p3ahr4gyj042bwr1knf2phpahgz9n9rn";
-  overrideModAttrs = (_: {
-      postBuild = ''
-      cp -r --reflink=auto ${hid}/libusb vendor/github.com/karalabe/hid
-      cp -r --reflink=auto ${hid}/hidapi vendor/github.com/karalabe/hid
-      '';
-    });
+  runVend = true;
+  vendorSha256 = "1kzihyx44sx6php4z58fzy6c3g0y713939yzxpgk3n03snn2x8sf";
 
   subPackages = [ "." "cmd/saml2aws" ];
 

pkgs/top-level/all-packages.nix

@@ -7428,6 +7428,8 @@ in
 
   vcstool = callPackage ../development/tools/vcstool { };
 
+  vend = callPackage ../development/tools/vend { };
+
   verilator = callPackage ../applications/science/electronics/verilator {};
 
   verilog = callPackage ../applications/science/electronics/verilog {};