Merge pull request #2 from ericbmerritt/erlang-improvements

rebar3: refactor to make hermetic

doc/erlang-users-guide.xml

@@ -3,6 +3,23 @@
          xml:id="users-guide-to-the-erlang-infrastructure">
 
 <title>User's Guide to the Erlang Infrastructure</title>
+<section xml:id="build-tools">
+  <title>Build Tools</title>
+  <para>
+    By default Rebar3 wants to manage it's own dependencies. In the
+    normal non-Nix, this is perfectly acceptable. In the Nix world it
+    is not. To support this we have created two versions of rebar3,
+    <literal>rebar3</literal> and <literal>rebar3-open</literal>. The
+    <literal>rebar3</literal> version has been patched to remove the
+    ability to download anything from it. If you are not running it a
+    nix-shell or a nix-build then its probably not going to work for
+    you. <literal>rebar3-open</literal> is the normal, un-modified
+    rebar3. It should work exactly as would any other version of
+    rebar3. Any Erlang package should rely on
+    <literal>rebar3</literal> and thats really what you should be
+    using too.
+  </para>
+</section>
 
 <section xml:id="how-to-install-erlang-packages">
   <title>How to install Erlang packages</title>

pkgs/development/erlang-modules/hex-packages.nix

@@ -99,33 +99,14 @@
 * ucol_nif_1_1_5
 * katipo_0_2_4
 * xref_runner_0_2_4
+* erlexec_1_0_1
+* exec_1_0_1
 */
 { stdenv, callPackage }:
 
 let
   self = rec {
 
-    aws_http_0_2_4 = callPackage
-      (
-        {  buildHex, barrel_jiffy_0_14_4, lhttpc_1_3_0 }:
-          buildHex {
-            name = "aws_http";
-            version = "0.2.4";
-            sha256 =
-              "96065da0d348a8e47e01531cfa720615e15a21c1bd4e5c82decf56026cde128f";
-
-            erlangDeps = [ barrel_jiffy_0_14_4 lhttpc_1_3_0 ];
-
-            meta = {
-              description = "Amazon AWS HTTP helpers";
-              license = stdenv.lib.licenses.free;
-              homepage = "https://github.com/anha0825/erl_aws_http";
-            };
-          }
-      ) {};
-
-    aws_http = aws_http_0_2_4;
-
     backoff_1_1_3 = callPackage
       (
         {  buildHex }:
@@ -625,13 +606,15 @@ let
 
     denrei_0_2_3 = callPackage
       (
-        {  buildHex }:
+        {  buildHex, lager_3_0_1, ranch }:
           buildHex {
             name = "denrei";
             version = "0.2.3";
             sha256 =
               "bc0e8cf7e085dda6027df83ef5d63c41b93988bcd7f3db7c68e4dad3cd599744";
 
+            erlangDeps = [ lager_3_0_1 ranch ];
+
             meta = {
               description = "Denrei - a lightweight Erlang messaging system.";
               license = stdenv.lib.licenses.mit;
@@ -1072,26 +1055,6 @@ let
 
     erldn = erldn_1_0_2;
 
-    erlexec_1_0_1 = callPackage
-      (
-        {  buildHex }:
-          buildHex {
-            name = "erlexec";
-            version = "1.0.1";
-            sha256 =
-              "eb1e11f16288db4ea35af08503eabf1250d5540c1e8bd35ba04312f5f703e14f";
-            compilePort = true;
-
-            meta = {
-              description = "OS Process Manager";
-              license = stdenv.lib.licenses.bsd3;
-              homepage = "https://github.com/saleyn/erlexec";
-            };
-          }
-      ) {};
-
-    erlexec = erlexec_1_0_1;
-
     erlsh_0_1_0 = callPackage
       (
         {  buildHex }:
@@ -1250,26 +1213,6 @@ let
 
     ex_bitcask = ex_bitcask_0_1_0;
 
-    exec_1_0_1 = callPackage
-      (
-        {  buildHex }:
-          buildHex {
-            name = "exec";
-            version = "1.0.1";
-            sha256 =
-              "87c7ef2dea2bb503bb0eec8cb34776172999aecc6e12d90f7629796a7a3ccb1f";
-            compilePort = true;
-
-            meta = {
-              description = "OS Process Manager";
-              license = stdenv.lib.licenses.bsd3;
-              homepage = "https://github.com/saleyn/erlexec";
-            };
-          }
-      ) {};
-
-    exec = exec_1_0_1;
-
     exmerl_0_1_1 = callPackage
       (
         {  buildHex }:
@@ -1796,13 +1739,15 @@ let
 
     jc_1_0_4 = callPackage
       (
-        {  buildHex }:
+        {  buildHex, jsone_1_2_0, jwalk_1_1_0, lager_3_0_1, ranch }:
           buildHex {
             name = "jc";
             version = "1.0.4";
             sha256 =
               "8bcfe202084109fc80fcf521e630466fc53cbb909aff4283bed43252664023df";
 
+            erlangDeps = [ jsone_1_2_0 jwalk_1_1_0 lager_3_0_1 ranch ];
+
             meta = {
               description = "A simple, distributed, in-memory caching system";
               license = stdenv.lib.licenses.mit;
@@ -2012,13 +1957,15 @@ let
 
     key2value_1_4_0 = callPackage
       (
-        {  buildHex }:
+        {  buildHex, lager_3_0_1, barrel_jiffy }:
           buildHex {
             name = "key2value";
             version = "1.4.0";
             sha256 =
               "ad63453fcf54ab853581b78c6d2df56be41ea691ba4bc05920264c19f35a0ded";
 
+            erlangDeps = [ lager_3_0_1 barrel_jiffy ];
+
             meta = {
               description = "Erlang 2-way Map";
               license = stdenv.lib.licenses.bsd3;
@@ -2029,13 +1976,15 @@ let
 
     key2value_1_5_1 = callPackage
       (
-        {  buildHex }:
+        {  buildHex, lager_3_0_1, barrel_jiffy }:
           buildHex {
             name = "key2value";
             version = "1.5.1";
             sha256 =
               "2a40464b9f8ef62e8828d869ac8d2bf9135b4956d29ba4eb044e8522b2d35ffa";
 
+            erlangDeps = [ lager_3_0_1 barrel_jiffy ];
+
             meta = {
               description = "Erlang 2-way Map";
               license = stdenv.lib.licenses.bsd3;
@@ -2432,27 +2381,6 @@ let
 
     neotoma = neotoma_1_7_3;
 
-    observer_cli_1_0_3 = callPackage
-      (
-        {  buildHex, recon_2_2_1 }:
-          buildHex {
-            name = "observer_cli";
-            version = "1.0.3";
-            sha256 =
-              "18e5d9aa5412ec063cf9719bcfe73bf990c5fed5c9a3c8422c2b5d9529fc8b0d";
-
-            erlangDeps = [ recon_2_2_1 ];
-
-            meta = {
-              description = "Visualize Erlang Nodes On The Command Line";
-              license = stdenv.lib.licenses.mit;
-              homepage = "https://github.com/zhongwencool/observer_cli";
-            };
-          }
-      ) {};
-
-    observer_cli = observer_cli_1_0_3;
-
     p1_stringprep_1_0_0 = callPackage
       (
         {  buildHex, p1_utils_1_0_1 }:
@@ -2702,26 +2630,6 @@ let
 
     pqueue = pqueue_1_5_1;
 
-    proper_1_1_1_beta = callPackage
-      (
-        {  buildHex }:
-          buildHex {
-            name = "proper";
-            version = "1.1.1-beta";
-            sha256 =
-              "bde5c0fef0f8d804a7c06aab4f293d19f42149e5880b3412b75efa608e86d342";
-
-            meta = {
-              description =
-                "QuickCheck-inspired property-based testing tool for Erlang.";
-              license = stdenv.lib.licenses.free;
-              homepage = "https://github.com/manopapad/proper";
-            };
-          }
-      ) {};
-
-    proper = proper_1_1_1_beta;
-
     providers_1_6_0 = callPackage
       (
         {  buildHex, getopt_0_8_2 }:

pkgs/development/tools/build-managers/rebar3/default.nix

@@ -1,5 +1,5 @@
 { stdenv, writeText, callPackage, fetchurl,
-  fetchHex, erlang, rebar3-nix-bootstrap, tree, fetchFromGitHub }:
+  fetchHex, erlang, hermeticRebar3 ? true, rebar3-nix-bootstrap, tree, fetchFromGitHub }:
 
 
 let
@@ -67,6 +67,12 @@ let
     version = "0.2.0";
     sha256 = "03kiszlbgzscfd2ns7na6bzbfzmcqdb5cx3p6qy3657jk2fai332";
   };
+  # {eunit_formatters,    "0.2.0"}
+  rebar3_hex = fetchHex {
+    pkg = "rebar3_hex";
+    version = "1.12.0";
+    sha256 = "45467e93ae8d776c6038fdaeaffbc55d8f2f097f300a54dab9b81c6d1cf21f73";
+  };
 
 in
 stdenv.mkDerivation {
@@ -78,7 +84,9 @@ stdenv.mkDerivation {
     sha256 = "0px66scjdia9aaa5z36qzxb848r56m0k98g0bxw065a2narsh4xy";
   };
 
-  patches = [ ./hermetic-bootstrap.patch ];
+  patches = if hermeticRebar3 == true
+  then  [ ./hermetic-bootstrap.patch ./hermetic-rebar3.patch ]
+  else [];
 
   buildInputs = [ erlang tree  ];
   propagatedBuildInputs = [ registrySnapshot rebar3-nix-bootstrap ];
@@ -88,6 +96,7 @@ stdenv.mkDerivation {
     rebar3-nix-bootstrap registry-only
     echo "$ERL_LIBS"
     mkdir -p _build/default/lib/
+    mkdir -p _build/default/plugins
     cp --no-preserve=mode -R ${erlware_commons} _build/default/lib/erlware_commons
     cp --no-preserve=mode -R ${providers} _build/default/lib/providers
     cp --no-preserve=mode -R ${getopt} _build/default/lib/getopt
@@ -98,6 +107,7 @@ stdenv.mkDerivation {
     cp --no-preserve=mode -R ${eunit_formatters} _build/default/lib/eunit_formatters
     cp --no-preserve=mode -R ${relx} _build/default/lib/relx
     cp --no-preserve=mode -R ${ssl_verify_hostname} _build/default/lib/ssl_verify_hostname
+    cp --no-preserve=mode -R ${rebar3_hex} _build/default/plugins/rebar3_hex
   '';
 
   buildPhase = ''

pkgs/development/tools/build-managers/rebar3/hermetic-rebar3.patch

@@ -0,0 +1,108 @@
+diff --git a/src/rebar3.erl b/src/rebar3.erl
+index 2b73844..af1d871 100644
+--- a/src/rebar3.erl
++++ b/src/rebar3.erl
+@@ -282,9 +282,11 @@ start_and_load_apps(Caller) ->
+     ensure_running(crypto, Caller),
+     ensure_running(asn1, Caller),
+     ensure_running(public_key, Caller),
+-    ensure_running(ssl, Caller),
+-    inets:start(),
+-    inets:start(httpc, [{profile, rebar}]).
++    ensure_running(ssl, Caller).
++%%    Removed due to the hermicity requirements of Nix
++%%
++%%    inets:start(),
++%%    inets:start(httpc, [{profile, rebar}]).
+ 
+ ensure_running(App, Caller) ->
+     case application:start(App) of
+@@ -339,4 +341,4 @@ safe_define_test_macro(Opts) ->
+ test_defined([{d, 'TEST'}|_]) -> true;
+ test_defined([{d, 'TEST', true}|_]) -> true;
+ test_defined([_|Rest]) -> test_defined(Rest);
+-test_defined([]) -> false.
+\ No newline at end of file
++test_defined([]) -> false.
+diff --git a/src/rebar_hermicity.erl b/src/rebar_hermicity.erl
+new file mode 100644
+index 0000000..d814e2a
+--- /dev/null
++++ b/src/rebar_hermicity.erl
+@@ -0,0 +1,42 @@
++%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
++%% ex: ts=4 sw=4 et
++%% -------------------------------------------------------------------
++%%
++%% rebar: Erlang Build Tools
++%%
++%% Copyright (c) 2016 Eric Merritt (eric@merritt.tech)
++%%
++%% Permission is hereby granted, free of charge, to any person obtaining a copy
++%% of this software and associated documentation files (the "Software"), to deal
++%% in the Software without restriction, including without limitation the rights
++%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
++%% copies of the Software, and to permit persons to whom the Software is
++%% furnished to do so, subject to the following conditions:
++%%
++%% The above copyright notice and this permission notice shall be included in
++%% all copies or substantial portions of the Software.
++%%
++%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
++%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
++%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
++%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
++%% THE SOFTWARE.
++%% -------------------------------------------------------------------
++-module(rebar_hermicity).
++
++-export([request/5]).
++
++-include("rebar.hrl").
++
++%% ====================================================================
++%% Public API
++%% ====================================================================
++
++request(Method, {Url, _Headers}, _HTTPOptions, _Options, _Profile) ->
++    ?ERROR("A request is being made that violates Nix hermicity "
++           "This request has been stopped. Details of the request "
++           "are as follows:", []),
++    ?ERROR("Requesnt: ~p ~s", [Method, Url]),
++    erlang:halt(1).
+diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl
+index 4f55ad1..f76fd5d 100644
+--- a/src/rebar_pkg_resource.erl
++++ b/src/rebar_pkg_resource.erl
+@@ -100,10 +100,10 @@ make_vsn(_) ->
+     {error, "Replacing version of type pkg not supported."}.
+ 
+ request(Url, ETag) ->
+-    case httpc:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]},
+-                       [{ssl, ssl_opts(Url)}, {relaxed, true}],
+-                       [{body_format, binary}],
+-                       rebar) of
++    case rebar_hermicity:request(get, {Url, [{"if-none-match", ETag} || ETag =/= false]},
++                                 [{ssl, ssl_opts(Url)}, {relaxed, true}],
++                                 [{body_format, binary}],
++                                 rebar) of
+         {ok, {{_Version, 200, _Reason}, Headers, Body}} ->
+             ?DEBUG("Successfully downloaded ~s", [Url]),
+             {"etag", ETag1} = lists:keyfind("etag", 1, Headers),
+diff --git a/src/rebar_prv_update.erl b/src/rebar_prv_update.erl
+index 6637ebe..d82c1d8 100644
+--- a/src/rebar_prv_update.erl
++++ b/src/rebar_prv_update.erl
+@@ -44,8 +44,8 @@ do(State) ->
+         TmpFile = filename:join(TmpDir, "packages.gz"),
+ 
+         Url = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_HEX_REGISTRY),
+-        case httpc:request(get, {Url, []},
+-                           [], [{stream, TmpFile}, {sync, true}],
++        case rebar_hermicity:request(get, {Url, []},
++                                     [], [{stream, TmpFile}, {sync, true}],
+                            rebar) of
+             {ok, saved_to_file} ->
+                 {ok, Data} = file:read_file(TmpFile),

pkgs/top-level/all-packages.nix

@@ -5096,7 +5096,8 @@ let
   erlang_odbc_javac = erlangR18_odbc_javac;
 
   rebar = callPackage ../development/tools/build-managers/rebar { };
-  rebar3 = callPackage ../development/tools/build-managers/rebar3 { };
+  rebar3-open = callPackage ../development/tools/build-managers/rebar3 { hermeticRebar3 = false; };
+  rebar3 = callPackage ../development/tools/build-managers/rebar3 { hermeticRebar3 = true; };
   rebar3-nix-bootstrap = callPackage ../development/tools/erlang/rebar3-nix-bootstrap { };
   fetchHex = callPackage ../development/tools/build-managers/rebar3/fetch-hex.nix { };