Merge pull request #107623 from orivej/aws

aws-sdk-cpp: 1.7.90 -> 1.8.113

pkgs/development/libraries/aws-c-cal/default.nix

@@ -0,0 +1,30 @@
+{ lib, stdenv, fetchFromGitHub, cmake, aws-c-common, openssl }:
+
+stdenv.mkDerivation rec {
+  pname = "aws-c-cal";
+  version = "0.4.5";
+
+  src = fetchFromGitHub {
+    owner = "awslabs";
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "04acra1mnzw9q7jycs5966akfbgnx96hkrq90nq0dhw8pvarlyv6";
+  };
+
+  nativeBuildInputs = [ cmake ];
+
+  buildInputs = [ aws-c-common openssl ];
+
+  cmakeFlags = [
+    "-DBUILD_SHARED_LIBS=ON"
+    "-DCMAKE_MODULE_PATH=${aws-c-common}/lib/cmake"
+  ];
+
+  meta = with lib; {
+    description = "AWS Crypto Abstraction Layer ";
+    homepage = "https://github.com/awslabs/aws-c-cal";
+    license = licenses.asl20;
+    platforms = platforms.unix;
+    maintainers = with maintainers; [ orivej ];
+  };
+}

pkgs/development/libraries/aws-c-common/default.nix

@@ -2,17 +2,21 @@
 
 stdenv.mkDerivation rec {
   pname = "aws-c-common";
-  version = "0.3.11";
+  version = "0.4.63";
 
   src = fetchFromGitHub {
     owner = "awslabs";
     repo = pname;
     rev = "v${version}";
-    sha256 = "0a7hi4crnc3j1j39qcnd44zqdfwzw1xghcf80marx5vdf1qdzy6p";
+    sha256 = "16bc6fn1gq3nqcrzgpi2kjphq7xkkr73aljakrg89ysm6hyzyim9";
   };
 
   nativeBuildInputs = [ cmake ];
 
+  cmakeFlags = [
+    "-DBUILD_SHARED_LIBS=ON"
+  ];
+
   NIX_CFLAGS_COMPILE = lib.optionalString stdenv.isDarwin
     "-Wno-nullability-extension -Wno-typedef-redefinition";
 

pkgs/development/libraries/aws-c-event-stream/default.nix

@@ -1,19 +1,20 @@
-{ lib, stdenv, fetchFromGitHub, cmake, aws-c-common, aws-checksums, libexecinfo }:
+{ lib, stdenv, fetchFromGitHub, cmake, aws-c-cal, aws-c-common, aws-c-io, aws-checksums, s2n, libexecinfo }:
 
 stdenv.mkDerivation rec {
   pname = "aws-c-event-stream";
-  version = "0.1.1";
+  version = "0.2.6";
 
   src = fetchFromGitHub {
     owner = "awslabs";
     repo = pname;
     rev = "v${version}";
-    sha256 = "0anjynfghk3inysy21wqvhxha33xsswh3lm8pr7nx7cpj6cmr37m";
+    sha256 = "1hbri4dv924ph3cxkmwkl4kdca4dvhc9qda60rdzs6355l76k23n";
   };
 
   nativeBuildInputs = [ cmake ];
 
-  buildInputs = [ aws-c-common aws-checksums ] ++ lib.optional stdenv.hostPlatform.isMusl libexecinfo;
+  buildInputs = [ aws-c-cal aws-c-common aws-c-io aws-checksums s2n ]
+    ++ lib.optional stdenv.hostPlatform.isMusl libexecinfo;
 
   cmakeFlags = [
     "-DBUILD_SHARED_LIBS:BOOL=ON"

pkgs/development/libraries/aws-c-io/default.nix

@@ -0,0 +1,30 @@
+{ lib, stdenv, fetchFromGitHub, cmake, aws-c-cal, aws-c-common, s2n }:
+
+stdenv.mkDerivation rec {
+  pname = "aws-c-io";
+  version = "0.7.0";
+
+  src = fetchFromGitHub {
+    owner = "awslabs";
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "0wagc1205r57llqd39wqjasq3bgc8h1mfdqk4r5lcrnn4jbpcill";
+  };
+
+  nativeBuildInputs = [ cmake ];
+
+  buildInputs = [ aws-c-cal aws-c-common s2n ];
+
+  cmakeFlags = [
+    "-DBUILD_SHARED_LIBS=ON"
+    "-DCMAKE_MODULE_PATH=${aws-c-common}/lib/cmake"
+  ];
+
+  meta = with lib; {
+    description = "AWS SDK for C module for IO and TLS";
+    homepage = "https://github.com/awslabs/aws-c-io";
+    license = licenses.asl20;
+    platforms = platforms.unix;
+    maintainers = with maintainers; [ orivej ];
+  };
+}

pkgs/development/libraries/aws-checksums/default.nix

@@ -1,19 +1,24 @@
-{ lib, stdenv, fetchFromGitHub, cmake }:
+{ lib, stdenv, fetchFromGitHub, cmake, aws-c-common }:
 
 stdenv.mkDerivation rec {
   pname = "aws-checksums";
-  version = "0.1.7";
+  version = "0.1.10";
 
   src = fetchFromGitHub {
     owner = "awslabs";
     repo = pname;
     rev = "v${version}";
-    sha256 = "0am1hfzqir44zcx6y6c7jw74qvbsav8ppr9dahpdh3ac95cjf38a";
+    sha256 = "1f9scl5734pgjlsixspwljrrlndzhllwlfygdcr1gx5p0za08zjb";
   };
 
   nativeBuildInputs = [ cmake ];
 
-  cmakeFlags = [ "-DBUILD_SHARED_LIBS:BOOL=ON" ];
+  buildInputs = [ aws-c-common ];
+
+  cmakeFlags = [
+    "-DBUILD_SHARED_LIBS=ON"
+    "-DCMAKE_MODULE_PATH=${aws-c-common}/lib/cmake"
+  ];
 
   meta = with lib; {
     description = "HW accelerated CRC32c and CRC32";

pkgs/development/libraries/aws-sdk-cpp/cmake-dirs.patch

@@ -1,15 +1,5 @@
-diff --git a/cmake/AWSSDKConfig.cmake b/cmake/AWSSDKConfig.cmake
-index e87252123e..5457bd5910 100644
 --- a/cmake/AWSSDKConfig.cmake
 +++ b/cmake/AWSSDKConfig.cmake
-@@ -82,6 +82,7 @@ if (AWSSDK_ROOT_DIR)
-             )
- else()
-     find_file(AWSSDK_CORE_HEADER_FILE Aws.h
-+        "/${AWSSDK_INSTALL_INCLUDEDIR}/aws/core"
-         "/usr/${AWSSDK_INSTALL_INCLUDEDIR}/aws/core"
-         "/usr/local/${AWSSDK_INSTALL_INCLUDEDIR}/aws/core"
-         "C:/Progra~1/AWSSDK/${AWSSDK_INSTALL_INCLUDEDIR}/aws/core"
 @@ -97,14 +98,18 @@ if (NOT AWSSDK_CORE_HEADER_FILE)
      message(FATAL_ERROR "AWS SDK for C++ is missing, please install it first")
  endif()

pkgs/development/libraries/aws-sdk-cpp/default.nix

@@ -1,5 +1,5 @@
-{ lib, stdenv, fetchFromGitHub, cmake, curl, openssl, zlib, fetchpatch
+{ lib, stdenv, fetchFromGitHub, cmake, curl, openssl, s2n, zlib
-, aws-c-common, aws-c-event-stream, aws-checksums
+, aws-c-cal, aws-c-common, aws-c-event-stream, aws-c-io, aws-checksums
 , CoreAudio, AudioToolbox
 , # Allow building a limited set of APIs, e.g. ["s3" "ec2"].
   apis ? ["*"]
@@ -9,13 +9,13 @@
 
 stdenv.mkDerivation rec {
   pname = "aws-sdk-cpp";
-  version = "1.7.90";
+  version = "1.8.113";
 
   src = fetchFromGitHub {
     owner = "awslabs";
     repo = "aws-sdk-cpp";
     rev = version;
-    sha256 = "0zpqi612qmm0n53crxiisv0vdif43ymg13kafy6vv43j2wmh66ga";
+    sha256 = "0y784cjrxgrin3ck5f2lk0riyy9kv928kcb9y0gzka65imgma48c";
   };
 
   # FIXME: might be nice to put different APIs in different outputs
@@ -25,8 +25,8 @@ stdenv.mkDerivation rec {
   nativeBuildInputs = [ cmake curl ];
 
   buildInputs = [
-    curl openssl zlib
+    curl openssl s2n zlib
-    aws-c-common aws-c-event-stream aws-checksums
+    aws-c-cal aws-c-common aws-c-event-stream aws-c-io aws-checksums
   ] ++ lib.optionals (stdenv.isDarwin &&
                         ((builtins.elem "text-to-speech" apis) ||
                          (builtins.elem "*" apis)))
@@ -59,10 +59,6 @@ stdenv.mkDerivation rec {
   __darwinAllowLocalNetworking = true;
 
   patches = [
-    (fetchpatch {
-      url = "https://github.com/aws/aws-sdk-cpp/commit/42991ab549087c81cb630e5d3d2413e8a9cf8a97.patch";
-      sha256 = "0myq5cm3lvl5r56hg0sc0zyn1clbkd9ys0wr95ghw6bhwpvfv8gr";
-    })
     ./cmake-dirs.patch
   ];
 

pkgs/development/libraries/s2n/default.nix

@@ -0,0 +1,29 @@
+{ lib, stdenv, fetchFromGitHub, cmake, openssl }:
+
+stdenv.mkDerivation rec {
+  pname = "s2n";
+  version = "0.10.23";
+
+  src = fetchFromGitHub {
+    owner = "awslabs";
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "063wqpszhfcbxm7a7s6d6kinqd6b6dxij85lk9jjkrslg5fgqbki";
+  };
+
+  nativeBuildInputs = [ cmake ];
+
+  propagatedBuildInputs = [ openssl ]; # s2n-config has find_dependency(LibCrypto).
+
+  cmakeFlags = [
+    "-DBUILD_SHARED_LIBS=ON"
+  ];
+
+  meta = with lib; {
+    description = "C99 implementation of the TLS/SSL protocols";
+    homepage = "https://github.com/awslabs/s2n";
+    license = licenses.asl20;
+    platforms = platforms.unix;
+    maintainers = with maintainers; [ orivej ];
+  };
+}

pkgs/development/tools/misc/hydra/default.nix

@@ -9,6 +9,9 @@
       rev = "79d34ed7c93af2daf32cf44ee0e3e0768f13f97c";
       sha256 = "1lql899430137l6ghnhyz0ivkayy83fdr087ck2wq3gf1jv8pccj";
     };
+    patches = [
+      ./hydra-nix-receiveContents.patch
+    ];
     nix = nixFlakes;
 
     tests = {

pkgs/development/tools/misc/hydra/hydra-nix-receiveContents.patch

@@ -0,0 +1,18 @@
+Update for https://github.com/NixOS/nix/commit/faa31f40
+
+--- a/src/hydra-queue-runner/nar-extractor.cc
++++ b/src/hydra-queue-runner/nar-extractor.cc
+@@ -48,9 +48,9 @@
+-    void receiveContents(unsigned char * data, size_t len) override
++    void receiveContents(std::string_view data) override
+     {
+         assert(expectedSize);
+         assert(curMember);
+         assert(hashSink);
+-        *curMember->fileSize += len;
+-        (*hashSink)(data, len);
++        *curMember->fileSize += data.size();
++        (*hashSink)(data);
+         if (curMember->contents) {
+-            curMember->contents->append((char *) data, len);
++            curMember->contents->append(data);

pkgs/tools/package-management/nix/aws-sdk-cpp-TransferManager-ContentEncoding.patch

@@ -0,0 +1,127 @@
+From 7d58e303159b2fb343af9a1ec4512238efa147c7 Mon Sep 17 00:00:00 2001
+From: Eelco Dolstra <edolstra@gmail.com>
+Date: Mon, 6 Aug 2018 17:15:04 +0200
+Subject: [PATCH] TransferManager: Allow setting a content-encoding for S3 uploads
+
+--- a/aws-cpp-sdk-transfer/include/aws/transfer/TransferHandle.h
++++ b/aws-cpp-sdk-transfer/include/aws/transfer/TransferHandle.h
+@@ -297,6 +297,14 @@ namespace Aws
+              * Content type of the object being transferred
+              */
+             inline void SetContentType(const Aws::String& value) { std::lock_guard<std::mutex> locker(m_getterSetterLock); m_contentType = value; }
++            /**
++             * Content encoding of the object being transferred
++             */
++            inline const Aws::String GetContentEncoding() const { std::lock_guard<std::mutex> locker(m_getterSetterLock); return m_contentEncoding; }
++            /**
++             * Content type of the object being transferred
++             */
++            inline void SetContentEncoding(const Aws::String& value) { std::lock_guard<std::mutex> locker(m_getterSetterLock); m_contentEncoding = value; }
+             /**
+              * In case of an upload, this is the metadata that was placed on the object when it was uploaded.
+              * In the case of a download, this is the object metadata from the GetObject operation.
+@@ -383,6 +391,7 @@ namespace Aws
+             Aws::String m_key;
+             Aws::String m_fileName;
+             Aws::String m_contentType;
++            Aws::String m_contentEncoding;
+             Aws::String m_versionId;
+             Aws::Map<Aws::String, Aws::String> m_metadata;
+             TransferStatus m_status;
+--- a/aws-cpp-sdk-transfer/include/aws/transfer/TransferManager.h
++++ b/aws-cpp-sdk-transfer/include/aws/transfer/TransferManager.h
+@@ -154,7 +154,8 @@ namespace Aws
+                                                        const Aws::String& keyName,
+                                                        const Aws::String& contentType, 
+                                                        const Aws::Map<Aws::String, Aws::String>& metadata,
+-                                                       const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context = nullptr);
++                                                       const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context = nullptr,
++                                                       const Aws::String& contentEncoding = "");
+ 
+             /**
+              * Downloads the contents of bucketName/keyName in S3 to the file specified by writeToFile. This will perform a GetObject operation.
+@@ -246,7 +247,8 @@ namespace Aws
+                                                                    const Aws::Map<Aws::String,
+                                                                    Aws::String>& metadata,
+                                                                    const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context,
+-                                                                   const Aws::String& fileName = "");
++                                                                   const Aws::String& fileName = "",
++                                                                   const Aws::String& contentEncoding = "");
+ 
+             /**
+              * Submits the actual task to task schecduler
+@@ -262,7 +264,8 @@ namespace Aws
+                                                          const Aws::String& keyName,
+                                                          const Aws::String& contentType,
+                                                          const Aws::Map<Aws::String, Aws::String>& metadata,
+-                                                         const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context);
++                                                         const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context,
++                                                         const Aws::String& contentEncoding);
+ 
+             /**
+              * Uploads the contents of file, to bucketName/keyName in S3. contentType and metadata will be added to the object. If the object is larger than the configured bufferSize,
+--- a/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp
++++ b/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp
+@@ -87,9 +87,10 @@ namespace Aws
+                                                                     const Aws::String& bucketName,
+                                                                     const Aws::String& keyName, const Aws::String& contentType,
+                                                                     const Aws::Map<Aws::String, Aws::String>& metadata,
+-                                                                    const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context)
++                                                                    const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context,
++                                                                    const Aws::String& contentEncoding)
+         {
+-            return this->DoUploadFile(fileStream, bucketName, keyName, contentType, metadata, context);
++            return this->DoUploadFile(fileStream, bucketName, keyName, contentType, metadata, context, contentEncoding);
+         }
+ 
+         std::shared_ptr<TransferHandle> TransferManager::DownloadFile(const Aws::String& bucketName,
+@@ -286,6 +287,9 @@ namespace Aws
+                 createMultipartRequest.WithKey(handle->GetKey());
+                 createMultipartRequest.WithMetadata(handle->GetMetadata());
+ 
++                if (handle->GetContentEncoding() != "")
++                    createMultipartRequest.WithContentEncoding(handle->GetContentEncoding());
++
+                 auto createMultipartResponse = m_transferConfig.s3Client->CreateMultipartUpload(createMultipartRequest);
+                 if (createMultipartResponse.IsSuccess())
+                 {
+@@ -441,6 +445,9 @@ namespace Aws
+ 
+             putObjectRequest.SetContentType(handle->GetContentType());
+ 
++            if (handle->GetContentEncoding() != "")
++                putObjectRequest.SetContentEncoding(handle->GetContentEncoding());
++
+             auto buffer = m_bufferManager.Acquire();
+ 
+             auto lengthToWrite = (std::min)(m_transferConfig.bufferSize, handle->GetBytesTotalSize());
+@@ -1140,12 +1147,15 @@ namespace Aws
+                                                                                 const Aws::String& contentType,
+                                                                                 const Aws::Map<Aws::String, Aws::String>& metadata,
+                                                                                 const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context,
+-                                                                                const Aws::String& fileName)
++                                                                                const Aws::String& fileName,
++                                                                                const Aws::String& contentEncoding)
+         {
+             auto handle = Aws::MakeShared<TransferHandle>(CLASS_TAG, bucketName, keyName, 0, fileName);
+             handle->SetContentType(contentType);
+             handle->SetMetadata(metadata);
+             handle->SetContext(context);
++            if (contentEncoding != "")
++                handle->SetContentEncoding(contentEncoding);
+ 
+             if (!fileStream->good())
+             {
+@@ -1213,9 +1223,10 @@ namespace Aws
+                                                                       const Aws::String& keyName,
+                                                                       const Aws::String& contentType,
+                                                                       const Aws::Map<Aws::String, Aws::String>& metadata,
+-                                                                      const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context)
++                                                                      const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context,
++                                                                      const Aws::String& contentEncoding)
+         {
+-            auto handle = CreateUploadFileHandle(fileStream.get(), bucketName, keyName, contentType, metadata, context);
++            auto handle = CreateUploadFileHandle(fileStream.get(), bucketName, keyName, contentType, metadata, context, "", contentEncoding);
+             return SubmitUpload(handle, fileStream);
+         }
+ 

pkgs/tools/package-management/nix/default.nix

@@ -1,4 +1,4 @@
-{ lib, fetchurl, fetchpatch, fetchFromGitHub, callPackage
+{ lib, fetchurl, fetchFromGitHub, fetchpatch, callPackage
 , storeDir ? "/nix/store"
 , stateDir ? "/nix/var"
 , confDir ? "/etc"
@@ -8,7 +8,7 @@
 let
 
 common =
-  { lib, stdenv, fetchpatch, perl, curl, bzip2, sqlite, openssl ? null, xz
+  { lib, stdenv, perl, curl, bzip2, sqlite, openssl ? null, xz
   , bash, coreutils, gzip, gnutar
   , pkgconfig, boehmgc, perlPackages, libsodium, brotli, boost, editline, nlohmann_json
   , autoreconfHook, autoconf-archive, bison, flex
@@ -23,8 +23,8 @@ common =
   , withLibseccomp ? lib.any (lib.meta.platformMatch stdenv.hostPlatform) libseccomp.meta.platforms, libseccomp
   , withAWS ? !enableStatic && (stdenv.isLinux || stdenv.isDarwin), aws-sdk-cpp
   , enableStatic ? stdenv.hostPlatform.isStatic
-  , name, suffix ? "", src, patches ? []
+  , name, suffix ? "", src
-
+  , patches ? [ ]
   }:
   let
      sh = busybox-sandbox-shell;
@@ -60,10 +60,9 @@ common =
               apis = ["s3" "transfer"];
               customMemoryManagement = false;
             }).overrideDerivation (args: {
-              patches = args.patches or [] ++ [(fetchpatch {
+              patches = args.patches or [] ++ [
-                url = "https://github.com/edolstra/aws-sdk-cpp/commit/7d58e303159b2fb343af9a1ec4512238efa147c7.patch";
+                ./aws-sdk-cpp-TransferManager-ContentEncoding.patch
-                sha256 = "103phn6kyvs1yc7fibyin3lgxz699qakhw671kl207484im55id1";
+              ];
-              })];
             }));
 
       propagatedBuildInputs = [ boehmgc ];
@@ -199,18 +198,25 @@ in rec {
       sha256 = "a8a85e55de43d017abbf13036edfb58674ca136691582f17080c1cd12787b7ab";
     };
 
+    patches = [(
+      fetchpatch {
+        url = "https://github.com/NixOS/nix/pull/4316.patch";
+        sha256 = "0bqlm4n9sac9prgr9xlfng92arisp1hiqvc9pfh4fibsppkgdfc5";
+      }
+    )];
+
     inherit storeDir stateDir confDir boehmgc;
   });
 
   nixUnstable = lib.lowPrio (callPackage common rec {
     name = "nix-2.4${suffix}";
-    suffix = "pre20201201_5a6ddb3";
+    suffix = "pre20201205_a5d85d0";
 
     src = fetchFromGitHub {
       owner = "NixOS";
       repo = "nix";
-      rev = "5a6ddb3de14a1684af6c793d663764d093fa7846";
+      rev = "a5d85d07faa94cf3518e98273be4bee3d495f06a";
-      sha256 = "0qhd3nxvqzszzsfvh89xhd239ycqb0kq2n0bzh9br78pcb60vj3g";
+      sha256 = "0g9jjhh0vs4hjrff5yx88x6sh7rk87ngvni3gnyxajqia957dipg";
     };
 
     patches = [

pkgs/top-level/all-packages.nix

@@ -12578,10 +12578,14 @@ in
     inherit (darwin.apple_sdk.frameworks) AudioUnit CoreServices;
   };
 
+  aws-c-cal = callPackage ../development/libraries/aws-c-cal { };
+
   aws-c-common = callPackage ../development/libraries/aws-c-common { };
 
   aws-c-event-stream = callPackage ../development/libraries/aws-c-event-stream { };
 
+  aws-c-io = callPackage ../development/libraries/aws-c-io { };
+
   aws-checksums = callPackage ../development/libraries/aws-checksums { };
 
   aws-sdk-cpp = callPackage ../development/libraries/aws-sdk-cpp {
@@ -16150,6 +16154,8 @@ in
 
   graphite2 = callPackage ../development/libraries/silgraphite/graphite2.nix {};
 
+  s2n = callPackage ../development/libraries/s2n { };
+
   simavr = callPackage ../development/tools/simavr {
     avrgcc = pkgsCross.avr.buildPackages.gcc;
     avrlibc = pkgsCross.avr.libcCross;