Merge pull request #99198 from timstott/terraform-013-compatible-providers

terraform-providers: Terraform 0.13 compatibility

pkgs/applications/networking/cluster/terraform-providers/default.nix

@@ -2,11 +2,16 @@
 , buildGoPackage
 , fetchFromGitHub
 , callPackage
+, runtimeShell
 }:
 let
-  list = import ./data.nix;
+  list = lib.importJSON ./providers.json;
 
-  toDrv = data:
+  toDrv = name: data:
+    let
+      fallbackProviderSourceAddress = "nixpkgs/${data.owner}/${name}";
+      providerSourceAddress = data.provider-source-address or fallbackProviderSourceAddress;
+    in
     buildGoPackage rec {
       inherit (data) owner repo rev version sha256;
       name = "${repo}-${version}";
@@ -18,6 +23,9 @@ let
       # Terraform allow checking the provider versions, but this breaks
       # if the versions are not provided via file paths.
       postBuild = "mv $NIX_BUILD_TOP/go/bin/${repo}{,_v${version}}";
+      passthru = {
+        inherit providerSourceAddress;
+      };
     };
 
   # Google is now using the vendored go modules, which works a bit differently
@@ -48,7 +56,7 @@ let
     });
 
   # These providers are managed with the ./update-all script
-  automated-providers = lib.mapAttrs (_: toDrv) list;
+  automated-providers = lib.mapAttrs (toDrv) list;
 
   # These are the providers that don't fall in line with the default model
   special-providers = {
@@ -57,6 +65,13 @@ let
     google-beta = patchGoModVendor automated-providers.google-beta;
     ibm = patchGoModVendor automated-providers.ibm;
 
+    acme = automated-providers.acme.overrideAttrs (attrs: {
+      prePatch = attrs.prePatch or "" + ''
+        substituteInPlace go.mod --replace terraform-providers/terraform-provider-acme getstackhead/terraform-provider-acme
+        substituteInPlace main.go --replace terraform-providers/terraform-provider-acme getstackhead/terraform-provider-acme
+      '';
+    });
+
     # providers that were moved to the `hashicorp` organization,
     # but haven't updated their references yet:
 

pkgs/applications/networking/cluster/terraform-providers/update-all

@@ -1,177 +0,0 @@
-#!/usr/bin/env nix-shell
-#! nix-shell -i bash -p bash coreutils jq nix gitAndTools.hub
-# vim: ft=sh sw=2 et
-# shellcheck shell=bash
-#
-# This scripts scans the github terraform-providers repo for new releases,
-# generates the corresponding nix code and finally generates an index of
-# all the providers given in ./providers.txt.
-set -euo pipefail
-
-# the maximum number of attempts before giving up inside of GET and prefetch_github
-readonly maxAttempts=30
-
-get_tf_providers_org() {
-  # returns all terraform providers in a given organization, and their the
-  # latest tags, in the format
-  # $org/$repo $rev
-  local org=$1
-  hub api --paginate graphql -f query="
-    query(\$endCursor: String) {
-      repositoryOwner(login: \"${org}\") {
-        repositories(first: 100, after: \$endCursor) {
-          nodes {
-            nameWithOwner
-            name
-            refs(first: 1, refPrefix: \"refs/tags/\", orderBy: {field: TAG_COMMIT_DATE, direction: DESC}) {
-              nodes {
-                name
-              }
-            }
-          }
-          pageInfo {
-            hasNextPage
-            endCursor
-          }
-        }
-      }
-  }" | \
-  jq -r '.data.repositoryOwner.repositories.nodes[] | select(.name | startswith("terraform-provider-")) | select((.refs.nodes | length) > 0) | .nameWithOwner + " " + .refs.nodes[0].name'
-  # filter the result with jq:
-  # - repos need to start with `teraform-provider-`
-  # - they need to have at least one tag
-  # for each of the remaining repos, assemble a string $org/$repo $rev
-}
-
-get_latest_repo_tag() {
-  # of a given repo and owner, retrieve the latest tag
-  local owner=$1
-  local repo=$2
-  hub api --paginate "https://api.github.com/repos/$owner/$repo/git/refs/tags" | \
-    jq -r '.[].ref' | \
-    grep -v 'v\.' | \
-    cut -d '/' -f 3- | \
-    sort --version-sort | \
-    tail -1
-}
-
-prefetch_github() {
-  # of a given owner, repo and rev, fetch the tarball and return the output of
-  # `nix-prefetch-url`
-  local owner=$1
-  local repo=$2
-  local rev=$3
-  local retry=1
-  while ! nix-prefetch-url --unpack "https://github.com/$owner/$repo/archive/$rev.tar.gz"; do
-    echo "The nix-prefetch-url command has failed. Attempt $retry/${maxAttempts}" >&2
-    if [[ "${retry}" -eq "${maxAttempts}" ]]; then
-      exit 1
-    fi
-    retry=$(( retry + 1 ))
-    sleep 5
-  done
-}
-
-echo_entry() {
-  local owner=$1
-  local repo=$2
-  local rev=$3
-  local version=${rev#v}
-  local sha256=$4
-  cat <<EOF
-{
-  owner   = "$owner";
-  repo    = "$repo";
-  rev     = "$rev";
-  version = "$version";
-  sha256  = "$sha256";
-};
-EOF
-}
-
-indent() { sed 's/^/    /'; }
-
-add_provider() {
-  org="${1}"
-  repo="${2}"
-  rev="${3}"
-
-  echo "*** $org/$repo $rev ***"
-  name=$(echo "$repo" | cut -d - -f 3-)
-  sha256=$(prefetch_github "$org" "$repo" "$rev")
-
-  {
-    echo "  $name ="
-    echo_entry "$org" "$repo" "$rev" "$sha256" | indent
-  } >> data.nix
-}
-
-## Main ##
-
-cd "$(dirname "$0")"
-
-# individual repos to fetch
-slugs=(
-  IBM-Cloud/terraform-provider-ibm
-  ajbosco/terraform-provider-segment
-  camptocamp/terraform-provider-pass
-  carlpett/terraform-provider-sops
-  poseidon/terraform-provider-matchbox
-  poseidon/terraform-provider-ct
-  tweag/terraform-provider-nixos
-  tweag/terraform-provider-secret
-)
-
-# a list of providers to ignore
-blacklist=(
-  terraform-providers/terraform-provider-azure-classic
-  terraform-providers/terraform-provider-cidr
-  terraform-providers/terraform-provider-circonus
-  terraform-providers/terraform-provider-cloudinit
-  terraform-providers/terraform-provider-quorum
-  hashicorp/terraform-provider-time
-  terraform-providers/terraform-provider-vmc
-)
-
-cat <<HEADER > data.nix
-# Generated with ./update-all
-{
-HEADER
-
-# assemble list of terraform providers
-providers=$(get_tf_providers_org "terraform-providers")
-providers=$(echo "$providers";get_tf_providers_org "hashicorp")
-
-# add terraform-providers from slugs
-for slug in "${slugs[@]}"; do
-  # retrieve latest tag
-  org=${slug%/*}
-  repo=${slug#*/}
-  rev=$(get_latest_repo_tag "$org" "$repo")
-
-  # add to list
-  providers=$(echo "$providers";echo "$org/$repo $rev")
-done
-
-# filter out all providers on the blacklist
-for repo in "${blacklist[@]}"; do
-  providers=$(echo "$providers" | grep -v "^${repo} ")
-done
-
-# sort results alphabetically by repo name
-providers=$(echo "$providers" | sort -t "/" --key=2)
-
-# render list
-IFS=$'\n'
-for provider in $providers; do
-  org=$(echo "$provider" | cut -d " " -f 1 | cut -d "/" -f1)
-  repo=$(echo "$provider" | cut -d " " -f 1 | cut -d "/" -f2)
-  rev=$(echo "$provider" | cut -d " " -f 2)
-  add_provider "${org}" "${repo}" "${rev}"
-done
-
-cat <<FOOTER >> data.nix
-}
-FOOTER
-
-echo Done.

pkgs/applications/networking/cluster/terraform-providers/update-all-providers

@@ -0,0 +1,22 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash -p jq
+# shellcheck shell=bash
+
+# Update all providers which have specified provider source address
+set -euo pipefail
+
+providers=$(
+  jq -r 'to_entries
+  | map_values(.value + { alias: .key })
+  | .[]
+  | select(."provider-source-address"?)
+  | .alias' providers.json
+)
+
+echo "Will update providers:"
+echo "$providers"
+
+for provider in $providers; do
+  echo "Updating $provider"
+  ./update-provider "$provider"
+done

pkgs/applications/networking/cluster/terraform-providers/update-provider

@@ -0,0 +1,77 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash -p coreutils curl jq
+# shellcheck shell=bash
+#
+# Update a terraform provider to the latest version advertised at
+# the provider source address.
+set -euo pipefail
+
+USAGE=$(cat<<DOC
+Specify the terraform provider name to update.
+
+Example:
+To update nixpkgs.terraform-providers.aws run:
+./update-provider aws
+DOC
+)
+
+provider_name="${1:-}"
+if [ -z "$provider_name" ]; then
+  echo "No providers specified!"
+  echo
+  echo "$USAGE"
+  exit 1
+fi
+
+provider_source_address="$(jq -r ".$provider_name.\"provider-source-address\"" providers.json)"
+
+if [ "$provider_source_address" == "null" ]; then
+  echo "No provider source address specified with provider: $provider_name"
+  exit 1
+fi
+
+# The provider source address (used inside Terraform `required_providers` block) is
+# used to compute the registry API endpoint
+#
+# registry.terraform.io/hashicorp/aws (provider source address)
+# registry.terraform.io/providers/hashicorp/aws (provider URL for the website)
+# registry.terraform.io/v1/providers/hashicorp/aws (provider URL for the JSON API)
+registry_response=$(curl -s https://"${provider_source_address/\///v1/providers/}")
+
+prefetch_github() {
+  # of a given owner, repo and rev, fetch the tarball and return the output of
+  # `nix-prefetch-url`
+  local owner=$1
+  local repo=$2
+  local rev=$3
+  nix-prefetch-url --unpack "https://github.com/$owner/$repo/archive/$rev.tar.gz"
+}
+
+provider_source_url="$(jq -r '.source' <<< "$registry_response")"
+
+org="$(echo "$provider_source_url" | cut -d '/' -f 4)"
+repo="$(echo "$provider_source_url" | cut -d '/' -f 5)"
+rev="$(jq -r '.tag' <<< "$registry_response")"
+
+sha256=$(prefetch_github "$org" "$repo" "$rev")
+
+version="$(jq -r '.version' <<< "$registry_response")"
+
+updated_provider="$(mktemp)"
+cat <<EOF >> "$updated_provider"
+{
+  "$provider_name": {
+    "owner": "$org",
+    "repo": "$repo",
+    "rev": "$rev",
+    "sha256": "$sha256",
+    "version": "$version",
+    "provider-source-address": "$provider_source_address"
+  }
+}
+EOF
+
+original_provider_list="$(mktemp)"
+cat providers.json > "$original_provider_list"
+
+jq --sort-keys --slurp '.[0] * .[1]' "$original_provider_list" "$updated_provider" > providers.json

pkgs/applications/networking/cluster/terraform/default.nix

@@ -1,5 +1,5 @@
 { stdenv, lib, buildEnv, buildGoPackage, fetchFromGitHub, makeWrapper, coreutils
-, runCommand, writeText, terraform-providers, fetchpatch }:
+, runCommand, runtimeShell, writeText, terraform-providers, fetchpatch }:
 
 let
   goPackagePath = "github.com/hashicorp/terraform";
@@ -43,12 +43,13 @@ let
         homepage = "https://www.terraform.io/";
         license = licenses.mpl20;
         maintainers = with maintainers; [
-          zimbatm
+          Chili-Man
-          peterhoeg
+          babariviere
           kalbasit
           marsam
-          babariviere
+          peterhoeg
-          Chili-Man
+          timstott
+          zimbatm
         ];
       };
     } // attrs');
@@ -59,6 +60,29 @@ let
         let
           actualPlugins = plugins terraform.plugins;
 
+          # Make providers available in Terraform 0.13 and 0.12 search paths.
+          pluginDir = lib.concatMapStrings (pl: let
+            inherit (pl) repo version GOOS GOARCH;
+            inherit (pl.passthru) providerSourceAddress;
+
+            shim = writeText "shim" ''
+              #!${runtimeShell}
+              exec ${pl}/bin/${repo}_v${version} \$@
+            '';
+            in ''
+              TF_0_13_PROVIDER_PATH=$out/plugins/${providerSourceAddress}/${version}/${GOOS}_${GOARCH}/${repo}_v${version}
+              mkdir -p "$(dirname $TF_0_13_PROVIDER_PATH)"
+
+              cp ${shim} "$TF_0_13_PROVIDER_PATH"
+              chmod +x "$TF_0_13_PROVIDER_PATH"
+
+              TF_0_12_PROVIDER_PATH=$out/plugins/${repo}_v${version}
+
+              cp ${shim} "$TF_0_12_PROVIDER_PATH"
+              chmod +x "$TF_0_12_PROVIDER_PATH"
+          ''
+          ) actualPlugins;
+
           # Wrap PATH of plugins propagatedBuildInputs, plugins may have runtime dependencies on external binaries
           wrapperInputs = lib.unique (lib.flatten
             (lib.catAttrs "propagatedBuildInputs"
@@ -87,15 +111,10 @@ let
             inherit (terraform) name;
             buildInputs = [ makeWrapper ];
 
-            buildCommand = ''
+            buildCommand = pluginDir + ''
               mkdir -p $out/bin/
               makeWrapper "${terraform}/bin/terraform" "$out/bin/terraform" \
-                --set NIX_TERRAFORM_PLUGIN_DIR "${
+                --set NIX_TERRAFORM_PLUGIN_DIR $out/plugins \
-                  buildEnv {
-                    name = "tf-plugin-env";
-                    paths = actualPlugins;
-                  }
-                }/bin" \
                 --prefix PATH : "${lib.makeBinPath wrapperInputs}"
             '';