public/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

terraform-providers: handle go modules (#104667)

Browse Source 
* terraform-providers: handle go modules

More and more terraform providers are switching from vendored
dependencies to Go modules.

Let's say that you update the "aws" provider and it fails. You can run
it again with:

    ./update-provider aws --vendor

Any package that has the "vendorSha256" key will be considered as a go
module package.

The script now also supports adding new providers by using the
<owner>/<repo> format. Eg:

     ./update-provider hetznercloud/hcloud --vendor

* address comment

Fixes https://github.com/NixOS/nixpkgs/pull/104667#discussion_r529788569

* support the null use-case

* escape provider name as well

* fix typo
This commit is contained in:
Jonas Chevalier 2020-12-01 10:05:00 +00:00 committed by GitHub
parent c2f1e7b1b0
commit 5851c31127
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 144 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

187
pkgs/applications/networking/cluster/terraform-providers/update-provider
View File

@ -1,42 +1,94 @@
#!/usr/bin/env nix-shell #!/usr/bin/env nix-shell
#! nix-shell -i bash -p coreutils curl jq #! nix-shell -i bash -p coreutils curl jq moreutils
# shellcheck shell=bash # shellcheck shell=bash
# vim: ft=sh
#
# Update a terraform provider to the latest version advertised at the
# provider source address.
# #
# Update a terraform provider to the latest version advertised at
# the provider source address.
set -euo pipefail set -euo pipefail
USAGE=$(cat<<DOC show_usage() {
Specify the terraform provider name to update. cat <<DOC
Usage: ./update-provider [--force] [--vendor] [<owner>/]<provider>
Example: Update a single provider in the providers.json inventory file.
To update nixpkgs.terraform-providers.aws run:
./update-provider aws For example to update 'terraform-providers.aws' run:
./update-provider aws
If the provider is not in the list already, use the form '<owner>/<provider>'
to add the provider to the list:
./update-provider hetznercloud/hcloud
Options:
* --force: Force the update even if the version matches.
* --vendor: Switch from go package to go modules with vendor.
* --vendor-sha256 <sha256>: Override the SHA256 or "null".
DOC DOC
) }
provider_name="${1:-}" force=
if [ -z "$provider_name" ]; then provider=
echo "No providers specified!" vendor=
vendorSha256=
while [[ $# -gt 0 ]]; do
case "$1" in
-h | --help)
show_usage
exit
;;
--force)
force=1
shift
;;
--vendor)
force=1
vendor=1
shift
;;
--vendor-sha256)
force=1
vendorSha256=$2
shift 2
;;
*)
if [[ -n "$provider" ]]; then
echo "ERROR: provider name was passed two times: '$provider' and '$1'"
echo "Use --help for more info"
exit 1
fi
provider=$1
shift
esac
done
if [[ -z "$provider" ]]; then
echo "ERROR: No providers specified!"
echo echo
echo "$USAGE" show_usage
exit 1 exit 1
fi fi
provider_source_address="$(jq -r ".$provider_name.\"provider-source-address\"" providers.json)" provider_name=$(basename "$provider")
if [ "$provider_source_address" == "null" ]; then # Usage: read_attr <key>
echo "No provider source address specified with provider: $provider_name" read_attr() {
exit 1 jq -r ".\"$provider_name\".\"$1\"" providers.json
fi }
# The provider source address (used inside Terraform `required_providers` block) is # Usage: update_attr <key> <value>
# used to compute the registry API endpoint update_attr() {
# if [[ "$2" == "null" ]]; then
# registry.terraform.io/hashicorp/aws (provider source address) jq -S ".\"$provider_name\".\"$1\" = null" providers.json | sponge providers.json
# registry.terraform.io/providers/hashicorp/aws (provider URL for the website) else
# registry.terraform.io/v1/providers/hashicorp/aws (provider URL for the JSON API) jq -S ".\"$provider_name\".\"$1\" = \"$2\"" providers.json | sponge providers.json
registry_response=$(curl -s https://"${provider_source_address/\///v1/providers/}") fi
}
prefetch_github() { prefetch_github() {
# of a given owner, repo and rev, fetch the tarball and return the output of # of a given owner, repo and rev, fetch the tarball and return the output of
@ -47,31 +99,80 @@ prefetch_github() {
nix-prefetch-url --unpack "https://github.com/$owner/$repo/archive/$rev.tar.gz" nix-prefetch-url --unpack "https://github.com/$owner/$repo/archive/$rev.tar.gz"
} }
old_source_address="$(read_attr provider-source-address)"
old_vendor_sha256=$(read_attr vendorSha256)
old_version=$(read_attr version)
if [[ $provider =~ ^[^/]+/[^/]+$ ]]; then
source_address=registry.terraform.io/$provider
else
source_address=$old_source_address
fi
if [[ "$source_address" == "null" ]]; then
echo "Could not find the source address for provider: $provider"
exit 1
fi
update_attr "provider-source-address" "$source_address"
# The provider source address (used inside Terraform `required_providers` block) is
# used to compute the registry API endpoint
#
# registry.terraform.io/hashicorp/aws (provider source address)
# registry.terraform.io/providers/hashicorp/aws (provider URL for the website)
# registry.terraform.io/v1/providers/hashicorp/aws (provider URL for the JSON API)
registry_response=$(curl -s https://"${source_address/\///v1/providers/}")
version="$(jq -r '.version' <<< "$registry_response")"
if [[ "$old_version" = "$version" && "$force" != 1 && -z "$vendorSha256" && "$old_vendor_sha256" != "$vendorSha256" ]]; then
echo "$provider_name is already at version $version"
exit
fi
update_attr version "$version"
provider_source_url="$(jq -r '.source' <<< "$registry_response")" provider_source_url="$(jq -r '.source' <<< "$registry_response")"
org="$(echo "$provider_source_url" | cut -d '/' -f 4)" org="$(echo "$provider_source_url" | cut -d '/' -f 4)"
update_attr owner "$org"
repo="$(echo "$provider_source_url" | cut -d '/' -f 5)" repo="$(echo "$provider_source_url" | cut -d '/' -f 5)"
update_attr repo "$repo"
rev="$(jq -r '.tag' <<< "$registry_response")" rev="$(jq -r '.tag' <<< "$registry_response")"
update_attr rev "$rev"
sha256=$(prefetch_github "$org" "$repo" "$rev") sha256=$(prefetch_github "$org" "$repo" "$rev")
update_attr sha256 "$sha256"
version="$(jq -r '.version' <<< "$registry_response")" repo_root=$(git rev-parse --show-toplevel)
updated_provider="$(mktemp)" if [[ -z "$vendorSha256" ]]; then
cat <<EOF >> "$updated_provider" if [[ "$old_vendor_sha256" == null ]]; then
{ vendorSha256=null
"$provider_name": { elif [[ -n "$old_vendor_sha256" || "$vendor" = 1 ]]; then
"owner": "$org", echo "=== Calculating vendorSha256 ==="
"repo": "$repo", update_attr vendorSha256 "0000000000000000000000000000000000000000000000000000000000000000"
"rev": "$rev", # Hackish way to find out the desired sha256. First build, then extract the
"sha256": "$sha256", # error message from the logs.
"version": "$version", set +e
"provider-source-address": "$provider_source_address" nix-build --no-out-link "$repo_root" -A "terraform-providers.$provider_name.go-modules" 2>vendor_log.txt
} set -e
} logs=$(< vendor_log.txt)
EOF if ! [[ $logs =~ got:\ +([^\ ]+) ]]; then
echo "ERROR: could not find new hash in output:"
cat vendor_log.txt
rm -f vendor_log.txt
exit 1
fi
rm -f vendor_log.txt
vendorSha256=${BASH_REMATCH[1]}
# Deal with nix unstable
if [[ $vendorSha256 = sha256-* ]]; then
vendorSha256=$(nix to-base32 "$vendorSha256")
fi
fi
fi
original_provider_list="$(mktemp)" if [[ -n "$vendorSha256" ]]; then
cat providers.json > "$original_provider_list" update_attr vendorSha256 "$vendorSha256"
fi
jq --sort-keys --slurp '.[0] * .[1]' "$original_provider_list" "$updated_provider" > providers.json # Check that the provider builds
echo "=== Building terraform-providers.$provider_name ==="
nix-build "$repo_root" -A "terraform-providers.$provider_name"