terraform-providers: refactor

Some terraform providers were moved to the `hashicorp` organization,
some are still in the `terraform-providers` organization.

GitHub doesn't show moved repositories when querying via the API (only
on the web interface). In addition, if there are more than 100 results
(there still are), we'd need to implement paging, which is cubersome in curl.

In addition to that, to look up tags, we used to creating a HTTP request
for each and every provider, which caused us to easily get blocked by
GitHubs rate-limiting.

Instead of all that, let's use `hub` (which implements paging) and the
GitHub's GraphQL endpoint to get a list all repositories for a given
organization AND the latest tag for each of them.

We post-process the output with `jq`.

This also removes the providers.txt and folds the data into the script.
The blacklist was becoming too annoying to handle as a config file.

For the whitelist, we still need to individually fetch tags.

Fixes #83215

Co-Authored-By: zimbatm <zimbatm@zimbatm.com>

pkgs/applications/networking/cluster/terraform-providers/providers.txt

@@ -1,18 +0,0 @@
-# lines starting with a # are comments
-
-# The accepted format (double escape all grep expresssions):
-#  <organisation>         [grep-expression] [grep-v-expression]  - include all repositories in the organisation.
-#                                                                   grep-expression:   filter repo matching the expression
-#                                                                   grep-v-expression: filter repo not matching the expression
-#  <organisation>/<repo>                                         - include only the named repository.
-
-# include all terraform-providers
-terraform-providers terraform-provider- terraform-provider-\\(azure-classic\\|scaffolding\\)
-
-# include providers from individual repos
-ajbosco/terraform-provider-segment
-camptocamp/terraform-provider-pass
-poseidon/terraform-provider-matchbox
-spaceapegames/terraform-provider-wavefront
-tweag/terraform-provider-nixos
-tweag/terraform-provider-secret

pkgs/applications/networking/cluster/terraform-providers/update-all

@@ -1,6 +1,7 @@
 #!/usr/bin/env nix-shell
-#! nix-shell -i bash -p bash coreutils curl jq nix
+#! nix-shell -i bash -p bash coreutils jq nix gitAndTools.hub
 # vim: ft=sh sw=2 et
+# shellcheck shell=bash
 #
 # This scripts scans the github terraform-providers repo for new releases,
 # generates the corresponding nix code and finally generates an index of
@@ -10,37 +11,53 @@ set -euo pipefail
 # the maximum number of attempts before giving up inside of GET and prefetch_github
 readonly maxAttempts=30
 
-GET() {
+get_tf_providers_org() {
-  local url=$1
+  # returns all terraform providers in a given organization, and their the
-  local retry=1
+  # latest tags, in the format
-  echo "fetching $url" >&2
+  # $org/$repo $rev
-  while ! curl -#fL -u "$GITHUB_AUTH" "$url"; do
-    echo "The curl command has failed. Attempt $retry/${maxAttempts}" >&2
-    if [[ "${retry}" -eq "${maxAttempts}" ]]; then
-      exit 1
-    fi
-    retry=$(( retry + 1 ))
-    sleep 5
-  done
-}
-
-get_org_repos() {
   local org=$1
-  local page=1
+  hub api --paginate graphql -f query="
-  GET "https://api.github.com/orgs/$org/repos?per_page=100" | jq -r '.[].name'
+    query(\$endCursor: String) {
+      repositoryOwner(login: \"${org}\") {
+        repositories(first: 100, after: \$endCursor) {
+          nodes {
+            nameWithOwner
+            name
+            refs(first: 1, refPrefix: \"refs/tags/\", orderBy: {field: TAG_COMMIT_DATE, direction: DESC}) {
+              nodes {
+                name
+              }
+            }
+          }
+          pageInfo {
+            hasNextPage
+            endCursor
+          }
+        }
+      }
+  }" | \
+  jq -r '.data.repositoryOwner.repositories.nodes[] | select(.name | startswith("terraform-provider-")) | select((.refs.nodes | length) > 0) | .nameWithOwner + " " + .refs.nodes[0].name'
+  # filter the result with jq:
+  # - repos need to start with `teraform-provider-`
+  # - they need to have at least one tag
+  # for each of the remaining repos, assemble a string $org/$repo $rev
 }
 
-get_repo_tags() {
+get_latest_repo_tag() {
+  # of a given repo and owner, retrieve the latest tag
   local owner=$1
   local repo=$2
-  GET "https://api.github.com/repos/$owner/$repo/git/refs/tags?per_page=100" | \
+  hub api --paginate "https://api.github.com/repos/$owner/$repo/git/refs/tags" | \
     jq -r '.[].ref' | \
     grep -v 'v\.' | \
     cut -d '/' -f 3- | \
-    sort --version-sort
+    sort --version-sort | \
+    tail -1
 }
 
 prefetch_github() {
+  # of a given owner, repo and rev, fetch the tarball and return the output of
+  # `nix-prefetch-url`
   local owner=$1
   local repo=$2
   local rev=$3
@@ -59,7 +76,7 @@ echo_entry() {
   local owner=$1
   local repo=$2
   local rev=$3
-  local version=$(echo $3 | sed 's/^v//')
+  local version=${rev#v}
   local sha256=$4
   cat <<EOF
 {
@@ -74,18 +91,18 @@ EOF
 
 indent() { sed 's/^/    /'; }
 
-add_repo() {
+add_provider() {
   org="${1}"
   repo="${2}"
+  rev="${3}"
 
-  echo "*** $repo ***"
+  echo "*** $org/$repo $rev ***"
   name=$(echo "$repo" | cut -d - -f 3-)
-  last_tag=$(get_repo_tags "$org" "$repo" | tail -1)
+  sha256=$(prefetch_github "$org" "$repo" "$rev")
-  last_tag_sha256=$(prefetch_github "$org" "$repo" "$last_tag")
 
   {
     echo "  $name ="
-    echo_entry "$org" "$repo" "$last_tag" "$last_tag_sha256" | indent
+    echo_entry "$org" "$repo" "$rev" "$sha256" | indent
   } >> data.nix
 }
 
@@ -93,50 +110,61 @@ add_repo() {
 
 cd "$(dirname "$0")"
 
-if [[ -z "${GITHUB_AUTH:-}" ]]; then
+# individual repos to fetch
-  cat <<'HELP'
+slugs=(
-Missing the GITHUB_AUTH env. This is required to work around the 60 request
+  ajbosco/terraform-provider-segment
-per hour rate-limit.
+  camptocamp/terraform-provider-pass
+  poseidon/terraform-provider-matchbox
+  spaceapegames/terraform-provider-wavefront
+  tweag/terraform-provider-nixos
+  tweag/terraform-provider-secret
+)
 
-Go to https://github.com/settings/tokens and create a new token with the
+# a list of providers to ignore
-"public_repo" scope.
+blacklist=(
-
+  terraform-providers/terraform-provider-azure-classic
-Then `export GITHUB_AUTH=<your user>:<your token>` and run this script again.
+  terraform-providers/terraform-provider-cidr
-HELP
+  terraform-providers/terraform-provider-quorum
-  exit 1
+  terraform-providers/terraform-provider-cloudinit
-fi
+  terraform-providers/terraform-provider-time
+)
 
 cat <<HEADER > data.nix
 # Generated with ./update-all
 {
 HEADER
 
-while read line; do
+# assemble list of terraform providers
-  IFS=' ' read -r -a fields <<< "$line"
+providers=$(get_tf_providers_org "terraform-providers")
-  if [[ "${#fields[@]}" -eq 0 ]]; then
+providers=$(echo "$providers";get_tf_providers_org "hashicorp")
-    continue
-  fi
 
-  if [[ "${fields[0]}" = *"/"* ]]; then
+# add terraform-providers from slugs
-    org="$(echo "${fields[0]}" | cut -d/ -f1)"
+for slug in "${slugs[@]}"; do
-    repo="$(echo "${fields[0]}" | cut -d/ -f2)"
+  # retrieve latest tag
-    add_repo "${org}" "${repo}"
+  org=${slug%/*}
-  else
+  repo=${slug#*/}
-    org="${fields[0]}"
+  rev=$(get_latest_repo_tag "$org" "$repo")
-    repos=$(get_org_repos "$org")
-    if [[ "${#fields[@]}" -ge 2 ]] && [[ -n "${fields[1]}" ]]; then
-      repos="$( echo "${repos[@]}" | grep "${fields[1]}" )"
-    fi
-    if [[ "${#fields[@]}" -eq 3 ]] && [[ -n "${fields[2]}" ]]; then
-      repos="$( echo "${repos[@]}" | grep -v "${fields[2]}" )"
-    fi
-    repos="$( echo "${repos[@]}" | sort )"
 
-    for repo in $repos; do
+  # add to list
-      add_repo "$org" "$repo"
+  providers=$(echo "$providers";echo "$org/$repo $rev")
+done
+
+# filter out all providers on the blacklist
+for repo in "${blacklist[@]}"; do
+  providers=$(echo "$providers" | grep -v "^${repo} ")
+done
+
+# sort results alphabetically by repo name
+providers=$(echo "$providers" | sort -t "/" --key=2)
+
+# render list
+IFS=$'\n'
+for provider in $providers; do
+  org=$(echo "$provider" | cut -d " " -f 1 | cut -d "/" -f1)
+  repo=$(echo "$provider" | cut -d " " -f 1 | cut -d "/" -f2)
+  rev=$(echo "$provider" | cut -d " " -f 2)
+  add_provider "${org}" "${repo}" "${rev}"
 done
-  fi
-done < <(grep -v '^#\|^$' providers.txt)
 
 cat <<FOOTER >> data.nix
 }