diff --git a/pkgs/applications/networking/cluster/terraform-providers/providers.txt b/pkgs/applications/networking/cluster/terraform-providers/providers.txt deleted file mode 100644 index bdde6600678..00000000000 --- a/pkgs/applications/networking/cluster/terraform-providers/providers.txt +++ /dev/null @@ -1,18 +0,0 @@ -# lines starting with a # are comments - -# The accepted format (double escape all grep expresssions): -# [grep-expression] [grep-v-expression] - include all repositories in the organisation. -# grep-expression: filter repo matching the expression -# grep-v-expression: filter repo not matching the expression -# / - include only the named repository. - -# include all terraform-providers -terraform-providers terraform-provider- terraform-provider-\\(azure-classic\\|scaffolding\\) - -# include providers from individual repos -ajbosco/terraform-provider-segment -camptocamp/terraform-provider-pass -poseidon/terraform-provider-matchbox -spaceapegames/terraform-provider-wavefront -tweag/terraform-provider-nixos -tweag/terraform-provider-secret diff --git a/pkgs/applications/networking/cluster/terraform-providers/update-all b/pkgs/applications/networking/cluster/terraform-providers/update-all index 893a6b1c7d7..e1c2e94b3ff 100755 --- a/pkgs/applications/networking/cluster/terraform-providers/update-all +++ b/pkgs/applications/networking/cluster/terraform-providers/update-all @@ -1,6 +1,7 @@ #!/usr/bin/env nix-shell -#! nix-shell -i bash -p bash coreutils curl jq nix +#! nix-shell -i bash -p bash coreutils jq nix gitAndTools.hub # vim: ft=sh sw=2 et +# shellcheck shell=bash # # This scripts scans the github terraform-providers repo for new releases, # generates the corresponding nix code and finally generates an index of @@ -10,37 +11,53 @@ set -euo pipefail # the maximum number of attempts before giving up inside of GET and prefetch_github readonly maxAttempts=30 -GET() { - local url=$1 - local retry=1 - echo "fetching $url" >&2 - while ! curl -#fL -u "$GITHUB_AUTH" "$url"; do - echo "The curl command has failed. Attempt $retry/${maxAttempts}" >&2 - if [[ "${retry}" -eq "${maxAttempts}" ]]; then - exit 1 - fi - retry=$(( retry + 1 )) - sleep 5 - done -} - -get_org_repos() { +get_tf_providers_org() { + # returns all terraform providers in a given organization, and their the + # latest tags, in the format + # $org/$repo $rev local org=$1 - local page=1 - GET "https://api.github.com/orgs/$org/repos?per_page=100" | jq -r '.[].name' + hub api --paginate graphql -f query=" + query(\$endCursor: String) { + repositoryOwner(login: \"${org}\") { + repositories(first: 100, after: \$endCursor) { + nodes { + nameWithOwner + name + refs(first: 1, refPrefix: \"refs/tags/\", orderBy: {field: TAG_COMMIT_DATE, direction: DESC}) { + nodes { + name + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + }" | \ + jq -r '.data.repositoryOwner.repositories.nodes[] | select(.name | startswith("terraform-provider-")) | select((.refs.nodes | length) > 0) | .nameWithOwner + " " + .refs.nodes[0].name' + # filter the result with jq: + # - repos need to start with `teraform-provider-` + # - they need to have at least one tag + # for each of the remaining repos, assemble a string $org/$repo $rev } -get_repo_tags() { +get_latest_repo_tag() { + # of a given repo and owner, retrieve the latest tag local owner=$1 local repo=$2 - GET "https://api.github.com/repos/$owner/$repo/git/refs/tags?per_page=100" | \ + hub api --paginate "https://api.github.com/repos/$owner/$repo/git/refs/tags" | \ jq -r '.[].ref' | \ grep -v 'v\.' | \ cut -d '/' -f 3- | \ - sort --version-sort + sort --version-sort | \ + tail -1 } prefetch_github() { + # of a given owner, repo and rev, fetch the tarball and return the output of + # `nix-prefetch-url` local owner=$1 local repo=$2 local rev=$3 @@ -59,7 +76,7 @@ echo_entry() { local owner=$1 local repo=$2 local rev=$3 - local version=$(echo $3 | sed 's/^v//') + local version=${rev#v} local sha256=$4 cat <> data.nix } @@ -93,50 +110,61 @@ add_repo() { cd "$(dirname "$0")" -if [[ -z "${GITHUB_AUTH:-}" ]]; then - cat <<'HELP' -Missing the GITHUB_AUTH env. This is required to work around the 60 request -per hour rate-limit. +# individual repos to fetch +slugs=( + ajbosco/terraform-provider-segment + camptocamp/terraform-provider-pass + poseidon/terraform-provider-matchbox + spaceapegames/terraform-provider-wavefront + tweag/terraform-provider-nixos + tweag/terraform-provider-secret +) -Go to https://github.com/settings/tokens and create a new token with the -"public_repo" scope. - -Then `export GITHUB_AUTH=:` and run this script again. -HELP - exit 1 -fi +# a list of providers to ignore +blacklist=( + terraform-providers/terraform-provider-azure-classic + terraform-providers/terraform-provider-cidr + terraform-providers/terraform-provider-quorum + terraform-providers/terraform-provider-cloudinit + terraform-providers/terraform-provider-time +) cat <
data.nix # Generated with ./update-all { HEADER -while read line; do - IFS=' ' read -r -a fields <<< "$line" - if [[ "${#fields[@]}" -eq 0 ]]; then - continue - fi +# assemble list of terraform providers +providers=$(get_tf_providers_org "terraform-providers") +providers=$(echo "$providers";get_tf_providers_org "hashicorp") - if [[ "${fields[0]}" = *"/"* ]]; then - org="$(echo "${fields[0]}" | cut -d/ -f1)" - repo="$(echo "${fields[0]}" | cut -d/ -f2)" - add_repo "${org}" "${repo}" - else - org="${fields[0]}" - repos=$(get_org_repos "$org") - if [[ "${#fields[@]}" -ge 2 ]] && [[ -n "${fields[1]}" ]]; then - repos="$( echo "${repos[@]}" | grep "${fields[1]}" )" - fi - if [[ "${#fields[@]}" -eq 3 ]] && [[ -n "${fields[2]}" ]]; then - repos="$( echo "${repos[@]}" | grep -v "${fields[2]}" )" - fi - repos="$( echo "${repos[@]}" | sort )" +# add terraform-providers from slugs +for slug in "${slugs[@]}"; do + # retrieve latest tag + org=${slug%/*} + repo=${slug#*/} + rev=$(get_latest_repo_tag "$org" "$repo") - for repo in $repos; do - add_repo "$org" "$repo" - done - fi -done < <(grep -v '^#\|^$' providers.txt) + # add to list + providers=$(echo "$providers";echo "$org/$repo $rev") +done + +# filter out all providers on the blacklist +for repo in "${blacklist[@]}"; do + providers=$(echo "$providers" | grep -v "^${repo} ") +done + +# sort results alphabetically by repo name +providers=$(echo "$providers" | sort -t "/" --key=2) + +# render list +IFS=$'\n' +for provider in $providers; do + org=$(echo "$provider" | cut -d " " -f 1 | cut -d "/" -f1) + repo=$(echo "$provider" | cut -d " " -f 1 | cut -d "/" -f2) + rev=$(echo "$provider" | cut -d " " -f 2) + add_provider "${org}" "${repo}" "${rev}" +done cat <