From 2e19bc707476670f9beb7fa90b3a06f20dcf8762 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Tue, 24 Mar 2020 12:43:04 +0100 Subject: [PATCH] terraform-providers: refactor Some terraform providers were moved to the `hashicorp` organization, some are still in the `terraform-providers` organization. GitHub doesn't show moved repositories when querying via the API (only on the web interface). In addition, if there are more than 100 results (there still are), we'd need to implement paging, which is cubersome in curl. In addition to that, to look up tags, we used to creating a HTTP request for each and every provider, which caused us to easily get blocked by GitHubs rate-limiting. Instead of all that, let's use `hub` (which implements paging) and the GitHub's GraphQL endpoint to get a list all repositories for a given organization AND the latest tag for each of them. We post-process the output with `jq`. This also removes the providers.txt and folds the data into the script. The blacklist was becoming too annoying to handle as a config file. For the whitelist, we still need to individually fetch tags. Fixes #83215 Co-Authored-By: zimbatm --- .../cluster/terraform-providers/providers.txt | 18 --- .../cluster/terraform-providers/update-all | 152 +++++++++++------- 2 files changed, 90 insertions(+), 80 deletions(-) delete mode 100644 pkgs/applications/networking/cluster/terraform-providers/providers.txt diff --git a/pkgs/applications/networking/cluster/terraform-providers/providers.txt b/pkgs/applications/networking/cluster/terraform-providers/providers.txt deleted file mode 100644 index bdde6600678..00000000000 --- a/pkgs/applications/networking/cluster/terraform-providers/providers.txt +++ /dev/null @@ -1,18 +0,0 @@ -# lines starting with a # are comments - -# The accepted format (double escape all grep expresssions): -# [grep-expression] [grep-v-expression] - include all repositories in the organisation. -# grep-expression: filter repo matching the expression -# grep-v-expression: filter repo not matching the expression -# / - include only the named repository. - -# include all terraform-providers -terraform-providers terraform-provider- terraform-provider-\\(azure-classic\\|scaffolding\\) - -# include providers from individual repos -ajbosco/terraform-provider-segment -camptocamp/terraform-provider-pass -poseidon/terraform-provider-matchbox -spaceapegames/terraform-provider-wavefront -tweag/terraform-provider-nixos -tweag/terraform-provider-secret diff --git a/pkgs/applications/networking/cluster/terraform-providers/update-all b/pkgs/applications/networking/cluster/terraform-providers/update-all index 893a6b1c7d7..e1c2e94b3ff 100755 --- a/pkgs/applications/networking/cluster/terraform-providers/update-all +++ b/pkgs/applications/networking/cluster/terraform-providers/update-all @@ -1,6 +1,7 @@ #!/usr/bin/env nix-shell -#! nix-shell -i bash -p bash coreutils curl jq nix +#! nix-shell -i bash -p bash coreutils jq nix gitAndTools.hub # vim: ft=sh sw=2 et +# shellcheck shell=bash # # This scripts scans the github terraform-providers repo for new releases, # generates the corresponding nix code and finally generates an index of @@ -10,37 +11,53 @@ set -euo pipefail # the maximum number of attempts before giving up inside of GET and prefetch_github readonly maxAttempts=30 -GET() { - local url=$1 - local retry=1 - echo "fetching $url" >&2 - while ! curl -#fL -u "$GITHUB_AUTH" "$url"; do - echo "The curl command has failed. Attempt $retry/${maxAttempts}" >&2 - if [[ "${retry}" -eq "${maxAttempts}" ]]; then - exit 1 - fi - retry=$(( retry + 1 )) - sleep 5 - done -} - -get_org_repos() { +get_tf_providers_org() { + # returns all terraform providers in a given organization, and their the + # latest tags, in the format + # $org/$repo $rev local org=$1 - local page=1 - GET "https://api.github.com/orgs/$org/repos?per_page=100" | jq -r '.[].name' + hub api --paginate graphql -f query=" + query(\$endCursor: String) { + repositoryOwner(login: \"${org}\") { + repositories(first: 100, after: \$endCursor) { + nodes { + nameWithOwner + name + refs(first: 1, refPrefix: \"refs/tags/\", orderBy: {field: TAG_COMMIT_DATE, direction: DESC}) { + nodes { + name + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + }" | \ + jq -r '.data.repositoryOwner.repositories.nodes[] | select(.name | startswith("terraform-provider-")) | select((.refs.nodes | length) > 0) | .nameWithOwner + " " + .refs.nodes[0].name' + # filter the result with jq: + # - repos need to start with `teraform-provider-` + # - they need to have at least one tag + # for each of the remaining repos, assemble a string $org/$repo $rev } -get_repo_tags() { +get_latest_repo_tag() { + # of a given repo and owner, retrieve the latest tag local owner=$1 local repo=$2 - GET "https://api.github.com/repos/$owner/$repo/git/refs/tags?per_page=100" | \ + hub api --paginate "https://api.github.com/repos/$owner/$repo/git/refs/tags" | \ jq -r '.[].ref' | \ grep -v 'v\.' | \ cut -d '/' -f 3- | \ - sort --version-sort + sort --version-sort | \ + tail -1 } prefetch_github() { + # of a given owner, repo and rev, fetch the tarball and return the output of + # `nix-prefetch-url` local owner=$1 local repo=$2 local rev=$3 @@ -59,7 +76,7 @@ echo_entry() { local owner=$1 local repo=$2 local rev=$3 - local version=$(echo $3 | sed 's/^v//') + local version=${rev#v} local sha256=$4 cat <> data.nix } @@ -93,50 +110,61 @@ add_repo() { cd "$(dirname "$0")" -if [[ -z "${GITHUB_AUTH:-}" ]]; then - cat <<'HELP' -Missing the GITHUB_AUTH env. This is required to work around the 60 request -per hour rate-limit. +# individual repos to fetch +slugs=( + ajbosco/terraform-provider-segment + camptocamp/terraform-provider-pass + poseidon/terraform-provider-matchbox + spaceapegames/terraform-provider-wavefront + tweag/terraform-provider-nixos + tweag/terraform-provider-secret +) -Go to https://github.com/settings/tokens and create a new token with the -"public_repo" scope. - -Then `export GITHUB_AUTH=:` and run this script again. -HELP - exit 1 -fi +# a list of providers to ignore +blacklist=( + terraform-providers/terraform-provider-azure-classic + terraform-providers/terraform-provider-cidr + terraform-providers/terraform-provider-quorum + terraform-providers/terraform-provider-cloudinit + terraform-providers/terraform-provider-time +) cat <
data.nix # Generated with ./update-all { HEADER -while read line; do - IFS=' ' read -r -a fields <<< "$line" - if [[ "${#fields[@]}" -eq 0 ]]; then - continue - fi +# assemble list of terraform providers +providers=$(get_tf_providers_org "terraform-providers") +providers=$(echo "$providers";get_tf_providers_org "hashicorp") - if [[ "${fields[0]}" = *"/"* ]]; then - org="$(echo "${fields[0]}" | cut -d/ -f1)" - repo="$(echo "${fields[0]}" | cut -d/ -f2)" - add_repo "${org}" "${repo}" - else - org="${fields[0]}" - repos=$(get_org_repos "$org") - if [[ "${#fields[@]}" -ge 2 ]] && [[ -n "${fields[1]}" ]]; then - repos="$( echo "${repos[@]}" | grep "${fields[1]}" )" - fi - if [[ "${#fields[@]}" -eq 3 ]] && [[ -n "${fields[2]}" ]]; then - repos="$( echo "${repos[@]}" | grep -v "${fields[2]}" )" - fi - repos="$( echo "${repos[@]}" | sort )" +# add terraform-providers from slugs +for slug in "${slugs[@]}"; do + # retrieve latest tag + org=${slug%/*} + repo=${slug#*/} + rev=$(get_latest_repo_tag "$org" "$repo") - for repo in $repos; do - add_repo "$org" "$repo" - done - fi -done < <(grep -v '^#\|^$' providers.txt) + # add to list + providers=$(echo "$providers";echo "$org/$repo $rev") +done + +# filter out all providers on the blacklist +for repo in "${blacklist[@]}"; do + providers=$(echo "$providers" | grep -v "^${repo} ") +done + +# sort results alphabetically by repo name +providers=$(echo "$providers" | sort -t "/" --key=2) + +# render list +IFS=$'\n' +for provider in $providers; do + org=$(echo "$provider" | cut -d " " -f 1 | cut -d "/" -f1) + repo=$(echo "$provider" | cut -d " " -f 1 | cut -d "/" -f2) + rev=$(echo "$provider" | cut -d " " -f 2) + add_provider "${org}" "${repo}" "${rev}" +done cat <