terraform-providers: refactor

Some terraform providers were moved to the `hashicorp` organization,
some are still in the `terraform-providers` organization.

GitHub doesn't show moved repositories when querying via the API (only
on the web interface). In addition, if there are more than 100 results
(there still are), we'd need to implement paging, which is cubersome in curl.

In addition to that, to look up tags, we used to creating a HTTP request
for each and every provider, which caused us to easily get blocked by
GitHubs rate-limiting.

Instead of all that, let's use `hub` (which implements paging) and the
GitHub's GraphQL endpoint to get a list all repositories for a given
organization AND the latest tag for each of them.

We post-process the output with `jq`.

This also removes the providers.txt and folds the data into the script.
The blacklist was becoming too annoying to handle as a config file.

For the whitelist, we still need to individually fetch tags.

Fixes #83215

Co-Authored-By: zimbatm <zimbatm@zimbatm.com>
This commit is contained in:
Florian Klink 2020-03-24 12:43:04 +01:00
parent 7fedac34b1
commit 2e19bc7074
2 changed files with 90 additions and 80 deletions

View file

@ -1,18 +0,0 @@
# lines starting with a # are comments
# The accepted format (double escape all grep expresssions):
# <organisation> [grep-expression] [grep-v-expression] - include all repositories in the organisation.
# grep-expression: filter repo matching the expression
# grep-v-expression: filter repo not matching the expression
# <organisation>/<repo> - include only the named repository.
# include all terraform-providers
terraform-providers terraform-provider- terraform-provider-\\(azure-classic\\|scaffolding\\)
# include providers from individual repos
ajbosco/terraform-provider-segment
camptocamp/terraform-provider-pass
poseidon/terraform-provider-matchbox
spaceapegames/terraform-provider-wavefront
tweag/terraform-provider-nixos
tweag/terraform-provider-secret

View file

@ -1,6 +1,7 @@
#!/usr/bin/env nix-shell
#! nix-shell -i bash -p bash coreutils curl jq nix
#! nix-shell -i bash -p bash coreutils jq nix gitAndTools.hub
# vim: ft=sh sw=2 et
# shellcheck shell=bash
#
# This scripts scans the github terraform-providers repo for new releases,
# generates the corresponding nix code and finally generates an index of
@ -10,37 +11,53 @@ set -euo pipefail
# the maximum number of attempts before giving up inside of GET and prefetch_github
readonly maxAttempts=30
GET() {
local url=$1
local retry=1
echo "fetching $url" >&2
while ! curl -#fL -u "$GITHUB_AUTH" "$url"; do
echo "The curl command has failed. Attempt $retry/${maxAttempts}" >&2
if [[ "${retry}" -eq "${maxAttempts}" ]]; then
exit 1
fi
retry=$(( retry + 1 ))
sleep 5
done
}
get_org_repos() {
get_tf_providers_org() {
# returns all terraform providers in a given organization, and their the
# latest tags, in the format
# $org/$repo $rev
local org=$1
local page=1
GET "https://api.github.com/orgs/$org/repos?per_page=100" | jq -r '.[].name'
hub api --paginate graphql -f query="
query(\$endCursor: String) {
repositoryOwner(login: \"${org}\") {
repositories(first: 100, after: \$endCursor) {
nodes {
nameWithOwner
name
refs(first: 1, refPrefix: \"refs/tags/\", orderBy: {field: TAG_COMMIT_DATE, direction: DESC}) {
nodes {
name
}
}
}
pageInfo {
hasNextPage
endCursor
}
}
}
}" | \
jq -r '.data.repositoryOwner.repositories.nodes[] | select(.name | startswith("terraform-provider-")) | select((.refs.nodes | length) > 0) | .nameWithOwner + " " + .refs.nodes[0].name'
# filter the result with jq:
# - repos need to start with `teraform-provider-`
# - they need to have at least one tag
# for each of the remaining repos, assemble a string $org/$repo $rev
}
get_repo_tags() {
get_latest_repo_tag() {
# of a given repo and owner, retrieve the latest tag
local owner=$1
local repo=$2
GET "https://api.github.com/repos/$owner/$repo/git/refs/tags?per_page=100" | \
hub api --paginate "https://api.github.com/repos/$owner/$repo/git/refs/tags" | \
jq -r '.[].ref' | \
grep -v 'v\.' | \
cut -d '/' -f 3- | \
sort --version-sort
sort --version-sort | \
tail -1
}
prefetch_github() {
# of a given owner, repo and rev, fetch the tarball and return the output of
# `nix-prefetch-url`
local owner=$1
local repo=$2
local rev=$3
@ -59,7 +76,7 @@ echo_entry() {
local owner=$1
local repo=$2
local rev=$3
local version=$(echo $3 | sed 's/^v//')
local version=${rev#v}
local sha256=$4
cat <<EOF
{
@ -74,18 +91,18 @@ EOF
indent() { sed 's/^/ /'; }
add_repo() {
add_provider() {
org="${1}"
repo="${2}"
rev="${3}"
echo "*** $repo ***"
echo "*** $org/$repo $rev ***"
name=$(echo "$repo" | cut -d - -f 3-)
last_tag=$(get_repo_tags "$org" "$repo" | tail -1)
last_tag_sha256=$(prefetch_github "$org" "$repo" "$last_tag")
sha256=$(prefetch_github "$org" "$repo" "$rev")
{
echo " $name ="
echo_entry "$org" "$repo" "$last_tag" "$last_tag_sha256" | indent
echo_entry "$org" "$repo" "$rev" "$sha256" | indent
} >> data.nix
}
@ -93,50 +110,61 @@ add_repo() {
cd "$(dirname "$0")"
if [[ -z "${GITHUB_AUTH:-}" ]]; then
cat <<'HELP'
Missing the GITHUB_AUTH env. This is required to work around the 60 request
per hour rate-limit.
# individual repos to fetch
slugs=(
ajbosco/terraform-provider-segment
camptocamp/terraform-provider-pass
poseidon/terraform-provider-matchbox
spaceapegames/terraform-provider-wavefront
tweag/terraform-provider-nixos
tweag/terraform-provider-secret
)
Go to https://github.com/settings/tokens and create a new token with the
"public_repo" scope.
Then `export GITHUB_AUTH=<your user>:<your token>` and run this script again.
HELP
exit 1
fi
# a list of providers to ignore
blacklist=(
terraform-providers/terraform-provider-azure-classic
terraform-providers/terraform-provider-cidr
terraform-providers/terraform-provider-quorum
terraform-providers/terraform-provider-cloudinit
terraform-providers/terraform-provider-time
)
cat <<HEADER > data.nix
# Generated with ./update-all
{
HEADER
while read line; do
IFS=' ' read -r -a fields <<< "$line"
if [[ "${#fields[@]}" -eq 0 ]]; then
continue
fi
# assemble list of terraform providers
providers=$(get_tf_providers_org "terraform-providers")
providers=$(echo "$providers";get_tf_providers_org "hashicorp")
if [[ "${fields[0]}" = *"/"* ]]; then
org="$(echo "${fields[0]}" | cut -d/ -f1)"
repo="$(echo "${fields[0]}" | cut -d/ -f2)"
add_repo "${org}" "${repo}"
else
org="${fields[0]}"
repos=$(get_org_repos "$org")
if [[ "${#fields[@]}" -ge 2 ]] && [[ -n "${fields[1]}" ]]; then
repos="$( echo "${repos[@]}" | grep "${fields[1]}" )"
fi
if [[ "${#fields[@]}" -eq 3 ]] && [[ -n "${fields[2]}" ]]; then
repos="$( echo "${repos[@]}" | grep -v "${fields[2]}" )"
fi
repos="$( echo "${repos[@]}" | sort )"
# add terraform-providers from slugs
for slug in "${slugs[@]}"; do
# retrieve latest tag
org=${slug%/*}
repo=${slug#*/}
rev=$(get_latest_repo_tag "$org" "$repo")
for repo in $repos; do
add_repo "$org" "$repo"
done
fi
done < <(grep -v '^#\|^$' providers.txt)
# add to list
providers=$(echo "$providers";echo "$org/$repo $rev")
done
# filter out all providers on the blacklist
for repo in "${blacklist[@]}"; do
providers=$(echo "$providers" | grep -v "^${repo} ")
done
# sort results alphabetically by repo name
providers=$(echo "$providers" | sort -t "/" --key=2)
# render list
IFS=$'\n'
for provider in $providers; do
org=$(echo "$provider" | cut -d " " -f 1 | cut -d "/" -f1)
repo=$(echo "$provider" | cut -d " " -f 1 | cut -d "/" -f2)
rev=$(echo "$provider" | cut -d " " -f 2)
add_provider "${org}" "${repo}" "${rev}"
done
cat <<FOOTER >> data.nix
}