niv-updater

#!/usr/bin/env bash
# vim: set ft=bash

set -euo pipefail

error() {
    echo "::error::$*"
    exit 1
}

warn() {
    echo "::warning::$*"
}

# Check if GITHUB_TOKEN is correct (maybe due to misconfiguration, we might get an expired one)
checkCredentials() {
    if ! curl --fail -s -H "Authorization: token $GITHUB_TOKEN" 'https://api.github.com/'; then
        error 'GITHUB_TOKEN is incorrect, aborting'
    fi
}

# Install all the dependencies we need, using Nix
setupPrerequisites() {
    echo 'Installing Nix'

    # Check if some other step already installed Nix
    if [[ ! -d /nix/store ]] || ! nix --version >/dev/null 2>&1; then
        sudo mkdir -p /etc/nix
        # Workaround segfault: https://github.com/NixOS/nix/issues/2733
        sudo sh -c 'echo "http2 = false" >> /etc/nix/nix.conf'

        sh <(curl -sSL https://nixos.org/nix/install) --no-daemon
    fi

    echo 'Installing Nix - done'
    echo 'Installing dependencies'

    export PATH="${PATH}:/nix/var/nix/profiles/per-user/runner/profile/bin:/nix/var/nix/profiles/default/bin"

    # shellcheck disable=SC1090
    source "${HOME}/.nix-profile/etc/profile.d/nix.sh"

    # we also need hub, and git, but they both come with ubuntu-latest with GitHub Actions
    # https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-README.md
    nix-env -iA niv -f "https://github.com/nmattia/niv/tarball/$INPUT_NIV_VERSION" \
        --substituters https://niv.cachix.org \
        --trusted-public-keys niv.cachix.org-1:X32PCg2e/zAm3/uD1ScqW2z/K0LtDyNV7RdaxIuLgQM=
    # We need --rawfile for jq, available in jq 1.6, which is only available on ubuntu 20.04 and macos
    nix-env -iA nixpkgs.jq
    echo 'Installing dependencies - done'
}

# Setup netrc file to be used by nix-prefetch-url
setupNetrc() {
    if [[ $INPUT_SKIP_SSH_REPOS == 'false' ]]; then
        netrc=$(mktemp)
        sudo chmod 0600 "$netrc"
        sudo sh -c "echo 'netrc-file = $netrc' >> /etc/nix/nix.conf"
        cat <<EOF >>"$netrc"
machine github.com
login api
password $GITHUB_TOKEN
EOF
        # shellcheck disable=SC2064
        trap "rm -f '$netrc'" EXIT
    fi
}

# This function will modify all INPUT_* variables so that they don't contain any garbage
sanitizeInputs() {
    # remove all whitespace
    INPUT_WHITELIST="${INPUT_WHITELIST// /}"
    # remove all redundant commas, as well as those at the beginning and end
    shopt -s extglob
    INPUT_WHITELIST="${INPUT_WHITELIST//+(,)/,}"
    INPUT_WHITELIST="${INPUT_WHITELIST//^+(,)/}"
    INPUT_WHITELIST="${INPUT_WHITELIST//+(,)$/}"
    shopt -u extglob

    # prepare the blacklist
    # only need to remove spaces, commas are irrelevant
    INPUT_BLACKLIST="${INPUT_BLACKLIST// /}"

    # remove all empty lines from the labels
    # do not remove whitespace at the beginning and the end of the line, it might be legal
    shopt -s extglob
    INPUT_LABELS="${INPUT_LABELS##+(*([[:space:]])$'\n')}"
    INPUT_LABELS="${INPUT_LABELS%%+(*([[:space:]])$'\n')}"
    INPUT_LABELS="${INPUT_LABELS//+(*([[:space:]])$'\n')*([[:space:]])$'\n'/$'\n'}"
    shopt -s extglob
}

applyLabels() {
    if [[ -z $INPUT_LABELS ]]; then
        echo 'No labels to add'
        return
    fi

    echo 'Adding labels to the PR'

    local pr_data
    pr_data="$1"

    pr_number="$(jq -jr '.number // empty' <"$pr_data")"
    if [[ -z $pr_number ]]; then
        echo '::warning::could not get the PR number from the json payload, skipping adding labels'
        return
    fi

    if ! jq -n --arg labs "$INPUT_LABELS" '{labels: $labs | split("\n") }' \
        | hub api -XPOST --input - \
            "/repos/$GITHUB_REPOSITORY/issues/$pr_number/labels" >/dev/null; then
        echo "::warning::could not assign labels to the PR $pr_number"
    fi
}

# This function will turn any #123 into owner/repo#123.
# This is needed, since the changelogs from different repositories will all have issue links as #123,
# which, when printed as a PR description will create a link to the _current_ repository (one with
# nix/sources.json), not one where the changelog originated from. The owner/repo#123 format will
# ensure proper linking.
formatIssueLinksPlain() {
    dep_owner="$1"
    dep_repo="$2"
    sed "s~\(#[0-9]\+\)~$dep_owner/$dep_repo\1~g"
}

# Like formatIssuesLinkPlain, this function turns any #123 a direct reference to the original repository.
# Unlike formatIssuesLink, it will use a redirection service (DuckDuckGo), so that referenced PRs/Issues
# do not contain back references.
# For more details, see https://github.com/knl/niv-updater-action/issues/26
# NOTE: It has to be http://r.duckduckgo.com, https variant will not redirect
formatIssueLinksNoBackreferences() {
    dep_owner="$1"
    dep_repo="$2"
    # The sequence e2 81 a0 is \u2060 - UNICODE WORD JOINER
    # We need it in order prevent GitHub from making a backreference due to
    # the commit message text.
    sed "s~#\([0-9]\+\)~[$dep_owner/$dep_repo\xe2\x81\xa0#\1](http://r.duckduckgo.com/l/?uddg=https://github.com/$dep_owner/$dep_repo/issues/\1)~g"
}

# This function formats mentions in the generated changelog in such a way that
# GitHub doesn't ping the mentioned person. This is to remove the noise we might
# throw at our fellow developers.
# Similarly to formatIssueLinksNoBackreferences, we add UNICODE WORD JOINER
# between the '@' and the username
formatMentions() {
    sed 's~\B@\([a-zA-Z0-9]\)~@\xe2\x81\xa0\1~g'
}

# A dispatcher for formatIssueLinksPlain and formatIssueLinksNoBackreferences, based on config.
formatIssueLinks() {
    if [[ $INPUT_GITHUB_CHANGELOG_NO_BACKREFERENCES == 'true' ]]; then
        formatIssueLinksNoBackreferences "$@"
    else
        formatIssueLinksPlain "$@"
    fi
}

createPullRequestsOnUpdate() {
    echo 'Checking for updates'
    if [[ -z $INPUT_PULL_REQUEST_BASE ]]; then
        INPUT_PULL_REQUEST_BASE="$GITHUB_REF"
        base="$GITHUB_SHA"
    else
        # get the SHA of the current base, so that it remains fixed during the run
        # This can fail if the branch doesn't exist. jq would return nothing in that case.
        set +euo pipefail
        base="$(hub api "/repos/$GITHUB_REPOSITORY/branches/$INPUT_PULL_REQUEST_BASE" | jq -jr '.commit.sha // empty')"
        if [[ -z $base ]]; then
            error "Could not get the SHA for branch '$INPUT_PULL_REQUEST_BASE'"
        fi
        set -euo pipefail
    fi

    echo "Will use branch '$INPUT_PULL_REQUEST_BASE' (ref: $base) as the base branch"

    merges_filter=''
    if [[ $INPUT_SHOW_MERGES == 'false' ]]; then
        # a filter for jq, to be used in the query for getting the changelog
        merges_filter='| select((.parents | length) < 2)'
    fi

    SOURCES_JSON='nix/sources.json'

    echo "Getting $INPUT_SOURCES_FILE from $GITHUB_REPOSITORY (ref: $base)"
    # get the content
    sj=$(mktemp)
    if ! hub api -XGET -F ref="$base" "/repos/$GITHUB_REPOSITORY/contents/$INPUT_SOURCES_FILE" >>"$sj"; then
        error 'could not fetch sources.json'
    fi
    echo "Getting $INPUT_SOURCES_FILE from $GITHUB_REPOSITORY (ref: $base) - done"

    file_sha="$(jq -jr '.sha' <"$sj")"
    content="$(jq -r '.content' <"$sj" | base64 -d)"

    if [[ -n $INPUT_WHITELIST ]]; then
        # Can't do <<< as it *appends* a newline :(
        mapfile -td , all_deps < <(printf '%s' "$INPUT_WHITELIST")
    else
        mapfile -t all_deps < <(jq -r 'keys[]' <<<"$content")
    fi

    echo 'Going through all dependencies'
    for dep in "${all_deps[@]}"; do
        echo "Processing dependency '$dep'"

        if [[ ",$INPUT_BLACKLIST," == *",$dep,"* ]]; then
            echo "Dependency '$dep' is blacklisted, skipping."
            continue
        fi

        revision="$(jq -jr ".\"$dep\".rev" <<<"$content")"

        # check if revision doesn't look like sha, and if skip_versioned_revisions is set, skip
        if [[ $INPUT_SKIP_VERSIONED_REVISIONS == 'true' && ! ($revision =~ ^[a-f0-9A-F]{40}$ || $revision =~ ^[a-f0-9A-F]{7}) ]]; then
            echo "Revision '$revision' looks like a regular version string, and skip_versioned_revisions is set, skipping."
            continue
        fi

        # check if the branch exists first.
        # If it exists and we're not updating the PR, then skip.
        branch_name="$INPUT_BRANCH_PREFIX$dep-$revision"

        branch_exists='no'
        if hub api "/repos/$GITHUB_REPOSITORY/branches/$branch_name" >/dev/null; then
            if [[ $INPUT_KEEP_UPDATING == 'true' ]]; then
                branch_exists='yes'
            else
                echo "branch '$branch_name' already exists, skipping."
                continue
            fi
        fi

        echo "Will use branch '$branch_name' for the possible update"

        # since this is an action, we don't have SSH keys available
        # but we do have tokens that are useful
        # check if the dependency is to github
        dep_owner="$(jq -jr ".\"$dep\".owner // empty" <<<"$content")"
        dep_repo="$(jq -jr ".\"$dep\".repo // empty" <<<"$content")"
        dep_url="$(jq -jr ".\"$dep\".url // empty" <<<"$content" | { grep github.com || true; })"
        # Here, we want to recognize the following URLs:
        # ((git+)?ssh://)?git@github.com(:<port>)?[:/]owner/repo[.git]?
        # That is, optional scheme (ssh), followed by git@github.com, followed with optional port and either : or /
        github_ssh="$(jq -jr ".\"$dep\".repo // empty" <<<"$content" | { grep -E '^((git\+)?ssh://)?git@github.com(:[[:digit:]]+)?[:/]?' || true; })"
        [[ -n $dep_url || -n $github_ssh ]] && is_github='yes' || is_github=''

        # skip if github_ssh and skip_ssh_repos is in effect
        if [[ -n $github_ssh && $INPUT_SKIP_SSH_REPOS == 'true' ]]; then
            echo 'Hosted by a repository accessible over SSH, and skip_ssh_repos is set, skipping.'
            continue
        fi

        # try extracting the owner and the repo
        if [[ -n $github_ssh ]]; then
            # Here, we can be lenient. If niv already added some entries, we know they are correct.
            # Thus, we extract something that looks like 'owner/repo.git' from the end of the string.
            dep_owner="$(echo "$github_ssh" | perl -nle 'print $1 if m/[:\/]([\w-]{1,39})\/([\w_.-]+?)(?:\.git)?$/;')"
            dep_repo="$(echo "$github_ssh" | perl -nle 'print $2 if m/[:\/]([\w-]{1,39})\/([\w_.-]+?)(?:\.git)?$/;')"
        fi

        # check if there is an update by running niv
        wdir=$(mktemp -d)
        mkdir -p "$wdir/nix"
        echo "$content" >"$wdir/$SOURCES_JSON.orig"
        echo "$content" >"$wdir/$SOURCES_JSON"
        (
            cd "$wdir"

            # rewrite the entry so that we can use token instead of SSH
            if [[ -n $github_ssh ]]; then
                echo 'As this is a dependency fetched with SSH, trying to switch to https type'
                niv_branch="$(jq -jr ".\"$dep\".branch // empty" <"$wdir/$SOURCES_JSON")"
                niv drop "$dep"
                niv add "$dep_owner/$dep_repo" -a rev="$revision" -a branch="$niv_branch"
                rm "$SOURCES_JSON.orig"
                cp "$SOURCES_JSON" "$SOURCES_JSON.orig"
            fi

            # TODO: make sure niv can update from tag to tag, instead of always using shas
            if ! niv update "$dep"; then
                echo "::warning:: Cannot update '$dep', skipping it"
                touch .skip
            fi
        )
        # the only way to continue when using subshells
        if [[ -e "$wdir/.skip" ]]; then
            rm -rf "$wdir"
            continue
        fi

        if diff -q "$wdir/$SOURCES_JSON.orig" "$wdir/$SOURCES_JSON" &>/dev/null; then
            echo "There is no update for '$dep', skipping."
            continue
        fi

        new_revision="$(jq -jr ".\"$dep\".rev" <"$wdir/$SOURCES_JSON")"

        if [[ $revision == "$new_revision" ]]; then
            echo "The new version ($new_revision) is the same as old ($revision) for $dep, skipping"
            continue
        fi

        # since in the previous step we possibly changed from ssh to https, revert back
        (
            cd "$wdir"

            # rewrite the entry so that we can use token instead of SSH
            if [[ -n $github_ssh ]]; then
                echo 'Reverting the dependency back to SSH'
                echo "$content" >"$wdir/$SOURCES_JSON"
                niv modify "$dep" -a rev="$new_revision"
            fi
        )

        # generate the message
        title=$(mktemp)
        message=$(mktemp)

        printf 'Will generate the Pull Request message for '%s', update from %.8s to %.8s\n' "$dep" "$revision" "$new_revision"

        printf '%sniv %s: update %.8s -> %.8s' "$INPUT_TITLE_PREFIX${INPUT_TITLE_PREFIX:+ }" "$dep" "$revision" "$new_revision" >>"$title"

        # print with a new line appended, as yaml swallows those
        printf '%s%s' "$INPUT_MESSAGE_PREFIX" "${INPUT_MESSAGE_PREFIX:+$'\n'}" >>"$message"

        # get a short changelog if we're on github
        if [[ -z $is_github ]]; then
            # pretty sure this is not github
            echo "Dependency '$dep' isn't hosted on github.com, cannot fetch the changelog" >>"$message"
        else
            echo "Dependency '$dep' is hosted on github.com"

            {
                niv_branch="$(jq -jr ".\"$dep\".branch // empty" <"$wdir/$SOURCES_JSON")"
                printf '## Changelog for %s:\n' "$dep"
                printf 'Branch: %s\n' "$niv_branch"
                printf 'Commits: [%s/%s@%.8s...%.8s](https://github.com/%s/%s/compare/%s...%s)\n\n' "$dep_owner" "$dep_repo" "$revision" "$new_revision" "$dep_owner" "$dep_repo" "$revision" "$new_revision"
                {
                    hub api "/repos/$dep_owner/$dep_repo/compare/${revision}...${new_revision}" || true
                } | jq -r '.commits[] '"$merges_filter"' | "* [`\(.sha[0:8])`](\(.html_url)) \(.commit.message | split("\n") | first)"' \
                    | formatIssueLinks "$dep_owner" "$dep_repo" \
                    | formatMentions
            } >>"$message"
        fi

        # print with a new line appended, as yaml swallows those
        printf '%s%s' "$INPUT_MESSAGE_SUFFIX" "${INPUT_MESSAGE_SUFFIX:+$'\n'}" >>"$message"

        commit_message=$(mktemp)
        {
            cat "$title"
            printf '\n\n'
            cat "$message"
        } >"$commit_message"

        new_content=$(mktemp)
        base64 "$wdir/$SOURCES_JSON" >>"$new_content"

        # This is the behaviour when the branch doesn't exist
        if [[ $branch_exists == 'no' ]]; then
            # create the branch
            echo "Creating branch '$branch_name' for the update of '$dep'"
            if ! hub api \
                -F ref="refs/heads/$branch_name" \
                -F sha="$base" \
                "/repos/$GITHUB_REPOSITORY/git/refs" >/dev/null; then
                error "could not create branch $branch_name"
            fi

            # upload the content
            echo "Uploading the new $INPUT_SOURCES_FILE"
            # https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#create-or-update-file-contents
            if ! hub api -XPUT \
                -F message=@"$commit_message" \
                -F content=@"$new_content" \
                -F sha="$file_sha" \
                -F branch="$branch_name" \
                "/repos/$GITHUB_REPOSITORY/contents/$INPUT_SOURCES_FILE" >/dev/null; then
                # try to delete the branch if adding a new content fails
                hub api -XDELETE "/repos/$GITHUB_REPOSITORY/git/refs/heads/$branch_name" >/dev/null || true
                error "could not upload content to $branch_name"
            fi

            # create a PR, use API to avoid the need for a local checkout
            echo "Creating a PR for updating '$dep', branch name is '$branch_name'"
            pr_data=$(mktemp)
            if ! hub api -XPOST \
                -F head="$branch_name" \
                -F base="$INPUT_PULL_REQUEST_BASE" \
                -F title=@"$title" \
                -F body=@"$message" \
                "/repos/$GITHUB_REPOSITORY/pulls" >>"$pr_data"; then
                # try to delete the branch
                hub api -XDELETE "/repos/$GITHUB_REPOSITORY/git/refs/heads/$branch_name" >/dev/null || true
                error 'could not create a PR'
            fi

            applyLabels "$pr_data"

            # cleanup
            rm -f "$pr_data"
        else
            # The branch already exists and we need to update it.
            # The update will actually be the force update, where we create a new tree and commit objects,
            # and then update the reference and the PR description.

            # The steps are:

            # 0. check if the file is the same and skip in that case
            current_content=$(mktemp)
            if ! hub api -XGET -F ref="$branch_name" "/repos/$GITHUB_REPOSITORY/contents/$INPUT_SOURCES_FILE" \
                | jq -jr '.content' | base64 -d >"$current_content"; then
                # We can't just warn and proceed here, as subsequent calls to create a tree might fail
                # if the tree already exists
                error 'could not get the old branch content, aborting'
            fi

            # This is a trick to exit early if the content is the same
            # shellcheck disable=SC2034,SC2043
            for irrelevant in "run_once"; do
                if diff -q "$current_content" "$wdir/$SOURCES_JSON"; then
                    echo 'files are the same, no need to update the PR'
                    break
                fi
                # 1. get the base commit sha - done before the if statement

                # 2. get the base tree from the base commit sha
                base_tree="$(hub api "/repos/$GITHUB_REPOSITORY/git/commits/$base" | jq -jr '.tree.sha')"

                # 3a. create a new blob with the content
                blob_data=$(mktemp)
                if ! hub api -XPOST -f encoding=base64 \
                    -F content=@"$new_content" \
                    "/repos/$GITHUB_REPOSITORY/git/blobs" >"$blob_data"; then
                    # There is nothing better to do
                    error 'could not create a blob for the new content'
                fi

                # 3b. create a new tree with the file update
                tree_data=$(mktemp)
                # Per: https://docs.github.com/en/free-pro-team@latest/rest/reference/git#create-a-tree
                # one must not provide both sha and content
                if ! jq --arg path "$INPUT_SOURCES_FILE" --arg base_tree "$base_tree" \
                    '{base_tree: $base_tree, tree: [{path: $path, mode: "100644", type: "blob", content: null, sha}]}' \
                    <"$blob_data" \
                    | hub api -XPOST --input - \
                        "/repos/$GITHUB_REPOSITORY/git/trees" >>"$tree_data"; then
                    # There is nothing better to do
                    error 'could not create a new tree with updated content'
                fi

                # 4. create a new commit
                commit_data=$(mktemp)
                if ! jq -ncj \
                    --arg parent "$base" \
                    --arg tree "$(jq -jr '.sha' <"$tree_data")" \
                    --rawfile message "$commit_message" \
                    '{message: $message, tree: $tree, parents: [$parent]}' \
                    | hub api -XPOST \
                        --input - \
                        "/repos/$GITHUB_REPOSITORY/git/commits" \
                        >"$commit_data"; then
                    # There is nothing better to do
                    error 'could not create a new commit with PR updates'
                fi

                # 5. update reference to point to the new commit
                old_branch_sha="$(hub api "/repos/$GITHUB_REPOSITORY/git/ref/heads/$branch_name" | jq -jr '.object.sha')"
                if ! hub api -XPATCH \
                    -F force=true \
                    -F sha="$(jq -jr '.sha' <"$commit_data")" \
                    "/repos/$GITHUB_REPOSITORY/git/refs/heads/$branch_name" >/dev/null; then
                    # There is nothing better to do
                    error 'could not update the branch to point to the new commit'
                fi

                # 6. find the PR for the branch
                # The best way to do it is via GraphQL
                pr_number_data=$(mktemp)
                # shellcheck disable=SC2016
                if ! hub api graphql -f owner="${GITHUB_REPOSITORY%/*}" -f name="${GITHUB_REPOSITORY#*/}" -f branch="$branch_name" -f query='
                    query PrNumber($owner: String!, $name: String!, $branch: String!) {
                      repository(owner: $owner, name: $name) {
                        pullRequests(last: 1, states: OPEN, headRefName: $branch) {
                          nodes {
                            number
                          }
                        }
                      }
                    }' >"$pr_number_data"; then
                    # There is nothing better to do
                    error 'could not get the PR number for this branch'
                fi

                pr_number="$(jq '.data.repository.pullRequests.nodes[].number' <"$pr_number_data")"

                # 7. update PR description and title
                if ! hub api -XPATCH \
                    -F body=@"$message" \
                    -F title=@"$title" \
                    "/repos/$GITHUB_REPOSITORY/pulls/$pr_number" >/dev/null; then
                    # It would be good to roll back here to the previous commit
                    warn 'could not update the PR description, reverting to the previous commit'
                    if ! hub api -XPATCH \
                        -F force=true \
                        -F sha="$old_branch_sha" \
                        "/repos/$GITHUB_REPOSITORY/git/refs/heads/$branch_name" >/dev/null; then
                        # There is nothing better to do
                        error 'could not revert the branch to point to the old commit'
                    fi
                fi

                # cleanup
                rm -f "$tree_data"
                rm -f "$commit_data"
                rm -f "$pr_number_data"
            done

            # cleanup
            rm -f "$current_content"
        fi

        # cleanup
        rm -rf "$wdir"
        rm -f "$new_content"
        rm -f "$commit_message"
        rm -f "$message"
        rm -f "$title"

        echo "Processing dependency '$dep' - done"
    done
    rm -f "$sj"
    echo 'Checking for updates - done'
}

if [[ $INPUT_DEBUG_OUTPUT == 'true' ]]; then
    set -x
fi

checkCredentials
setupPrerequisites
setupNetrc
sanitizeInputs
createPullRequestsOnUpdate