feat: improve building from alternate sources

.gitignore

@@ -26,3 +26,6 @@ venv.bak/
 
 .DS_Store
 .vscode
+
+*.private.yml
+*.private.yaml

jobs/ci-run/ci-run-master.yml

@@ -1,30 +1,9 @@
 - job:
-    name: "ci-build-juju"
+    name: "ci-trigger-github-com-juju-juju"
     disabled: false
     concurrent: false
-    description: |
-      Build Juju for subsequent CI run jobs to use.
-      <br/>
-      The rough outline of a build stage is:
-        <ul>
-            <li>Checkout the code that triggered the change (a build run uses the commit hash as the ID.)</li>
-            <li>Create a tarball of the source that also contains the dependencies</li>
-            <li>Build the code and upload the resulting binaries and buildvars to s3</li>
-            <li>Package the resulting jujud as an agent and upload that too for later processing</li>
-            <li>Generate streams for this builds version (and only this version, no historic stream data) and upload results to testing stream servers</li>
-            <li>
-                for phase in [unit-tests, functional-tests]:
-                <ul>
-                    <li>Pull down the binaries (or source) for this build and run the test using it.</li>
-                </ul>
-            </li>
-        </ul>
     project-type: "multijob"
     node: noop-parent-jobs
-    # Any changes to the refspec for branches watched must be updated in the
-    # 'generate-testing-streams' job otherwise they will never persist in
-    # streams.
-    #
     properties:
       - github:
           url: https://github.com/juju/juju/
@@ -62,7 +41,39 @@
               </events>
               <preStatus>false</preStatus>
             </com.github.kostyasha.github.integration.branch.GitHubBranchTrigger>
+    publishers:
+      - trigger-parameterized-builds:
+          - project: "ci-build-juju"
+            condition: SUCCESS
+            current-parameters: false
+            predefined-parameters: |-
+              GITHUB_REPO=juju/juju
+              GITHUB_BRANCH_HEAD_SHA=${{GITHUB_BRANCH_HEAD_SHA}}
+
 
+- job:
+    name: "ci-build-juju"
+    disabled: false
+    concurrent: false
+    description: |
+      Build Juju for subsequent CI run jobs to use.
+      <br/>
+      The rough outline of a build stage is:
+        <ul>
+            <li>Checkout the code that triggered the change (a build run uses the commit hash as the ID.)</li>
+            <li>Create a tarball of the source that also contains the dependencies</li>
+            <li>Build the code and upload the resulting binaries and buildvars to s3</li>
+            <li>Package the resulting jujud as an agent and upload that too for later processing</li>
+            <li>Generate streams for this builds version (and only this version, no historic stream data) and upload results to testing stream servers</li>
+            <li>
+                for phase in [unit-tests, functional-tests]:
+                <ul>
+                    <li>Pull down the binaries (or source) for this build and run the test using it.</li>
+                </ul>
+            </li>
+        </ul>
+    project-type: "multijob"
+    node: noop-parent-jobs
     wrappers:
       - ansicolor
       - workspace-cleanup
@@ -71,36 +82,44 @@
           timeout: 120
           fail: true
           type: absolute
-
+      - credentials-binding:
+          - text:
+              credential-id: github-token-private
+              variable: GITHUB_TOKEN
     parameters:
       - string:
+          name: GITHUB_REPO
+          default: juju/juju
+          description: "Specifc github repo to pull from"
+      - string:
+          name: GITHUB_BRANCH_HEAD_SHA
           default: ""
           description: "Specific git SHA to build (used to overwrite triggered runs)."
-          name: GITHUB_BRANCH_HEAD_SHA
-
     builders:
       - resolve-git-sha
       - detect-commit-go-version
+      - inject:
+          properties-content: |-
+            GITHUB_TOKEN=
       - multijob:
           name: "Packaging"
           condition: SUCCESSFUL
           projects:
             - name: "package-juju-source"
-              current-parameters: true
+              current-parameters: false
               predefined-parameters: |-
+                GITHUB_REPO=${{GITHUB_REPO}}
                 GITHUB_BRANCH_HEAD_SHA=${{GITHUB_BRANCH_HEAD_SHA}}
-                SHORT_GIT_COMMIT=${{SHORT_GIT_COMMIT}}
                 JUJU_BUILD_NUMBER=${{BUILD_NUMBER}}
                 GOVERSION=${{GOVERSION}}
-
       - get-s3-source-payload
       - inject:
           properties-content: |-
             PATH=/snap/bin:$PATH
             series=focal
       - get-build-details
       - description-setter:
-          description: "${{JUJU_VERSION}}:${{SHORT_GIT_COMMIT}} (go ${{GOVERSION}})"
+          description: "${{GITHUB_REPO}} ${{JUJU_VERSION}}:${{SHORT_GIT_COMMIT}} (go ${{GOVERSION}})"
       - multijob:
           name: "Building Juju Binaries"
           condition: SUCCESSFUL
@@ -172,7 +191,6 @@
                 BUILD_LABEL=k8s-linux-amd64
                 SHORT_GIT_COMMIT=${{SHORT_GIT_COMMIT}}
                 GOVERSION=${{GOVERSION}}
-
       - multijob:
           name: "Building Juju OCI Images"
           condition: SUCCESSFUL
@@ -182,7 +200,6 @@
               predefined-parameters: |-
                 SHORT_GIT_COMMIT=${{SHORT_GIT_COMMIT}}
                 GOVERSION=${{GOVERSION}}
-
       - multijob:
           name: "Simple Streams"
           condition: SUCCESSFUL
@@ -192,7 +209,6 @@
               current-parameters: true
               predefined-parameters: |-
                 SHORT_GIT_COMMIT=${{SHORT_GIT_COMMIT}}
-
       - multijob:
           name: "Simple Streams Publish"
           condition: SUCCESSFUL
@@ -206,7 +222,6 @@
               current-parameters: true
               predefined-parameters: |-
                 SHORT_GIT_COMMIT=${{SHORT_GIT_COMMIT}}
-
     publishers:
       - trigger-parameterized-builds:
           - project: "ci-gating-tests"
@@ -235,25 +250,24 @@
           timeout: 1200
           fail: true
           type: absolute
-
     parameters:
       - validating-string:
-          description: The git short hash for the commit you wish to test
           name: SHORT_GIT_COMMIT
+          description: The git short hash for the commit you wish to test
           regex: ^\S{{7}}$
           msg: Enter a valid 7 char git sha
       - string:
+          name: series
           default: ""
           description: "Series to use with charms in the functional tests"
-          name: series
       - string:
+          name: BOOTSTRAP_SERIES
           default: ''
           description: 'Ubuntu series to use when bootstrapping Juju'
-          name: BOOTSTRAP_SERIES
       - string:
+          name: GOVERSION
           default: ''
           description: 'Go version used for build.'
-          name: GOVERSION
     builders:
       - get-build-details
       - set-test-description
@@ -296,24 +310,37 @@
           fail: true
           timeout: 240 # Might have to queue for nodes, give some flex
           type: absolute
+      - credentials-binding:
+          - ssh-user-private-key:
+              credential-id: github-pull-ssh-key
+              key-file-variable: GITHUB_SSH_KEY
     parameters:
+      - string:
+          name: GITHUB_REPO
+          default: ""
+          description: "Specifc github repo to pull from."
       - string:
           name: GITHUB_BRANCH_HEAD_SHA
           default: ""
           description: "Specific git SHA to package."
       - string:
-          default: ''
-          description: 'Go version used for build.'
           name: GOVERSION
+          default: ""
+          description: 'Go version used for build.'
+      - string:
+          name: JUJU_BUILD_NUMBER
+          default: ""
+          description: "Juju build number."
     builders:
       - install-go
       - inject:
           properties-content: |-
             JUJU_SOURCE_CHECKOUT=${{WORKSPACE}}/tmp_initial_clone
       - shell: |-
+          export GIT_SSH_COMMAND="ssh -o IdentitiesOnly=yes -i $GITHUB_SSH_KEY -F /dev/null"
           git init "$JUJU_SOURCE_CHECKOUT"
           cd "$JUJU_SOURCE_CHECKOUT"
-          git remote add origin "https://github.com/juju/juju.git"
+          git remote add origin "git@github.com:$GITHUB_REPO.git"
           git config --local gc.auto 0
           git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin "+$GITHUB_BRANCH_HEAD_SHA"
           git checkout --progress --force $GITHUB_BRANCH_HEAD_SHA

jobs/ci-run/scripts/goversion.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 set -ex
 
-gomod=$(curl -s "https://raw.githubusercontent.com/juju/juju/$GIT_COMMIT/go.mod")
+GITHUB_REPO=${GITHUB_REPO:-juju/juju}
+
+if [ -z "${GITHUB_TOKEN}" ]; then
+gomod=$(curl -s "https://raw.githubusercontent.com/$GITHUB_REPO/$GIT_COMMIT/go.mod")
+else
+gomod=$(curl -s "https://api.github.com/repos/$GITHUB_REPO/contents/go.mod?ref=$GIT_COMMIT" --header "Authorization: Bearer $GITHUB_TOKEN" | jq ".content" -r | base64 -d)
+fi
 goversion=$(echo "$gomod" | grep "go 1." | sed 's/^go 1\.\(.*\)$/1.\1/')
 
 if [[ "$goversion" < "1.14" ]]; then

jobs/ci-run/scripts/package-juju-source.sh

@@ -1,11 +1,6 @@
 #!/bin/bash
 set -xe
 
-if [ -z "$GITHUB_BRANCH_NAME" ] && [ -z "$GITHUB_BRANCH_HEAD_SHA" ]; then
-    echo "Must specify one of GITHUB_BRANCH_NAME or GITHUB_BRANCH_HEAD_SHA"
-    exit 1
-fi
-
 if [ -z "$JUJU_SOURCE_CHECKOUT" ]; then
     echo "Must specify JUJU_SOURCE_CHECKOUT"
     exit 1
@@ -26,15 +21,6 @@ full_path=${GOPATH}/src/github.com/juju/juju
 
 export PATH=/snap/bin:$PATH:$GOPATH/bin
 
-# If run with an overriding SHORT_GIT_COMMIT we need to
-# determine the branch name ourselves.
-if [ -z ${GITHUB_BRANCH_NAME} ]; then
-    GITHUB_BRANCH_NAME=$(git -C ${JUJU_SOURCE_CHECKOUT} branch --all \
-        --contains ${GITHUB_BRANCH_HEAD_SHA} \
-        --column \
-        | awk '{print$NF}' \
-        | awk -F/ '{print$NF}')
-fi
 rm -rf ${full_path}
 git clone --depth 1 file://${JUJU_SOURCE_CHECKOUT} ${full_path}
 rm -fr ${JUJU_SOURCE_CHECKOUT}

jobs/ci-run/scripts/resolve-git-sha.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 set -xe
 
+echo "GITHUB_REPO=$GITHUB_REPO"
 echo "GITHUB_BRANCH_NAME=$GITHUB_BRANCH_NAME"
 echo "GITHUB_BRANCH_HEAD_SHA=$GITHUB_BRANCH_HEAD_SHA"
 echo "GIT_COMMIT=$GIT_COMMIT"
@@ -14,21 +15,21 @@ function cleanup {
 
 trap cleanup EXIT
 
-function query_github_simple {
-    curl -s "https://api.github.com/repos/juju/juju/commits/$1" | jq -r ".sha // empty"
+function query_github_simple { \
+    curl -s "https://api.github.com/repos/$GITHUB_REPO/commits/$1" --header "Authorization: Bearer $GITHUB_TOKEN" --header "X-GitHub-Api-Version: 2022-11-28" | jq -r ".sha // empty"
 }
 
 function query_github_treeish {
-    curl -s "https://api.github.com/repos/juju/juju/git/trees/$1" | jq -r ".sha // empty"
+    curl -s "https://api.github.com/repos/$GITHUB_REPO/git/trees/$1" --header "Authorization: Bearer $GITHUB_TOKEN" --header "X-GitHub-Api-Version: 2022-11-28" | jq -r ".sha // empty"
 }
 
 function search_github_hash {
-    curl -s -H "Accept: application/vnd.github.cloak-preview+json" "https://api.github.com/search/commits?q=repo:juju/juju+hash:$1" | jq -r ".items[0].sha // empty"
+    curl -s -H "Accept: application/vnd.github.cloak-preview+json" "https://api.github.com/search/commits?q=repo:$GITHUB_REPO+hash:$1" --header "Authorization: Bearer $GITHUB_TOKEN" --header "X-GitHub-Api-Version: 2022-11-28" | jq -r ".items[0].sha // empty"
 }
 
 function clone_search {
     if [ ! -d "$TMP_CLONE" ]; then
-        git clone -q --no-checkout "https://github.com/juju/juju.git" "$TMP_CLONE"
+        git clone -q --no-checkout "https://github.com/$GITHUB_REPO.git" "$TMP_CLONE"
     fi
     git -C "$TMP_CLONE" rev-parse "$1"
 }
@@ -95,5 +96,6 @@ echo "SHORT_GIT_COMMIT=${EXACT_COMMIT:0:7}" > $PROPS_PATH
 echo "GIT_COMMIT=${EXACT_COMMIT}" >> $PROPS_PATH
 echo "GITHUB_BRANCH_HEAD_SHA=${EXACT_COMMIT}" >> $PROPS_PATH
 echo "GITHUB_BRANCH_NAME=${GITHUB_BRANCH_NAME}" >> $PROPS_PATH
+echo "GITHUB_REPO=${GITHUB_REPO}" >> $PROPS_PATH
 
 cat $PROPS_PATH

tests/suites/static_analysis/deadcode/main.go

@@ -190,6 +190,10 @@ func extractMultiJobNames(multijob map[interface{}]interface{}) []string {
 
 func ignore(dir string, skip []string, fn func(string, os.FileInfo) error) func(string, os.FileInfo) error {
 	return func(path string, info os.FileInfo) error {
+		if strings.HasSuffix(path, ".private.yml") ||
+			strings.HasSuffix(path, ".private.yaml") {
+			return nil
+		}
 		b, err := filepath.Rel(dir, path)
 		if err != nil {
 			return err

tests/suites/static_analysis/lint_yaml.sh

@@ -42,6 +42,7 @@ jobs:
   ignore:
     - build-dqlite
     - build-musl
+    - ci-trigger-github-com-juju-juju
     - ci-build-juju
     - ci-gating-tests
     - ci-proving-ground-tests
@@ -78,7 +79,6 @@ jobs:
     - gating-functional-tests-ppc64el
     - test-refresh-multijob
     - build-jujud-operator-test
-
     # TODO (stickupkid): The followng jobs seem to be orphan jobs with in the
     # jenkins suite. We should clean them up to ensure that they do run, or
     # are removed.