[DRAFT] CI experiments

NVIDIA · Apr 21, 2024 · 0d24ee7 · 0d24ee7
1 parent 29a24f4
commit 0d24ee7
Show file tree

Hide file tree

Showing 8 changed files with 892 additions and 298 deletions.
diff --git a/.github/actions/compute-matrix/compute-matrix.sh b/.github/actions/compute-matrix/compute-matrix.sh
@@ -2,6 +2,9 @@
 
 set -euo pipefail
 
+# Ensure the script is being executed in its containing directory
+cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
 write_output() {
   local key="$1"
   local value="$2"
@@ -13,7 +16,7 @@ explode_std_versions() {
 }
 
 explode_libs() {
-  jq -cr 'map(. as $o | {lib: $o.lib[]} + del($o.lib))'
+  jq -cr 'map(. as $o | {project: $o.project[]} + del($o.project))'
 }
 
 # Filter out the libraries that are dirty
@@ -34,7 +37,10 @@ filter_libs() {
   # echo "Dirty libraries: ${dirty_libs[@]}" >> /dev/stderr
 
   # Construct a regex to filter out the dirty libraries
-  dirty_lib_regex=$(IFS="|"; echo "${dirty_libs[*]}")
+  dirty_lib_regex=$(
+    IFS="|"
+    echo "${dirty_libs[*]}"
+  )
   dirty_lib_regex="^(${dirty_lib_regex})\$"
   jq_filter="map(select(.lib | test(\"$dirty_lib_regex\")))"
   jq -cr "$jq_filter"
@@ -44,31 +50,63 @@ extract_matrix() {
   local file="$1"
   local type="$2"
   local matrix=$(yq -o=json "$file" | jq -cr ".$type")
-  write_output "DEVCONTAINER_VERSION" "$(yq -o json "$file" | jq -cr '.devcontainer_version')"
+  local devcontainer_version=$(yq -o json "$file" | jq -cr '.devcontainer_version')
+
+  write_output "DEVCONTAINER_VERSION" "$devcontainer_version"
 
-  local nvcc_full_matrix="$(echo "$matrix" | jq -cr '.nvcc' | explode_std_versions )"
-  local per_cuda_compiler_matrix="$(echo "$nvcc_full_matrix" | jq -cr ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')"
-  write_output "PER_CUDA_COMPILER_MATRIX"  "$per_cuda_compiler_matrix"
-  write_output "PER_CUDA_COMPILER_KEYS" "$(echo "$per_cuda_compiler_matrix" | jq -r 'keys | @json')"
+  local full_matrix="$(echo "$matrix" | explode_std_versions | explode_libs)" # | filter_libs)"
 
-  write_output "NVRTC_MATRIX" "$(echo "$matrix" | jq '.nvrtc' | explode_std_versions)"
+  # For each json object in the nvcc matrix array, accumulate the output json object from `generate-ci-dispatch-job.sh`
+  # into a single json object. If duplicate keys are found, merge the matching value arrays.
+  local wf_json="{}"
+  local input_objects=$(echo "$full_matrix" | jq -cr '.[]')
+  for input_object in $input_objects; do
+    local output_object=$(./generate-ci-dispatch-job.sh "$devcontainer_version" "${input_object}")
+    wf_json=$(
+      jq --argjson accum "$wf_json" --argjson new "$output_object" '
+        $new | . as $input |
+        reduce keys[] as $key (
+            $accum;
+            if .[$key] then
+                .[$key] |= (
+                    reduce ($input[$key] | keys_unsorted[]) as $nestedKey (.;
+                        .[$nestedKey] += $input[$key][$nestedKey]
+                    )
+                )
+            else
+                .[$key] = $input[$key]
+            end
+        )
+    ' <<<'{}')
+  done
+  wf_keys=$(echo "$wf_json" | jq -r 'keys | @json')
 
-  local clang_cuda_matrix="$(echo "$matrix" | jq -cr '.["clang-cuda"]' | explode_std_versions | explode_libs | filter_libs)"
-  write_output "CLANG_CUDA_MATRIX" "$clang_cuda_matrix"
-  write_output "CCCL_INFRA_MATRIX" "$(echo "$matrix" | jq -cr '.["cccl-infra"]' )"
+  write_output "WORKFLOW" "$wf_json"
+  write_output "WORKFLOW_KEYS" "$wf_keys"
 }
 
+# local per_cuda_compiler_matrix="$(echo "$nvcc_full_matrix" | jq -cr ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')"
+# write_output "PER_CUDA_COMPILER_MATRIX"  "$per_cuda_compiler_matrix"
+# write_output "PER_CUDA_COMPILER_KEYS" "$(echo "$per_cuda_compiler_matrix" | jq -r 'keys | @json')"
+
+# write_output "NVRTC_MATRIX" "$(echo "$matrix" | jq '.nvrtc' | explode_std_versions)"
+
+# local clang_cuda_matrix="$(echo "$matrix" | jq -cr '.["clang-cuda"]' | explode_std_versions | explode_libs | filter_libs)"
+# write_output "CLANG_CUDA_MATRIX" "$clang_cuda_matrix"
+# write_output "CCCL_INFRA_MATRIX" "$(echo "$matrix" | jq -cr '.["cccl-infra"]' )"
+# }
+
 main() {
   if [ "$1" == "-v" ]; then
     set -x
     shift
   fi
 
-  if [ $# -ne 2 ] || [ "$2" != "pull_request" ]; then
+  if [ $# -ne 2 ]; then
     echo "Usage: $0 [-v] MATRIX_FILE MATRIX_TYPE"
     echo "  -v            : Enable verbose output"
     echo "  MATRIX_FILE   : The path to the matrix file."
-    echo "  MATRIX_TYPE   : The desired matrix. Supported values: 'pull_request'"
+    echo "  MATRIX_TYPE   : The desired matrix."
     exit 1
   fi