diff --git a/.github/actions/check-downstream-compiles/action.yml b/.github/actions/check-downstream-compiles/action.yml index b3ec153..964fe8d 100644 --- a/.github/actions/check-downstream-compiles/action.yml +++ b/.github/actions/check-downstream-compiles/action.yml @@ -14,107 +14,24 @@ inputs: description: 'Path to upstream repo' required: true type: string - # Patch with an HTTPS or SSH URL. Defaults to HTTPS - patch-ssh: - description: 'Patch with HTTPS or SSH' - required: false - default: false - type: boolean - # Comma-separated list of crates to patch further down the dependency chain - more-paths: - description: 'Paths to further downstream' + # `[patch.]`, defaults to HTTPS URL from upstream caller repo + patch-type: + description: 'Patch with HTTPS, SSH, or crates.io' required: false + default: 'https' type: string runs: using: "composite" steps: # Assumes at least one dependency in the current workspace is used by the downstream crate - - name: Patch Cargo.toml + - name: Patch Cargo.toml files in the downstream repo with all uses of the upstream-repo shell: bash - working-directory: ${{ github.workspace }}/${{ inputs.downstream-path }} + working-directory: ${{ github.workspace }}/ci-workflows/crates/check-downstream-compiles run: | - if [[ "${{ inputs.patch-ssh }}" == "true" ]]; then - URL=ssh://git@github.com/${{ github.repository }} - else - URL=https://github.com/${{ github.repository }} - fi - - # Get a list of all upstream dependencies used by the downstream crate workspace - # This is done by checking for each instance of `git = ` in any of the downstream `Cargo.toml` files - DEPENDENCIES=$(grep -rsh "git = \"$URL" --include="Cargo.toml" .) - - # Extract the dependency names and check for package renames, removing duplicates - DEP_NAMES=$(echo "$DEPENDENCIES" | awk '/package =/{for (i=1; i<=NF; i++) if ($i == "package") {name=$(i+2); print substr(name, 2, length(name)-2);} found=1} !/package =/{print $1}' | sort -u) - - shopt -s nullglob - # Collect the `(path, package)` pairs for most subcrates in the upstream directory, regardless of whether they are workspace members - # Patch paths are absolute so they can be used in any Cargo.toml - SUBCRATES=$(find ${{ github.workspace }}/${{ inputs.upstream-path }} \( -type d \( -name target -o -name examples -o -name tests -o -name cli \) -prune \) -o -name Cargo.toml -exec awk '/^\[package\]/{flag=1} flag && /name\s*=/ {gsub(/"/, "", $3); package=$3} /^\[[^p]/ && flag {exit} END{if(package != "") print FILENAME, package}' {} \; | sed 's|/Cargo.toml | |g') - shopt -u nullglob - - # Store the subcrates in associative array for retrieval when patching `Cargo.toml` - declare -A subcrates - while IFS= read -r line; do - pair=($line) - subcrates[${pair[1]}]=${pair[0]} - done <<< "$SUBCRATES" - - # Get list of Git patches for each dependency used downstream - for crate in $DEP_NAMES; do - crate_path="${subcrates[$crate]}" - echo "$crate = { path = \"$crate_path\" }" | tee -a patches.txt - done - - ESCAPED_URL=$(printf '%s\n' "$URL" | sed 's/[\/&]/\\&/g') - - # Write patches to all Cargo.toml files in the directory - for file in $(find . -name Cargo.toml -not -path "./target/*"); do - # Add the `[patch]` section if it doesn't exist already, as duplicates aren't allowed - if ! grep -q "\[patch.'$URL'\]" $file; then - printf "\n\n[patch.'$URL']\n" >> $file - # If `[patch]` does exist, remove any duplicate keys before inserting patches - else - for crate in $DEP_NAMES; do - sed -i "/\[patch.'$ESCAPED_URL'\]/,/^$/ { /^$crate.*$/d }" $file - done - fi - - # Append all patches after the `[patch.'ssh']` line - sed -i "/\[patch.'$ESCAPED_URL'\]/r patches.txt" $file - done - - # If more crates were specified, patch them too - if [[ ! -z "${{ inputs.more-paths }}" ]]; then - for path in "${{ inputs.more-paths }}"; do - for file in $(find $path -name Cargo.toml -not -path "./target/*"); do - # Add the `[patch]` section if it doesn't exist already, as duplicates aren't allowed - if ! grep -q "\[patch.'$URL'\]" $file; then - printf "\n\n[patch.'$URL']\n" >> $file - # If `[patch]` does exist, remove any duplicate keys before inserting patches - else - for crate in $DEP_NAMES; do - sed -i "/\[patch.'$ESCAPED_URL'\]/,/^$/ { /^$crate.*$/d }" $file - done - fi - - # Append all patches after the `[patch.'ssh']` line - sed -i "/\[patch.'$ESCAPED_URL'\]/r patches.txt" $file - done - done - fi + cargo run -- --upstream ${{ github.workspace }}/${{ inputs.upstream-path }} --downstream ${{ github.workspace }}/${{ inputs.downstream-path }} --repo ${{ github.repository }} --patch-type ${{ inputs.patch-type }} - name: Check downstream types don't break spectacularly shell: bash working-directory: ${{ github.workspace }}/${{ inputs.downstream-path }} run: | - cargo check --workspace --tests --benches --examples - - name: Check more downstream types don't break spectacularly - if: ${{ inputs.more-paths != '' }} - shell: bash - working-directory: ${{ github.workspace }} - run: | - for path in "${{ inputs.more-paths }}"; do - cd $path - cargo check --workspace --tests --benches --examples - cd ${{ github.workspace }} - done + cargo check --workspace --all-targets diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9ae90a3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +name: CI Tests + +on: + push: + branches: main + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + test: + name: CI Test Suite + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + repository: argumentcomputer/ci-workflows + - uses: ./.github/actions/ci-env + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@nextest + - name: Run tests + run: cargo nextest run + working-directory: ${{ github.workspace }}/crates/check-downstream-compiles + + clippy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + repository: argumentcomputer/ci-workflows + - uses: ./.github/actions/ci-env + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Check Rustfmt Code Style + run: cargo fmt --all --check + working-directory: ${{ github.workspace }}/crates/check-downstream-compiles + - name: check *everything* compiles + run: cargo check --all-targets --all-features --all --examples --tests --benches + working-directory: ${{ github.workspace }}/crates/check-downstream-compiles + # See '.cargo/config' for list of enabled/disabled clippy lints + - name: Check clippy warnings + run: cargo xclippy -D warnings + working-directory: ${{ github.workspace }}/crates/check-downstream-compiles diff --git a/.gitignore b/.gitignore index 2c96eb1..764515c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -target/ -Cargo.lock +**/target/ +**/Cargo.lock diff --git a/README.md b/README.md index 9c751d9..9f20cdf 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ # ci-workflows -GitHub Actions workflows and templates for use in Lurk repos +GitHub Actions workflows and templates for use in Argument repos diff --git a/Cargo.toml b/crates/benchmark-plotter/Cargo.toml similarity index 100% rename from Cargo.toml rename to crates/benchmark-plotter/Cargo.toml diff --git a/build.rs b/crates/benchmark-plotter/build.rs similarity index 100% rename from build.rs rename to crates/benchmark-plotter/build.rs diff --git a/rust-toolchain.toml b/crates/benchmark-plotter/rust-toolchain.toml similarity index 100% rename from rust-toolchain.toml rename to crates/benchmark-plotter/rust-toolchain.toml diff --git a/src/json.rs b/crates/benchmark-plotter/src/json.rs similarity index 100% rename from src/json.rs rename to crates/benchmark-plotter/src/json.rs diff --git a/src/main.rs b/crates/benchmark-plotter/src/main.rs similarity index 100% rename from src/main.rs rename to crates/benchmark-plotter/src/main.rs diff --git a/src/plot.rs b/crates/benchmark-plotter/src/plot.rs similarity index 100% rename from src/plot.rs rename to crates/benchmark-plotter/src/plot.rs diff --git a/crates/check-downstream-compiles/.cargo/config b/crates/check-downstream-compiles/.cargo/config new file mode 100644 index 0000000..0e515fd --- /dev/null +++ b/crates/check-downstream-compiles/.cargo/config @@ -0,0 +1,50 @@ +[alias] +# Collection of project wide clippy lints. This is done via an alias because +# clippy doesn't currently allow for specifiying project-wide lints in a +# configuration file. This is a similar workaround to the ones presented here: +# +xclippy = [ + "clippy", "--workspace", "--all-targets", "--all-features", "--", + "-Wclippy::all", + "-Wclippy::cast_lossless", + "-Wclippy::checked_conversions", + "-Wclippy::dbg_macro", + "-Wclippy::disallowed_methods", + "-Wclippy::derive_partial_eq_without_eq", + "-Wclippy::enum_glob_use", + "-Wclippy::explicit_into_iter_loop", + "-Wclippy::fallible_impl_from", + "-Wclippy::filter_map_next", + "-Wclippy::flat_map_option", + "-Wclippy::from_iter_instead_of_collect", + "-Wclippy::implicit_clone", + "-Wclippy::inefficient_to_string", + "-Wclippy::invalid_upcast_comparisons", + "-Wclippy::large_stack_arrays", + "-Wclippy::large_types_passed_by_value", + "-Wclippy::macro_use_imports", + "-Wclippy::manual_assert", + "-Wclippy::manual_ok_or", + "-Wclippy::map_flatten", + "-Wclippy::map_unwrap_or", + "-Wclippy::match_same_arms", + "-Wclippy::match_wild_err_arm", + "-Wclippy::missing_const_for_fn", + "-Wclippy::needless_borrow", + "-Wclippy::needless_continue", + "-Wclippy::needless_for_each", + "-Wclippy::needless_pass_by_value", + "-Wclippy::option_option", + "-Wclippy::same_functions_in_if_condition", + "-Wclippy::single_match_else", + "-Wclippy::trait_duplication_in_bounds", + "-Wclippy::unnecessary_wraps", + "-Wclippy::unnested_or_patterns", + "-Wnonstandard_style", + "-Wrust_2018_idioms", + "-Wtrivial_numeric_casts", + "-Wunused_lifetimes", + "-Wunreachable_pub", + "-Wtrivial_numeric_casts", + "-Wunused_qualifications", +] diff --git a/crates/check-downstream-compiles/Cargo.toml b/crates/check-downstream-compiles/Cargo.toml new file mode 100644 index 0000000..9b6ef50 --- /dev/null +++ b/crates/check-downstream-compiles/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "check-downstream-compiles" +version = "0.1.0" +edition = "2021" + +[dependencies] +camino = "1.1.7" +clap = { version = "4.5.13", features = ["derive"] } +toml_edit = "0.22.20" +walkdir = "2.5.0" diff --git a/crates/check-downstream-compiles/src/main.rs b/crates/check-downstream-compiles/src/main.rs new file mode 100644 index 0000000..f144f48 --- /dev/null +++ b/crates/check-downstream-compiles/src/main.rs @@ -0,0 +1,172 @@ +use std::collections::BTreeMap; +use std::fs; + +use camino::Utf8PathBuf; +use clap::{Parser, ValueEnum}; +use toml_edit::{value, DocumentMut, Item, Table}; +use walkdir::WalkDir; + +/// CLI to patch a downstream repo and check it compiles +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +struct Args { + /// Path to upstream crate + #[arg(long)] + upstream: String, + + /// Path to downstream crate + #[arg(long)] + downstream: String, + + /// The type of patch in `[patch.]` + #[arg(long, value_enum, default_value_t = PatchType::default())] + patch_type: PatchType, + + /// The org/repo name to be patched via GitHub URL, e.g. argumentcomputer/sphinx + #[arg(long)] + repo: String, +} + +#[derive(Debug, Clone, Default, ValueEnum)] +enum PatchType { + // TODO + CratesIO, + Ssh, + #[default] + Https, +} + +fn main() { + let args = Args::parse(); + + let mut upstream_packages: BTreeMap = BTreeMap::new(); + + // Get all the upstream crates and their paths + for entry in WalkDir::new(args.upstream) + .into_iter() + .filter_map(|e| e.ok()) + { + let path = entry.path(); + if let Some(file_name) = path.file_name() { + if file_name == "Cargo.toml" { + let dir = path.parent().expect("No parent for Cargo.toml"); + let cargo_toml_content = fs::read_to_string(path).expect("FS err"); + let doc = cargo_toml_content + .parse::() + .expect("Parse err"); + if let Some(package) = doc.get("package") { + if let Some(name) = package.get("name") { + let dep_name = name.as_str().unwrap().to_string(); + upstream_packages.insert( + dep_name, + Utf8PathBuf::from_path_buf(dir.to_owned()).unwrap(), + ); + } + } + } + } + } + + //println!("upstream packages: {:?}", upstream_packages); + + let mut downstream_packages: BTreeMap = BTreeMap::new(); + + // Get all the upstream crates that are used in the downstream repo + for entry in WalkDir::new(&args.downstream) + .into_iter() + .filter_map(|e| e.ok()) + { + let path = entry.path(); + if let Some(file_name) = path.file_name() { + if file_name == "Cargo.toml" { + let cargo_toml_content = fs::read_to_string(path).expect("FS err"); + let doc = cargo_toml_content + .parse::() + .expect("Parse err"); + + if let Some(Item::Table(deps)) = doc.get("dependencies") { + for (dep_name, dep_value) in deps.iter() { + if let Some(table) = dep_value.as_inline_table() { + if table.get("git").is_some() { + if let Some(dir) = upstream_packages.get(dep_name) { + downstream_packages.insert(dep_name.to_owned(), dir.clone()); + } + } + } + } + } + } + } + } + + //println!("downstream packages: {:?}", downstream_packages); + + let patch_str = patch_string(&args.patch_type, &args.repo); + + // Patch each downstream crate with the upstream crates + // Iterate through each crate in the downstream workspace + // Read each Cargo.toml file into toml_edit + // Write the patches for each patch in downstream_packages + for entry in WalkDir::new(&args.downstream) + .into_iter() + .filter_map(|e| e.ok()) + { + let path = entry.path(); + if let Some(file_name) = path.file_name() { + if file_name == "Cargo.toml" { + let cargo_toml_content = fs::read_to_string(path).expect("FS err"); + let mut doc = cargo_toml_content + .parse::() + .expect("Parse err"); + + // Ensure [patch.] table exists, create it if it doesn't + if let Some(Item::Table(patch)) = doc.get_mut("patch") { + if let Some(Item::Table(patch_table)) = patch.get_mut(&patch_str) { + // Add entries to the existing [patch.] table + for (pkg, dir) in downstream_packages.iter() { + add_patch(patch_table, pkg, dir.as_str()); + } + } else { + // Create the [patch.] table and add entries + let mut patch_table = Table::new(); + for (pkg, dir) in downstream_packages.iter() { + add_patch(&mut patch_table, pkg, dir.as_str()); + } + patch[&patch_str] = Item::Table(patch_table); + } + } else { + // Create the [patch] table, then the [patch.] table and add entries + let mut patch = Table::new(); + patch.set_implicit(true); + let mut patch_table = Table::new(); + for (pkg, dir) in downstream_packages.iter() { + add_patch(&mut patch_table, pkg, dir.as_str()); + } + patch[&patch_str] = Item::Table(patch_table); + doc["patch"] = Item::Table(patch); + } + + //println!("file: {:?}\n{doc}", path); + + fs::write(path, doc.to_string()).expect("Failed to write"); + } + } + } +} + +// TODO: Clean this up with a From/Display impl +fn patch_string(patch_type: &PatchType, repo: &str) -> String { + match patch_type { + PatchType::CratesIO => String::from("crates-io"), + PatchType::Ssh => format!("ssh://git@github.com/{repo}"), + PatchType::Https => format!("https://github.com/{repo}"), + } +} + +fn add_patch(patch_table: &mut Table, dep: &str, path: &str) { + patch_table[dep]["path"] = value(path); + patch_table[dep] + .as_inline_table_mut() + .unwrap() //_or_else(|| bail!("Failed to get mutable table for {dep}")) + .fmt(); +}