bats.yaml

name: BATS
on:
  workflow_dispatch:
    inputs:
      owner:
        description: Override owner (e.g. rancher-sandbox)
        type: string
      repo:
        description: Override repository (e.g. rancher-desktop)
        type: string
      branch:
        description: Override branch (e.g. main, or PR#)
        type: string
      tests:
        description: 'Tests (in the tests/ directory, e.g. "containers")'
        default: '*'
        type: string
      platforms:
        description: Platforms to run
        default: 'linux mac win'
        type: string
      engines:
        description: Container engines to run
        default: 'containerd moby'
        type: string
      kubernetes-version:
        description: Primary Kubernetes version to test
        default: '1.22.7' # Must also change in calculate step
        type: string
      kubernetes-alt-version:
        description: Secondary Kubernetes version to test (e.g. for upgrades)
        default: '1.28.11' # Must also change in calculate step
        type: string
      package-id:
        description: Package run ID override; leave empty to use latest.
        default: ''
        type: string
  schedule:
  - cron: '0 8 * * 1-5' # 8AM UTC weekdays as a baseline

permissions:
  contents: read

env:
  GH_OWNER:      ${{ github.repository_owner }}
  GH_REPOSITORY: ${{ github.repository }}
  GH_REF_NAME:   ${{ github.ref_name }}

jobs:
  get-tests:
    name: Calculate tests to run
    runs-on: ubuntu-latest
    steps:
    - name: Fetch install script
      uses: actions/checkout@v4
      with:
        persist-credentials: false
        sparse-checkout-cone-mode: false
        sparse-checkout: |
          scripts/install-latest-ci.sh
          .github/workflows/bats/get-tests.py

    - id: repo
      name: Calculate short repository
      run: echo "repo=${GITHUB_REPOSITORY#*/}" >> "$GITHUB_OUTPUT"

    - name: Fetch tests
      run: |
        : ${OWNER:=$GH_OWNER}
        : ${REPO:=${GH_REPOSITORY#$GH_OWNER/}}
        : ${BRANCH:=$GH_REF_NAME}
        # If BRANCH is a number, assume it is supposed to be a PR
        [[ $BRANCH =~ ^[0-9]+$ ]] && export PR=$BRANCH
        "scripts/install-latest-ci.sh"
      env:
        GH_TOKEN:     ${{ github.token }}
        OWNER:        ${{ inputs.owner || github.repository_owner }}
        REPO:         ${{ inputs.repo || steps.repo.outputs.repo }}
        BRANCH:       ${{ inputs.branch || github.ref_name }}
        ID:           ${{ inputs.package-id }}
        BATS_DIR:     ${{ github.workspace }}/bats
        INSTALL_MODE: skip

    - name: Calculate tests
      id: calculate
      # This script is not inline to make local testing easier
      run: python3 ${{ github.workspace }}/.github/workflows/bats/get-tests.py
      env:
        TESTS:                  ${{ inputs.tests }}
        PLATFORMS:              ${{ inputs.platforms }}
        ENGINES:                ${{ inputs.engines }}
        KUBERNETES_VERSION:     ${{ inputs.kubernetes-version || '1.22.7' }}
        # rancher/rancher helm chart 2.8.5 supports up to 1.28.*
        KUBERNETES_ALT_VERSION: ${{ inputs.kubernetes-alt-version || '1.28.11' }}
      working-directory: bats/tests
    outputs:
      repo: ${{ steps.repo.outputs.repo }}
      tests: ${{ steps.calculate.outputs.tests }}

  bats:
    needs: get-tests
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.get-tests.outputs.tests )}}
    runs-on: ${{ matrix.host }}
    steps:
    - name: Fetch install script
      uses: actions/checkout@v4
      with:
        persist-credentials: false
        sparse-checkout-cone-mode: false
        sparse-checkout: |
          scripts/install-latest-ci.sh
          .github/actions/setup-environment/action.yaml
          .github/workflows/bats/sanitize-artifact-name.sh

    - name: Install latest CI build
      run: |
        : ${OWNER:=$GH_OWNER}
        : ${REPO:=${GH_REPOSITORY#$GH_OWNER/}}
        : ${BRANCH:=$GH_REF_NAME}
        # If BRANCH is a number, assume it is supposed to be a PR
        [[ $BRANCH =~ ^[0-9]+$ ]] && export PR=$BRANCH
        scripts/install-latest-ci.sh
      shell: bash
      env:
        GH_TOKEN:     ${{ github.token }}
        OWNER:        ${{ inputs.owner || github.repository_owner }}
        REPO:         ${{ inputs.repo || needs.get-tests.outputs.repo }}
        BRANCH:       ${{ inputs.branch || github.ref_name }}
        ID:           ${{ inputs.package-id }}
        BATS_DIR:     ${{ github.workspace }}/bats
        INSTALL_MODE: installer
        ZIP_NAME:     ${{ github.workspace }}/version.txt
        RD_LOCATION:  system

    - name: Set up environment
      uses: ./.github/actions/setup-environment

    - name: "Linux: Install prerequisites"
      if: runner.os == 'Linux'
      run: >-
        sudo DEBIAN_FRONTEND=noninteractive
        apt-get install coreutils

    - name: "macOS: Install prerequisites"
      if: runner.os == 'macOS'
      shell: bash
      run: brew install --force bash coreutils

    - name: "Windows: Install WSL2 Distribution"
      if: runner.os == 'Windows'
      shell: powershell # pwsh doesn't have Add-AppxPackage
      run: |
        # `wsl --install` seems to have issues in CI since 2024-06-14; however,
        # manually downloading the Debian installer and running it works.
        Invoke-WebRequest https://github.com/microsoft/WSL/raw/master/distributions/DistributionInfo.json |
          Select-Object -ExpandProperty Content |
          ConvertFrom-JSON |
          Select-Object -ExpandProperty Distributions |
          Where-Object Name -EQ "Debian" |
          Select-Object -ExpandProperty Amd64PackageUrl |
          % { Invoke-WebRequest $_ -OutFile Debian.AppxBundle }
        Add-AppxPackage Debian.AppxBundle
        # Initialize Debian, without going through any first-time setup
        debian.exe install --root

    - name: "Windows: Install prerequisites in WSL"
      if: runner.os == 'Windows'
      shell: pwsh
      run: >-
        wsl.exe -d Debian --exec /usr/bin/env DEBIAN_FRONTEND=noninteractive
        sh -c 'apt-get update && apt-get install --yes bsdextrautils coreutils curl'

    - name: Set log directory
      shell: bash
      run: |
        echo "LOGS_DIR=$(pwd)/logs" >> "$GITHUB_ENV"
        mkdir logs
    - name: "Windows: Override log directory"
      if: runner.os == 'Windows'
      shell: powershell
      run: >-
        wsl.exe -d Debian -- echo 'LOGS_DIR=$(pwd)'
        | Out-File -Encoding ASCII -Append "$ENV:GITHUB_ENV"
      working-directory: logs

    - name: Normalize test name
      id: normalize
      shell: bash
      run: |
        t="${{ matrix.name }}"
        if [[ ! -r "tests/$t" ]] && [[ -r "tests/${t}.bats" ]]; then
          t="${t}.bats"
        fi
        echo "test=$t" >> "$GITHUB_OUTPUT"
      working-directory: bats

    - name: "macOS: Set startup command"
      if: runner.os == 'macOS'
      run: echo "BATS_COMMAND=$BATS_COMMAND" >> "$GITHUB_ENV"
      env:
        BATS_COMMAND: exec
    - name: "Linux: Set startup command"
      if: runner.os == 'Linux'
      run: echo "BATS_COMMAND=$BATS_COMMAND" >> "$GITHUB_ENV"
      env:
        BATS_COMMAND: >-
          exec xvfb-run --auto-servernum
          --server-args='-screen 0 1280x960x24'
    - name: "Windows: Set startup command"
      if: runner.os == 'Windows'
      shell: bash
      run: echo "BATS_COMMAND=$BATS_COMMAND" >> "$GITHUB_ENV"
      env:
        BATS_COMMAND: wsl.exe -d Debian --exec

    - name: Run BATS
      # We use ${{ env.BATS_COMMAND }} instead of ${BATS_COMMAND} to let the
      # shell parse the command, instead of doing it via expansion which is then
      # parsed differently (--server-args isn't kept as one word).  Also, we
      # need to use the env.* form because PowerShell uses ${ENV:VAR} instead.
      run: >-
        ${{ env.BATS_COMMAND }}
        ./bats-core/bin/bats
        --gather-test-outputs-in '${{ env.LOGS_DIR }}'
        --print-output-on-failure
        --filter-tags '!ci-skip'
        --formatter cat
        --report-formatter tap
        'tests/${{ steps.normalize.outputs.test }}'
      env:
        BATS_COMMAND:              ${{ env.BATS_COMMAND }}
        GITHUB_TOKEN:              ${{ github.token }}
        LOGS_DIR:                  ${{ env.LOGS_DIR }}
        RD_CAPTURE_LOGS:           "true"
        RD_CONTAINER_ENGINE:       ${{ matrix.engine }}
        RD_KUBERNETES_VERSION:     ${{ matrix.k3sVersion }}
        RD_KUBERNETES_ALT_VERSION: ${{ matrix.k3sAltVersion }}
        RD_TAKE_SCREENSHOTS:       "true"
        RD_TRACE:                  "true"
        RD_USE_GHCR_IMAGES:        "true"
        RD_USE_RAMDISK:            "true"
        RD_USE_WINDOWS_EXE:        "${{ runner.os == 'Windows' }}"
        WSLENV: "\
          GITHUB_TOKEN:\
          RD_CAPTURE_LOGS:\
          RD_CONTAINER_ENGINE:\
          RD_KUBERNETES_VERSION:\
          RD_KUBERNETES_ALT_VERSION:\
          RD_TAKE_SCREENSHOTS:\
          RD_TRACE:\
          RD_USE_GHCR_IMAGES:\
          RD_USE_RAMDISK:\
          RD_USE_WINDOWS_EXE:\
          "
      working-directory: bats
      timeout-minutes: 120

    - name: Calculate log name
      id: log_name
      if: ${{ !cancelled() }}
      run: |
        name="$(.github/workflows/bats/sanitize-artifact-name.sh <<< "$name")"
        # For the artifact name, backslash and forward slash are also invalid.
        name=${name//\\/%3C}
        name=${name//\//%2F}
        echo "name=$name" >>"$GITHUB_OUTPUT"
      shell: bash
      env:
        name: ${{ matrix.host }}-${{ matrix.engine }}-${{ matrix.name }}.logs

    - name: Consolidate logs
      if: ${{ !cancelled() }}
      run: |
        # bats/logs may not exist if the workflow is being tested with e.g. tests/helpers/utils.bats
        if [ -d "bats/logs" ]; then
            cp -R "bats/logs/" logs
        fi
        cp "bats/report.tap" logs
        .github/workflows/bats/sanitize-artifact-name.sh logs
        echo "$NAME" > logs/name.txt
        echo "$OS" > logs/os.txt
        echo "$ENGINE" > logs/engine.txt
        echo "$LOG_NAME" > logs/log-name.txt
        mv version.txt logs/
      shell: bash
      env:
        NAME: ${{ matrix.name }}
        OS: ${{ matrix.host }}
        ENGINE: ${{ matrix.engine }}
        LOG_NAME: ${{ steps.log_name.outputs.name }}

    - name: Upload logs
      if: ${{ !cancelled() }}
      uses: actions/upload-artifact@v4
      with:
        name: ${{ steps.log_name.outputs.name }}
        path: logs/
        if-no-files-found: error

  summarize:
    name: Summarize output
    needs: [ get-tests, bats ]
    if: ${{ !cancelled() }}
    runs-on: ubuntu-latest
    steps:
    - name: Fetch summarizer script
      uses: actions/checkout@v4
      with:
        persist-credentials: false
        sparse-checkout-cone-mode: false
        sparse-checkout: |
          package.json
          .github/workflows/bats/summarize.mjs
    - uses: actions/setup-node@v4
      with:
        node-version-file: package.json
    - uses: actions/download-artifact@v4
      with:
        pattern: "*.logs"
    - run: node .github/workflows/bats/summarize.mjs
      env:
        EXPECTED_TESTS: ${{ needs.get-tests.outputs.tests }}