sbt.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


name: Celeborn SBT CI

on:
  push:
    branches:
      - main
      - branch-*
  pull_request:
    branches:
      - main
      - branch-*

jobs:
  service:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        java:
          - 8
          - 11
          - 17
        scala:
          - '2.12.15'
          - '2.13.5'
    steps:
    - uses: actions/checkout@v4
    - name: Setup JDK ${{ matrix.java }}
      uses: actions/setup-java@v4
      with:
        distribution: zulu
        java-version: ${{ matrix.java }}
        check-latest: false
    - name: Test Service with SBT
      run: |
        build/sbt ++${{ matrix.scala }} "clean; test"
    - name: Upload test log
      if: failure()
      uses: actions/upload-artifact@v4
      with:
          name: service-java-${{ matrix.java }}-scala-${{ matrix.scala }}-unit-test-log
          path: |
              **/target/test-reports/**
              **/target/unit-tests.log

  spark2:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        java:
          - 8
        spark:
          - '2.4'
        scala:
          - '2.11.12'
          - '2.12.10'
    steps:
      - uses: actions/checkout@v4
      - name: Setup JDK ${{ matrix.java }}
        uses: actions/setup-java@v4
        with:
          distribution: zulu
          java-version: ${{ matrix.java }}
          check-latest: false
      - name: Test with SBT
        run: |
          build/sbt -Pspark-${{ matrix.spark }} ++${{ matrix.scala }} "clean; celeborn-spark-group/test"
      - name: Upload test log
        if: failure()
        uses: actions/upload-artifact@v4
        with:
            name: spark-${{ matrix.spark }}-unit-test-log
            path: |
                **/target/test-reports/**
                **/target/unit-tests.log
  spark3:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        spark:
          - '3.0'
          - '3.1'
          - '3.2'
          - '3.3'
          - '3.4'
          - '3.5'
        java:
          - 8
          - 11
          - 17
        scala-binary:
          - '2.12'
          - '2.13'
        shuffle-plugin-class:
          - 'org.apache.spark.shuffle.sort.io.LocalDiskShuffleDataIO'
          - 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
        exclude:
          # SPARK-33772: Spark supports JDK 17 since 3.3.0
          - java: 17
            spark: '3.0'
          - java: 17
            spark: '3.1'
          - java: 17
            spark: '3.2'
          # Spark supports scala 2.13 since 3.2.0
          - scala-binary: '2.13'
            spark: '3.0'
          - scala-binary: '2.13'
            spark: '3.1'
          # ShuffleDriverComponents#supportsReliableStorage was introduced in 3.5.0
          - shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
            spark: '3.0'
          - shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
            spark: '3.1'
          - shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
            spark: '3.2'
          - shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
            spark: '3.3'
          - shuffle-plugin-class: 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
            spark: '3.4'
        include:
          # Spark 3.0
          - spark: '3.0'
            scala-binary: '2.12'
            scala: '2.12.10'
          # Spark 3.1
          - spark: '3.1'
            scala-binary: '2.12'
            scala: '2.12.10'
          - spark: '3.1'
          # Spark 3.2
          - spark: '3.2'
            scala-binary: '2.12'
            scala: '2.12.15'
          - spark: '3.2'
            scala-binary: '2.13'
            scala: '2.13.5'
          # Spark 3.3
          - spark: '3.3'
            scala-binary: '2.12'
            scala: '2.12.15'
          - spark: '3.3'
            scala-binary: '2.13'
            scala: '2.13.5'
          # Spark 3.4
          - spark: '3.4'
            scala-binary: '2.12'
            scala: '2.12.17'
          - spark: '3.4'
            scala-binary: '2.13'
            scala: '2.13.5'
          # Spark 3.5
          - spark: '3.5'
            scala-binary: '2.12'
            scala: '2.12.18'
          - spark: '3.5'
            scala-binary: '2.13'
            scala: '2.13.5'
    steps:
    - uses: actions/checkout@v4
    - name: Setup JDK ${{ matrix.java }}
      uses: actions/setup-java@v4
      with:
        distribution: zulu
        java-version: ${{ matrix.java }}
        check-latest: false
    - name: Test with SBT
      run: |
        build/sbt -Dspark.shuffle.plugin.class=${{ matrix.shuffle-plugin-class }} -Pspark-${{ matrix.spark }} ++${{ matrix.scala }} "clean; celeborn-spark-group/test"
    - name: Upload test log
      if: failure()
      uses: actions/upload-artifact@v4
      with:
          name: spark-${{ matrix.spark }}-scala-${{ matrix.scala }}-unit-test-log
          path: |
              **/target/test-reports/**
              **/target/unit-tests.log

  flink:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        java:
          - 8
          - 11
        flink:
          - '1.14'
          - '1.15'
          - '1.16'
          - '1.17'
          - '1.18'
          - '1.19'
          - '1.20'
    steps:
      - uses: actions/checkout@v4
      - name: Setup JDK ${{ matrix.java }}
        uses: actions/setup-java@v4
        with:
          distribution: zulu
          java-version: ${{ matrix.java }}
          check-latest: false
      - name: Test with SBT
        run: |
          build/sbt -Pflink-${{ matrix.flink }} "clean; celeborn-flink-group/test"
      - name: Upload test log
        if: failure()
        uses: actions/upload-artifact@v4
        with:
            name: flink-${{ matrix.flink }}-unit-test-log
            path: |
                **/target/test-reports/**
                **/target/unit-tests.log

  mr:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        java:
          - 8
          - 11
    steps:
      - uses: actions/checkout@v4
      - name: Setup JDK ${{ matrix.java }}
        uses: actions/setup-java@v4
        with:
          distribution: zulu
          java-version: ${{ matrix.java }}
          check-latest: false
      - name: Test with SBT
        run: |
          build/sbt -Pmr "clean; celeborn-mr-group/test"
      - name: Upload test log
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: mr-unit-test-log
          path: |
            **/target/test/
            **/target/test-reports/**
            **/target/unit-tests.log