ci.yml

name: CI
on:
  pull_request:
  push:
jobs:
  test:
    strategy:
      fail-fast: false
      matrix:
        include:
          - java: 17
          - java: 11
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v2
    - name: Sync the current branch with the latest
      if: github.repository != 'high-performance-spark/high-performance-spark-examples'
      id: sync-branch
      run: |
        git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF##*/}
        git -c user.name='Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD || echo "no merge needed."
        git -c user.name='Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" || echo "no merge needed."
    - name: Setup JDK
      uses: actions/setup-java@v3
      with:
        distribution: temurin
        java-version: ${{ matrix.java }}
        cache: sbt
    - name: Scala Build and Test
      run: sbt clean package +test
  python-test:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install tox tox-gh-actions
    - name: Run tox
      run: |
        cd python; tox
  run-sql-examples:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v2
    - name: Cache Spark and friends
      uses: actions/cache@v3
      with:
        path: |
          spark*.tgz
          iceberg*.jar
        key: spark-artifacts
    - name: Cache Data
      uses: actions/cache@v3
      with:
        path: |
          data/fetched/*
        key: data-fetched
    - name: Run sql examples
      run:
        ./run_sql_examples.sh
  run-gluten-sql-examples:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v2
    - name: Cache Spark and friends
      uses: actions/cache@v3
      with:
        path: |
          spark*.tgz
          iceberg*.jar
        key: spark-artifacts
    - name: Cache Data
      uses: actions/cache@v3
      with:
        path: |
          data/fetched/*
        key: data-fetched
    - name: Run gluten
      run:
        cd accelerators; ./gluten_spark_34_ex.sh || echo "Failed (as expected cause it's broken :()"
  run-target-examples:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v2
    - name: Cache Spark and friends
      uses: actions/cache@v3
      with:
        path: |
          spark*.tgz
          iceberg*.jar
        key: spark-artifacts
    - name: Cache Accel
      uses: actions/cache@v3
      with:
        path: |
          accelerators/*.jar
        key: accelerators-artifacts
    - name: Cache Data
      uses: actions/cache@v3
      with:
        path: |
          data/fetched/*
        key: data-fetched
    - name: Run the target validator example
      run:
        cd target-validator; ./runme.sh
  run-pyspark-examples:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v2
    - name: Cache Spark and friends
      uses: actions/cache@v3
      with:
        path: |
          spark*.tgz
          iceberg*.jar
        key: spark-artifacts
    - name: Cache Data
      uses: actions/cache@v3
      with:
        path: |
          data/fetched/*
        key: data-fetched
    - name: Run PySpark examples
      run:
        ./run_pyspark_examples.sh
  style:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v2
    - name: Shellcheck
      run: |
        sudo apt-get install -y shellcheck
        shellcheck $(find -name "*.sh")
    - name: Setup JDK
      uses: actions/setup-java@v3
      with:
        distribution: temurin
        java-version: 17
        cache: sbt
    - name: scala
      run:
        sbt scalastyle