master.yml

name: master

on:
  push:
    branches:
    - master
  pull_request:
    branches:
    - master

jobs:
  build:

    runs-on: ubuntu-latest
    strategy:
      matrix:
        java: [ '1.8', '11' ]
        hadoop: [ 'hadoop-2.7', 'hadoop-3.2' ]
        hive: [ 'hive-1.2', 'hive-2.3' ]
        exclude:
        - java: '11'
          hive: 'hive-1.2'
        - hadoop: 'hadoop-3.2'
          hive: 'hive-1.2'
    name: Build Spark - JDK${{ matrix.java }}/${{ matrix.hadoop }}/${{ matrix.hive }}

    steps:
    - uses: actions/checkout@master
    # We split caches because GitHub Action Cache has a 400MB-size limit.
    - uses: actions/cache@v1
      with:
        path: build
        key: build-${{ hashFiles('**/pom.xml') }}
        restore-keys: |
          build-
    - uses: actions/cache@v1
      with:
        path: ~/.m2/repository/com
        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-com-${{ hashFiles('**/pom.xml') }}
        restore-keys: |
          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-com-
    - uses: actions/cache@v1
      with:
        path: ~/.m2/repository/org
        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-org-${{ hashFiles('**/pom.xml') }}
        restore-keys: |
          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-org-
    - uses: actions/cache@v1
      with:
        path: ~/.m2/repository/net
        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-net-${{ hashFiles('**/pom.xml') }}
        restore-keys: |
          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-net-
    - uses: actions/cache@v1
      with:
        path: ~/.m2/repository/io
        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-io-${{ hashFiles('**/pom.xml') }}
        restore-keys: |
          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-io-
    - name: Set up JDK ${{ matrix.java }}
      uses: actions/setup-java@v1
      with:
        java-version: ${{ matrix.java }}
    - name: Build with Maven
      run: |
        export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
        export MAVEN_CLI_OPTS="--no-transfer-progress"
        mkdir -p ~/.m2
        ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -P${{ matrix.hive }} -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install
        rm -rf ~/.m2/repository/org/apache/spark


  lint:
    runs-on: ubuntu-latest
    name: Linters (Java/Scala/Python), licenses, dependencies
    steps:
    - uses: actions/checkout@master
    - uses: actions/setup-java@v1
      with:
        java-version: '11'
    - uses: actions/setup-python@v1
      with:
        python-version: '3.x'
        architecture: 'x64'
    - name: Scala
      run: ./dev/lint-scala
    - name: Java
      run: ./dev/lint-java
    - name: Python
      run: |
        pip install flake8 sphinx numpy
        ./dev/lint-python
    - name: License
      run: ./dev/check-license
    - name: Dependencies
      run: ./dev/test-dependencies.sh

  lintr:
    runs-on: ubuntu-latest
    name: Linter (R)
    steps:
    - uses: actions/checkout@master
    - uses: actions/setup-java@v1
      with:
        java-version: '11'
    - uses: r-lib/actions/setup-r@v1
      with:
        r-version: '3.6.2'
    - name: Install lib
      run: |
        sudo apt-get install -y libcurl4-openssl-dev
    - name: install R packages
      run: |
        sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
        sudo Rscript -e "devtools::install_github('jimhester/lintr@v2.0.0')"
    - name: package and install SparkR
      run: ./R/install-dev.sh
    - name: lint-r
      run: ./dev/lint-r

  docs:
    runs-on: ubuntu-latest
    name: Generate documents
    steps:
    - uses: actions/checkout@master
    - uses: actions/cache@v1
      with:
        path: ~/.m2/repository
        key: docs-maven-repo-${{ hashFiles('**/pom.xml') }}
        restore-keys: |
          docs-maven-repo-
    - uses: actions/setup-java@v1
      with:
        java-version: '1.8'
    - uses: actions/setup-python@v1
      with:
        python-version: '3.x'
        architecture: 'x64'
    - uses: actions/setup-ruby@v1
      with:
        ruby-version: '2.7'
    - uses: r-lib/actions/setup-r@v1
      with:
        r-version: '3.6.2'
    - name: Install lib and pandoc
      run: |
        sudo apt-get install -y libcurl4-openssl-dev pandoc
    - name: Install packages
      run: |
        pip install sphinx mkdocs numpy
        gem install jekyll jekyll-redirect-from rouge
        sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
    - name: Run jekyll build
      run: |
        cd docs
        jekyll build