diff --git a/.travis.yml b/.travis.yml index 4056ad78..c3f80137 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,149 +4,7 @@ language: java jdk: openjdk8 jobs: include: - #Other modules can refer to oap-cache-oap to build independent travis-ci job, - #oap-cache-oap is a CI building demo of the corresponding module oap-cache/oap. - - name: oap-cache-oap - before_install: - - sudo apt-get install libpthread-stubs0-dev - - sudo apt-get install libnuma-dev - - sudo apt-get install cmake - - sudo apt-get install asciidoctor libkmod-dev libudev-dev uuid uuid-dev libjson-c-dev systemd libkeyutils-dev - - sudo apt-get install pandoc libgtest-dev pkg-config libtbb-dev rapidjson-dev valgrind - install: - - # Download spark 3.0.0 - - "[ -f spark ] || mkdir spark && cd spark && wget https://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz && cd .." - - "tar -xf ./spark/spark-3.0.0-bin-hadoop2.7.tgz" - - "export SPARK_HOME=`pwd`/spark-3.0.0-bin-hadoop2.7" - before_script: - - cd ${TRAVIS_BUILD_DIR}/dev - - ./install_vmemcache.sh - - ./install_memkind.sh - - ./install_pmdk.sh - - ./install_pmemkv.sh - - ../oap-common/src/native/memkind/compile.sh - - ../oap-common/src/native/libpmemblk/compile.sh - - sudo cp ../oap-common/src/resources/linux/64/lib/libpmplatform.so /usr/lib/ - - sudo cp ../oap-common/src/resources/linux/64/lib/libpmblkplatform.so /usr/lib/ - - sudo ldconfig /usr/lib - script: - - cd ${TRAVIS_BUILD_DIR} - - mvn clean test -pl oap-common -q -Ppersistent-memory -Pvmemcache -Plibpmemblk -am - - mvn clean test -pl oap-cache/oap -q -Ppersistent-memory -Pvmemcache -am - - - name: oap-native-sql - dist: bionic - jdk: - - openjdk8 - before_install: - - echo ${TRAVIS_COMMIT_MESSAGE} - #- if [[ ${TRAVIS_COMMIT_MESSAGE} != \[oap-native-sql\]* ]]; then travis_terminate 0 ; fi ; - - sudo apt-get install cmake - - sudo apt-get install libboost-all-dev - - export | grep JAVA_HOME - install: - - # Download spark 3.0 - - "[ -f spark ] || mkdir spark && cd spark && wget http://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz && cd .." - - "tar -xf ./spark/spark-3.0.0-bin-hadoop2.7.tgz" - - "export SPARK_HOME=`pwd`/spark-3.0.0-bin-hadoop2.7" - before_script: - - cd /tmp - - git clone https://github.com/intel-bigdata/arrow.git - - cd arrow && git checkout oap-master && cd cpp - - sed -i "s/\${Python3_EXECUTABLE}/\/opt\/pyenv\/shims\/python3/g" CMakeLists.txt - - mkdir build && cd build - - cmake .. -DARROW_JNI=ON -DARROW_GANDIVA_JAVA=ON -DARROW_GANDIVA=ON -DARROW_PARQUET=ON -DARROW_HDFS=ON -DARROW_FILESYSTEM=ON -DARROW_WITH_SNAPPY=ON -DARROW_JSON=ON -DARROW_DATASET=ON -DARROW_WITH_LZ4=ON && make - - sudo make install - - cd ../../java - - mvn clean install -q -P arrow-jni -am -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release/ -DskipTests -Dcheckstyle.skip - script: - - cd ${TRAVIS_BUILD_DIR}/ - - mvn clean -q package -DskipTests -pl oap-native-sql/core -am #skip core tests - - # run native sql unit tests - - count=0; while [ $count -le 3 ]; do echo "Elapsed 3 minutes"; sleep 180; let count++; done & # print log each 3 minutes for 3 times to avoid no-log issue - - mvn test -DmembersOnlySuites=org.apache.spark.sql.travis -pl oap-native-sql/core -am -DfailIfNoTests=false -Dexec.skip=true &> log-file.log # skip cpp build - - echo '#!/bin/bash' > grep.sh - - echo "module_tested=0; module_should_test=4; tests_total=0; while read -r line; do num=\$(echo \"\$line\" | grep -o -E '[0-9]+'); tests_total=\$((tests_total+num)); done <<<\"\$(grep \"Total number of tests run:\" log-file.log)\"; succeed_total=0; while read -r line; do [[ \$line =~ [^0-9]*([0-9]+)\, ]]; num=\${BASH_REMATCH[1]}; succeed_total=\$((succeed_total+num)); let module_tested++; done <<<\"\$(grep \"succeeded\" log-file.log)\"; if test \$tests_total -eq \$succeed_total -a \$module_tested -eq \$module_should_test; then echo \"All unit tests succeed\"; else echo \"Unit tests failed\"; exit 1; fi" >> grep.sh - - bash grep.sh - - # run data source unit tests - - mvn -q test -pl oap-data-source/arrow -am - - name: oap-shuffle-remote-shuffle - install: - - #empty install step - script: - - cd ${TRAVIS_BUILD_DIR}/oap-shuffle/remote-shuffle/ - - mvn -q test - - - name: oap-Rpmem-shuffle - dist: bionic - jdk: - - openjdk8 - before_install: - - echo ${TRAVIS_COMMIT_MESSAGE} - #- if [[ ${TRAVIS_COMMIT_MESSAGE} != \[oap-native-sql\]* ]]; then travis_terminate 0 ; fi ; - - sudo apt-get install -y openjdk-8-jdk git maven g++-7 cmake build-essential libboost-dev libboost-system-dev autogen autoconf libtool pandoc asciidoctor libkmod-dev libdaxctl-dev pkg-config libkmod2 kmod libuuid1 libudev1 libudev-dev libjson-c-dev libjemalloc-dev - - export | grep JAVA_HOME - - "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" - - export |grep JAVA_HOME - install: - - # Download spark 3.0 - - "[ -f spark ] || mkdir spark && cd spark && wget http://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz && cd .." - - "tar -xf ./spark/spark-3.0.0-bin-hadoop2.7.tgz" - - "export SPARK_HOME=`pwd`/spark-3.0.0-bin-hadoop2.7" - before_script: - - cd /tmp - #libfabric - - git clone https://github.com/ofiwg/libfabric.git && cd libfabric && git checkout -b v1.8.0 tags/v1.8.0 && ./autogen.sh && ./configure --prefix=/usr/local --enable-sockets - - make -j && sudo make install - #HPNL - - cd /tmp - - git clone https://github.com/Intel-bigdata/HPNL.git - - cd HPNL && git submodule update --init --recursive && mkdir build && cd build - - cmake -DWITH_VERBS=ON -DWITH_JAVA=ON .. - - make -j && sudo make install - - cd ../java/hpnl - - sudo mvn install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - #PMDK - - cd /tmp - - git clone https://github.com/pmem/pmdk.git && cd pmdk && git checkout tags/1.8 - # PMDK uses pkg_config to find lbndctl, disable it for now - - make NDCTL_ENABLE=n - - sudo make NDCTL_ENABLE=n install - #RPMem - - cd /tmp - - git clone https://github.com/efficient/libcuckoo && cd libcuckoo && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_EXAMPLES=1 -DBUILD_TESTS=1 .. - - make all && sudo make install - script: - - cd ${TRAVIS_BUILD_DIR}/oap-shuffle/RPMem-shuffle - # skip tests, do tests locally - - sudo mvn install -DskipTests -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - - - name: oap-mllib - dist: bionic - jdk: - - openjdk8 - cache: - directories: - - /opt/intel/inteloneapi - - /opt/intel/oneapi - - $HOME/downloads - install: - - # Install Spark 3.0 - - "[ -d $HOME/downloads ] || mkdir $HOME/downloads" - - "cd $HOME/downloads && wget http://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz" - - "cd $HOME && tar -xf $HOME/downloads/spark-3.0.0-bin-hadoop2.7.tgz" - - "export SPARK_HOME=$HOME/spark-3.0.0-bin-hadoop2.7" - - # Install Dependencies - - ${TRAVIS_BUILD_DIR}/oap-mllib/dev/install-build-deps-ubuntu.sh - script: - - cd ${TRAVIS_BUILD_DIR}/oap-mllib/mllib-dal - - export ONEAPI_ROOT=/opt/intel/oneapi - - source /opt/intel/oneapi/dal/latest/env/vars.sh - - source /opt/intel/oneapi/tbb/latest/env/vars.sh - - source /tmp/oneCCL/build/_install/env/setvars.sh - - ./test.sh - - - name: oap-spark + - name: pmem-common before_install: - sudo apt-get install libpthread-stubs0-dev - sudo apt-get install libnuma-dev @@ -156,16 +14,19 @@ jobs: install: - #empty install step before_script: - - cd ${TRAVIS_BUILD_DIR}/dev + - git clone https://github.com/oap-project/oap-tools.git + - cd oap-tools/dev - ./install_memkind.sh - ./install_pmdk.sh - ./install_pmemkv.sh - - ../oap-common/src/native/memkind/compile.sh - - ../oap-common/src/native/libpmemblk/compile.sh - - sudo cp ../oap-common/src/resources/linux/64/lib/libpmplatform.so /usr/lib/ - - sudo cp ../oap-common/src/resources/linux/64/lib/libpmblkplatform.so /usr/lib/ + - git clone https://github.com/oap-project/pmem-common.git + - cd pmem-common + - . src/native/memkind/compile.sh + - . src/native/libpmemblk/compile.sh + - sudo cp src/resources/linux/64/lib/libpmplatform.so /usr/lib/ + - sudo cp src/resources/linux/64/lib/libpmblkplatform.so /usr/lib/ - sudo ldconfig /usr/lib + - mvn install DskipTest -Ppersistent-memory -Plibpmemblk package script: - cd ${TRAVIS_BUILD_DIR} - - mvn clean test -pl oap-common -q -Ppersistent-memory -Plibpmemblk -am - - mvn clean package -pl oap-spark -q -Ppersistent-memory -am + - mvn clean test -q -Ppersistent-memory \ No newline at end of file diff --git a/pom.xml b/pom.xml index c4bb75fc..7632c489 100644 --- a/pom.xml +++ b/pom.xml @@ -4,25 +4,22 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - - com.intel.oap - oap-parent - 0.9.0 - ../pom.xml - - - oap-spark + com.intel.oap + pmem-spill + 1.1.0 + OAP Project PMem Spill jar - OAP Project Spark Source Code Change - 2.12.10 - 2.12 - 3.0.0 - 1.2 - 2.6 - 9.4.18.v20190429 - + 1.8 + 4.12 + 2.12.10 + 2.12 + 3.0.0 + 1.2 + 2.6 + 9.4.18.v20190429 + @@ -32,7 +29,7 @@ com.intel.oap - oap-common + pmem-common ${project.version}