Merge pull request #3 from HongW2019/master

[pmem-spill-1]Make pmem-spill repo compile successfully
oap-project · Dec 29, 2020 · c668455 · c668455
2 parents c1def44 + 4b88f67
commit c668455
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 167 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -4,149 +4,7 @@ language: java
 jdk: openjdk8
 jobs:
   include:
-    #Other modules can refer to oap-cache-oap to build independent travis-ci job,
-    #oap-cache-oap is a CI building demo of the corresponding module oap-cache/oap.
-    - name: oap-cache-oap
-      before_install:
-      - sudo apt-get install libpthread-stubs0-dev
-      - sudo apt-get install libnuma-dev
-      - sudo apt-get install cmake
-      - sudo apt-get install asciidoctor libkmod-dev libudev-dev uuid uuid-dev libjson-c-dev systemd libkeyutils-dev
-      - sudo apt-get install pandoc libgtest-dev pkg-config libtbb-dev rapidjson-dev valgrind
-      install:
-      - # Download spark 3.0.0
-      - "[ -f spark ] || mkdir spark && cd spark && wget https://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz && cd .."
-      - "tar -xf ./spark/spark-3.0.0-bin-hadoop2.7.tgz"
-      - "export SPARK_HOME=`pwd`/spark-3.0.0-bin-hadoop2.7"
-      before_script:
-      - cd ${TRAVIS_BUILD_DIR}/dev
-      - ./install_vmemcache.sh
-      - ./install_memkind.sh
-      - ./install_pmdk.sh
-      - ./install_pmemkv.sh
-      - ../oap-common/src/native/memkind/compile.sh
-      - ../oap-common/src/native/libpmemblk/compile.sh
-      - sudo cp ../oap-common/src/resources/linux/64/lib/libpmplatform.so /usr/lib/
-      - sudo cp ../oap-common/src/resources/linux/64/lib/libpmblkplatform.so /usr/lib/
-      - sudo ldconfig /usr/lib
-      script:
-      - cd ${TRAVIS_BUILD_DIR}
-      - mvn clean test -pl oap-common -q -Ppersistent-memory -Pvmemcache -Plibpmemblk -am
-      - mvn clean test -pl oap-cache/oap -q -Ppersistent-memory  -Pvmemcache  -am
-
-    - name: oap-native-sql
-      dist: bionic
-      jdk:
-        - openjdk8
-      before_install:
-      - echo ${TRAVIS_COMMIT_MESSAGE}
-      #- if [[ ${TRAVIS_COMMIT_MESSAGE} != \[oap-native-sql\]* ]]; then travis_terminate 0 ; fi ;
-      - sudo apt-get install cmake
-      - sudo apt-get install libboost-all-dev
-      - export | grep JAVA_HOME
-      install:
-      - # Download spark 3.0
-      - "[ -f spark ] || mkdir spark && cd spark && wget http://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz && cd .."
-      - "tar -xf ./spark/spark-3.0.0-bin-hadoop2.7.tgz"
-      - "export SPARK_HOME=`pwd`/spark-3.0.0-bin-hadoop2.7"
-      before_script:
-      - cd /tmp
-      - git clone https://github.com/intel-bigdata/arrow.git
-      - cd arrow && git checkout oap-master && cd cpp
-      - sed -i "s/\${Python3_EXECUTABLE}/\/opt\/pyenv\/shims\/python3/g" CMakeLists.txt
-      - mkdir build && cd build 
-      - cmake .. -DARROW_JNI=ON -DARROW_GANDIVA_JAVA=ON -DARROW_GANDIVA=ON -DARROW_PARQUET=ON -DARROW_HDFS=ON -DARROW_FILESYSTEM=ON -DARROW_WITH_SNAPPY=ON -DARROW_JSON=ON -DARROW_DATASET=ON -DARROW_WITH_LZ4=ON && make 
-      - sudo make install
-      - cd ../../java
-      - mvn clean install -q -P arrow-jni -am -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release/ -DskipTests -Dcheckstyle.skip
-      script:
-      - cd ${TRAVIS_BUILD_DIR}/
-      - mvn clean -q package -DskipTests -pl oap-native-sql/core -am  #skip core tests
-      - # run native sql unit tests
-      - count=0; while [ $count -le 3 ]; do echo "Elapsed 3 minutes"; sleep 180; let count++; done & # print log each 3 minutes for 3 times to avoid no-log issue
-      - mvn test -DmembersOnlySuites=org.apache.spark.sql.travis -pl oap-native-sql/core -am -DfailIfNoTests=false -Dexec.skip=true &> log-file.log # skip cpp build
-      - echo '#!/bin/bash' > grep.sh
-      - echo "module_tested=0; module_should_test=4; tests_total=0; while read -r line; do num=\$(echo \"\$line\" | grep -o -E '[0-9]+'); tests_total=\$((tests_total+num)); done <<<\"\$(grep \"Total number of tests run:\" log-file.log)\"; succeed_total=0; while read -r line; do [[ \$line =~ [^0-9]*([0-9]+)\, ]]; num=\${BASH_REMATCH[1]}; succeed_total=\$((succeed_total+num)); let module_tested++; done <<<\"\$(grep \"succeeded\" log-file.log)\"; if test \$tests_total -eq \$succeed_total -a \$module_tested -eq \$module_should_test; then echo \"All unit tests succeed\"; else echo \"Unit tests failed\"; exit 1; fi" >> grep.sh
-      - bash grep.sh
-      - # run data source unit tests
-      - mvn -q test -pl oap-data-source/arrow -am
-    - name: oap-shuffle-remote-shuffle
-      install:
-      - #empty install step
-      script:
-      - cd ${TRAVIS_BUILD_DIR}/oap-shuffle/remote-shuffle/
-      - mvn -q test
-
-    - name: oap-Rpmem-shuffle
-      dist: bionic
-      jdk:
-        - openjdk8
-      before_install:
-      - echo ${TRAVIS_COMMIT_MESSAGE}
-      #- if [[ ${TRAVIS_COMMIT_MESSAGE} != \[oap-native-sql\]* ]]; then travis_terminate 0 ; fi ;
-      - sudo apt-get install -y openjdk-8-jdk git maven g++-7 cmake build-essential libboost-dev libboost-system-dev autogen autoconf libtool pandoc asciidoctor libkmod-dev libdaxctl-dev pkg-config libkmod2 kmod libuuid1 libudev1 libudev-dev libjson-c-dev libjemalloc-dev
-      - export | grep JAVA_HOME
-      - "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" 
-      - export |grep JAVA_HOME
-      install:
-      - # Download spark 3.0
-      - "[ -f spark ] || mkdir spark && cd spark && wget http://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz && cd .."
-      - "tar -xf ./spark/spark-3.0.0-bin-hadoop2.7.tgz"
-      - "export SPARK_HOME=`pwd`/spark-3.0.0-bin-hadoop2.7"
-      before_script:
-      - cd /tmp
-      #libfabric   
-      - git clone https://github.com/ofiwg/libfabric.git &&  cd libfabric && git checkout -b v1.8.0 tags/v1.8.0 && ./autogen.sh && ./configure --prefix=/usr/local --enable-sockets 
-      - make -j && sudo make install
-      #HPNL
-      - cd /tmp
-      - git clone https://github.com/Intel-bigdata/HPNL.git
-      - cd HPNL && git submodule update --init --recursive && mkdir build &&  cd build
-      - cmake -DWITH_VERBS=ON -DWITH_JAVA=ON ..
-      - make -j && sudo make install
-      - cd ../java/hpnl
-      - sudo mvn install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
-      #PMDK
-      - cd /tmp
-      - git clone https://github.com/pmem/pmdk.git &&  cd pmdk && git checkout tags/1.8
-      # PMDK uses pkg_config to find lbndctl, disable it for now
-      - make NDCTL_ENABLE=n
-      - sudo make NDCTL_ENABLE=n install
-      #RPMem
-      - cd /tmp
-      - git clone https://github.com/efficient/libcuckoo && cd libcuckoo && mkdir build && cd build  && cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_EXAMPLES=1 -DBUILD_TESTS=1 ..
-      - make all && sudo make install
-      script:
-      - cd ${TRAVIS_BUILD_DIR}/oap-shuffle/RPMem-shuffle
-      # skip tests, do tests locally 
-      - sudo mvn install -DskipTests -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
-
-    - name: oap-mllib
-      dist: bionic
-      jdk:
-      - openjdk8
-      cache:
-        directories:
-        - /opt/intel/inteloneapi
-        - /opt/intel/oneapi
-        - $HOME/downloads
-      install:
-      - # Install Spark 3.0
-      - "[ -d $HOME/downloads ] || mkdir $HOME/downloads"
-      - "cd $HOME/downloads && wget http://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz"
-      - "cd $HOME && tar -xf $HOME/downloads/spark-3.0.0-bin-hadoop2.7.tgz"
-      - "export SPARK_HOME=$HOME/spark-3.0.0-bin-hadoop2.7"
-      - # Install Dependencies
-      - ${TRAVIS_BUILD_DIR}/oap-mllib/dev/install-build-deps-ubuntu.sh
-      script:
-      - cd ${TRAVIS_BUILD_DIR}/oap-mllib/mllib-dal      
-      - export ONEAPI_ROOT=/opt/intel/oneapi
-      - source /opt/intel/oneapi/dal/latest/env/vars.sh
-      - source /opt/intel/oneapi/tbb/latest/env/vars.sh
-      - source /tmp/oneCCL/build/_install/env/setvars.sh
-      - ./test.sh
-
-    - name: oap-spark
+    - name: pmem-common
       before_install:
         - sudo apt-get install libpthread-stubs0-dev
         - sudo apt-get install libnuma-dev
@@ -156,16 +14,19 @@ jobs:
       install:
         - #empty install step
       before_script:
-        - cd ${TRAVIS_BUILD_DIR}/dev
+        - git clone https://github.com/oap-project/oap-tools.git
+        - cd oap-tools/dev
         - ./install_memkind.sh
         - ./install_pmdk.sh
         - ./install_pmemkv.sh
-        - ../oap-common/src/native/memkind/compile.sh
-        - ../oap-common/src/native/libpmemblk/compile.sh
-        - sudo cp ../oap-common/src/resources/linux/64/lib/libpmplatform.so /usr/lib/
-        - sudo cp ../oap-common/src/resources/linux/64/lib/libpmblkplatform.so /usr/lib/
+        - git clone https://github.com/oap-project/pmem-common.git
+        - cd pmem-common
+        - . src/native/memkind/compile.sh
+        - . src/native/libpmemblk/compile.sh
+        - sudo cp src/resources/linux/64/lib/libpmplatform.so /usr/lib/
+        - sudo cp src/resources/linux/64/lib/libpmblkplatform.so /usr/lib/
         - sudo ldconfig /usr/lib
+        - mvn install DskipTest -Ppersistent-memory -Plibpmemblk package
       script:
         - cd ${TRAVIS_BUILD_DIR}
-        - mvn clean test -pl oap-common -q -Ppersistent-memory -Plibpmemblk -am
-        - mvn clean package -pl oap-spark -q -Ppersistent-memory -am
+        - mvn clean test -q -Ppersistent-memory
diff --git a/pom.xml b/pom.xml
@@ -4,25 +4,22 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <parent>
-        <groupId>com.intel.oap</groupId>
-        <artifactId>oap-parent</artifactId>
-        <version>0.9.0</version>
-        <relativePath>../pom.xml</relativePath>
-    </parent>
-
-    <artifactId>oap-spark</artifactId>
+    <groupId>com.intel.oap</groupId>
+    <artifactId>pmem-spill</artifactId>
+    <version>1.1.0</version>
+    <name>OAP Project PMem Spill</name>
     <packaging>jar</packaging>
-    <name>OAP Project Spark Source Code Change</name>
 
     <properties>
-    <scala.version>2.12.10</scala.version>
-    <scala.binary.version>2.12</scala.binary.version>
-    <spark.internal.version>3.0.0</spark.internal.version>
-    <maven-patch-plugin.version>1.2</maven-patch-plugin.version>
-    <maven-resource-plugin.version>2.6</maven-resource-plugin.version>
-    <jetty.version>9.4.18.v20190429</jetty.version>
-  </properties>
+        <java.version>1.8</java.version>
+        <junit.version>4.12</junit.version>
+        <scala.version>2.12.10</scala.version>
+        <scala.binary.version>2.12</scala.binary.version>
+        <spark.internal.version>3.0.0</spark.internal.version>
+        <maven-patch-plugin.version>1.2</maven-patch-plugin.version>
+        <maven-resource-plugin.version>2.6</maven-resource-plugin.version>
+        <jetty.version>9.4.18.v20190429</jetty.version>
+    </properties>
 
     <dependencies>
         <dependency>
@@ -32,7 +29,7 @@
         </dependency>
         <dependency>
             <groupId>com.intel.oap</groupId>
-            <artifactId>oap-common</artifactId>
+            <artifactId>pmem-common</artifactId>
             <version>${project.version}</version>
             <exclusions>
                 <exclusion>