diff --git a/build.sbt b/build.sbt index 6b10803..41ddb0e 100644 --- a/build.sbt +++ b/build.sbt @@ -4,20 +4,27 @@ name := "obo-parser" scalaVersion := "2.12.12" organization := "bio.ferlab" javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint") -val spark_version = "3.4.2" +val sparkVersion = "3.5.1" val deltaCoreVersion = "2.4.0" +val datalakeSpark3Version = "14.0.0" /* Runtime */ -libraryDependencies += "org.apache.spark" %% "spark-sql" % spark_version % Provided -libraryDependencies += "org.apache.spark" %% "spark-hive" % spark_version % Provided -libraryDependencies += "org.apache.hadoop" % "hadoop-aws" % "3.3.6" % Provided -libraryDependencies += "org.apache.hadoop" % "hadoop-common" % "3.3.6" % Provided -libraryDependencies += "io.delta" %% "delta-core" % deltaCoreVersion % Provided -libraryDependencies += "io.delta" %% "delta-spark" % "3.1.0" % Provided +//libraryDependencies += "org.apache.spark" %% "spark-sql" % spark_version //% Provided +//libraryDependencies += "org.apache.hadoop" % "hadoop-aws" % "3.3.6" //% Provided +//libraryDependencies += "org.apache.hadoop" % "hadoop-common" % "3.3.6" //% Provided +//libraryDependencies += "io.delta" %% "delta-spark" % "3.1.0" //% Provided + + +libraryDependencies+= "bio.ferlab" %% "datalake-spark3" % "14.0.4" +libraryDependencies+= "org.apache.spark" %% "spark-sql" % sparkVersion % Provided +libraryDependencies+= "org.apache.spark" %% "spark-hive" % sparkVersion % Provided +libraryDependencies+= "org.apache.hadoop" % "hadoop-aws" % "3.3.4" % Provided +libraryDependencies+= "io.delta" %% "delta-spark" % "3.1.0" % Provided + libraryDependencies += "com.github.pureconfig" %% "pureconfig" % "0.15.0" libraryDependencies += "org.apache.poi" % "poi-ooxml" % "5.0.0" /* Test */ libraryDependencies += "org.scalatest" %% "scalatest" % "3.1.0" % "test" -libraryDependencies += "org.apache.spark" %% "spark-hive" % spark_version % "test" +libraryDependencies += "org.apache.spark" %% "spark-hive" % sparkVersion % "test" assembly / test := {} assembly / assemblyShadeRules := Seq( diff --git a/src/main/scala/bio/ferlab/HPOMain.scala b/src/main/scala/bio/ferlab/HPOMain.scala index d2b8ed3..872e8a1 100644 --- a/src/main/scala/bio/ferlab/HPOMain.scala +++ b/src/main/scala/bio/ferlab/HPOMain.scala @@ -3,10 +3,7 @@ package bio.ferlab import bio.ferlab.config.Config import bio.ferlab.ontology.{ICDTerm, OntologyTerm} import bio.ferlab.transform.{DownloadTransformer, WriteJson, WriteParquet} -import org.apache.spark.SparkConf -import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy.CORRECTED import org.apache.spark.sql.{SaveMode, SparkSession} -import pureconfig.ConfigReader.Result import pureconfig._ import pureconfig.generic.auto._