Merge branch 'release/0.1.1'

SANSA-Stack · Jun 7, 2019 · 36ac8d9 · 36ac8d9
2 parents bae0a94 + 8681aa4
commit 36ac8d9
Show file tree

Hide file tree

Showing 14 changed files with 536 additions and 417 deletions.
diff --git a/README.md b/README.md
@@ -90,8 +90,16 @@ Var         := "?string"
 Modifiers   := (LIMITk)? (ORDER BY(ASC|DESC)? Var)? (GROUP BYVar+)?
 ```
 
+### File Storage format
+The previous three files can be stored either locally, in HDFS on in an AWS S3 bucket. For the latter, make sure to have your credentials ([see](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/setup-credentials.html)) stored in ~/.aws/credentials (C:\Users\USERNAME\.aws\credentials on Windows), in the following form:
+```
+[default]
+aws_access_key_id=...
+aws_secret_access_key=...
+```
+
 ## Usage
-The usage of the Semantic Data Lake is documented under the respective SANSA-Query [datalake component](https://github.com/SANSA-Stack/SANSA-Query/tree/feature/datalake/sansa-query-spark/src/main/scala/net/sansa_stack/query/spark/datalake).
+The usage of the Semantic Data Lake is documented under the respective SANSA-Query [datalake component](https://github.com/SANSA-Stack/SANSA-Query/tree/develop/sansa-query-spark/src/main/scala/net/sansa_stack/query/spark/datalake).
 
 ## How to Contribute
 We always welcome new contributors to the project! Please see [our contribution guide](http://sansa-stack.net/contributing-to-sansa/) for more details on how to get started contributing to SANSA.
diff --git a/build.sbt b/build.sbt
diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
 
 	<groupId>net.sansa-stack</groupId>
 	<artifactId>sansa-datalake-parent_2.11</artifactId>
-	<version>0.1.0</version>
+	<version>0.1.1</version>
 	<packaging>pom</packaging>
 
 	<name>SANSA Stack - DataLake Layer - Parent</name>
@@ -27,7 +27,7 @@
 		<scala.binary.version>2.11</scala.binary.version>
 		<scala.version.suffix>_${scala.binary.version}</scala.version.suffix>
 
-		<spark.version>2.4.0</spark.version>
+		<spark.version>2.4.3</spark.version>
 		<jena.version>3.9.0</jena.version>
 
 		<scalastyle.config.path>${project.basedir}/scalastyle-config.xml</scalastyle.config.path>
@@ -178,12 +178,6 @@
 				<version>${jena.version}</version>
 			</dependency>
 
-			<dependency>
-				<groupId>com.typesafe.scala-logging</groupId>
-				<artifactId>scala-logging_${scala.binary.version}</artifactId>
-				<version>3.5.0</version>
-			</dependency>
-
 			<dependency>
 				<groupId>de.javakaffee</groupId>
 				<artifactId>kryo-serializers</artifactId>
@@ -510,7 +504,7 @@
 					</configuration>
 				</plugin>
 
-				<!--This plugin's configuration is used to store Eclipse m2e settings 
+				<!--This plugin's configuration is used to store Eclipse m2e settings
 					only. It has no influence on the Maven build itself. -->
 				<plugin>
 					<groupId>org.eclipse.m2e</groupId>

diff --git a/sansa-datalake-spark/pom.xml b/sansa-datalake-spark/pom.xml
@@ -7,17 +7,15 @@
 	<parent>
 		<groupId>net.sansa-stack</groupId>
 		<artifactId>sansa-datalake-parent_2.11</artifactId>
-		<version>0.1.0</version>
+		<version>0.1.1</version>
 	</parent>
 
 	<dependencies>
-
 		<dependency>
 			<groupId>org.apache.spark</groupId>
 			<artifactId>spark-core_${scala.binary.version}</artifactId>
 		</dependency>
 
-
 		<dependency>
 			<groupId>org.apache.spark</groupId>
 			<artifactId>spark-sql_${scala.binary.version}</artifactId>
@@ -44,7 +42,6 @@
 			<scope>provided</scope>
 		</dependency>
 
-
 		<!-- Test -->
 		<dependency>
 			<groupId>junit</groupId>
@@ -87,12 +84,6 @@
 			<artifactId>jena-arq</artifactId>
 		</dependency>
 
-		<dependency>
-			<groupId>com.typesafe.scala-logging</groupId>
-			<artifactId>scala-logging_${scala.binary.version}</artifactId>
-		</dependency>
-
-
 		<dependency>
 			<groupId>com.typesafe</groupId>
 			<artifactId>config</artifactId>
@@ -140,6 +131,25 @@
 			<artifactId>mysql-connector-java</artifactId>
 		</dependency>
 
+		<dependency>
+			<groupId>com.amazonaws</groupId>
+			<artifactId>aws-java-sdk</artifactId>
+			<version>1.9.6</version>
+		</dependency>
+
+		<!-- Loggings -->
+		<dependency>
+          <groupId>com.typesafe.scala-logging</groupId>
+          <artifactId>scala-logging_${scala.binary.version}</artifactId>
+          <version>3.9.0</version>
+	    </dependency>
+
+	    <dependency>
+		    <groupId>ch.qos.logback</groupId>
+		    <artifactId>logback-classic</artifactId>
+		   <version>1.2.3</version>
+		    <scope>test</scope>
+	    </dependency>
 	</dependencies>
 
 	<build>

diff --git a/sansa-datalake-spark/src/main/resources/log4j.properties b/sansa-datalake-spark/src/main/resources/log4j.properties
@@ -0,0 +1,12 @@
+log4j.rootLogger=OFF, console
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%-4r [%t] %-5p [%c] %x %m%n  
+
+log4j.logger.org.apache.spark = ERROR
+log4j.logger.org.spark_project.jetty.server = ERROR
+log4j.logger.org.apache.parquet = ERROR
diff --git a/sansa-datalake-spark/src/main/scala/net/sansa_stack/datalake/spark/Main.scala b/sansa-datalake-spark/src/main/scala/net/sansa_stack/datalake/spark/Main.scala
@@ -17,9 +17,8 @@ object Main extends App {
     val hadoopConfig = spark.conf
 
     val executor : SparkExecutor = new SparkExecutor(spark, mappingsFile)
-    // val finalResults = executor.getType()
 
     val run = new Run[DataFrame](executor)
     run.application(queryFile, mappingsFile, configFile)
 
-}
+}