From 127d42a1a3d104410fea896c49cb2db26eba4995 Mon Sep 17 00:00:00 2001 From: Jose Carlos Garcia Date: Mon, 16 Nov 2015 09:00:44 +0100 Subject: [PATCH 1/2] csv output control versioned policies --- doc/src/site/sphinx/outputs.rst | 4 ++++ .../com/stratio/sparkta/plugin/output/csv/CsvOutput.scala | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/src/site/sphinx/outputs.rst b/doc/src/site/sphinx/outputs.rst index 1a3efa4853..d53861a9de 100644 --- a/doc/src/site/sphinx/outputs.rst +++ b/doc/src/site/sphinx/outputs.rst @@ -359,3 +359,7 @@ Csv Configuration +--------------------------+-----------------------------------------------+----------+-----------------------+ | dateGranularity | Specify the granularity from second to year | Yes | Day | +--------------------------+-----------------------------------------------+----------+-----------------------+ + + In Csv each cube is storage in one path, but when you modify the policy that involve this cube, Sparkta create a +new path with the next version. The name of all tables are the dimensions separated by '_' and the first version is +"v1" when modify the policy the path is "dimensions_v2". \ No newline at end of file diff --git a/plugins/output-csv/src/main/scala/com/stratio/sparkta/plugin/output/csv/CsvOutput.scala b/plugins/output-csv/src/main/scala/com/stratio/sparkta/plugin/output/csv/CsvOutput.scala index 1e74a35153..2828527bd0 100644 --- a/plugins/output-csv/src/main/scala/com/stratio/sparkta/plugin/output/csv/CsvOutput.scala +++ b/plugins/output-csv/src/main/scala/com/stratio/sparkta/plugin/output/csv/CsvOutput.scala @@ -55,8 +55,11 @@ class CsvOutput(keyName: String, require(path.isDefined, "Destination path is required. You have to set 'path' on properties") val pathParsed = if (path.get.endsWith("/")) path.get else path.get + "/" val subPath = DateOperations.subPath(dateGranularityFile, datePattern) - - saveAction(s"$pathParsed$tableName$subPath.csv", dataFrame) + val tableNameVersioned = version match { + case Some(v) => s"$tableName${Output.Separator}v$v" + case None => tableName + } + saveAction(s"$pathParsed$tableNameVersioned$subPath.csv", dataFrame) } protected[csv] def saveAction(path: String, dataFrame: DataFrame): Unit = { From d277531525922c8b88a109d2a0f993d8eccaf9c1 Mon Sep 17 00:00:00 2001 From: Jose Carlos Garcia Date: Tue, 17 Nov 2015 08:52:26 +0100 Subject: [PATCH 2/2] rebase --- doc/src/site/sphinx/outputs.rst | 8 +++++--- .../com/stratio/sparkta/plugin/output/csv/CsvOutput.scala | 6 +----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/doc/src/site/sphinx/outputs.rst b/doc/src/site/sphinx/outputs.rst index d53861a9de..59e6d54118 100644 --- a/doc/src/site/sphinx/outputs.rst +++ b/doc/src/site/sphinx/outputs.rst @@ -360,6 +360,8 @@ Csv Configuration | dateGranularity | Specify the granularity from second to year | Yes | Day | +--------------------------+-----------------------------------------------+----------+-----------------------+ - In Csv each cube is storage in one path, but when you modify the policy that involve this cube, Sparkta create a -new path with the next version. The name of all tables are the dimensions separated by '_' and the first version is -"v1" when modify the policy the path is "dimensions_v2". \ No newline at end of file +When you are using CSV as output, it will save the result in a path concatenating a base path, the dimension names + of the cube and a version starting in one. +Example: if the basePath is /user/stratio, the cube has 2 dimensions (name, price) and it is the first time that + you save the policy then the final path will be /user/stratio/name_price_v1. +If you modify some dimension of this cube then the version will be increased in one: /user/stratio/name_price_v2 \ No newline at end of file diff --git a/plugins/output-csv/src/main/scala/com/stratio/sparkta/plugin/output/csv/CsvOutput.scala b/plugins/output-csv/src/main/scala/com/stratio/sparkta/plugin/output/csv/CsvOutput.scala index 2828527bd0..bcb8dbcfe4 100644 --- a/plugins/output-csv/src/main/scala/com/stratio/sparkta/plugin/output/csv/CsvOutput.scala +++ b/plugins/output-csv/src/main/scala/com/stratio/sparkta/plugin/output/csv/CsvOutput.scala @@ -55,11 +55,7 @@ class CsvOutput(keyName: String, require(path.isDefined, "Destination path is required. You have to set 'path' on properties") val pathParsed = if (path.get.endsWith("/")) path.get else path.get + "/" val subPath = DateOperations.subPath(dateGranularityFile, datePattern) - val tableNameVersioned = version match { - case Some(v) => s"$tableName${Output.Separator}v$v" - case None => tableName - } - saveAction(s"$pathParsed$tableNameVersioned$subPath.csv", dataFrame) + saveAction(s"$pathParsed${versionedTableName(tableName)}$subPath.csv", dataFrame) } protected[csv] def saveAction(path: String, dataFrame: DataFrame): Unit = {