diff --git a/client/src/main/scala/ch/wsl/box/client/services/REST.scala b/client/src/main/scala/ch/wsl/box/client/services/REST.scala index 8c43bda1b..0e92e0d26 100755 --- a/client/src/main/scala/ch/wsl/box/client/services/REST.scala +++ b/client/src/main/scala/ch/wsl/box/client/services/REST.scala @@ -2,7 +2,7 @@ package ch.wsl.box.client.services import ch.wsl.box.client.services.REST.get import ch.wsl.box.model.shared._ -import ch.wsl.box.shared.utils.CSV +import com.github.tototoshi.csv.CSV import io.circe.Json import org.scalajs.dom.File @@ -29,7 +29,7 @@ object REST { def list(kind:String, lang:String, entity:String, limit:Int): Future[Seq[Json]] = client.post[JSONQuery,Seq[Json]](s"/${EntityKind(kind).entityOrForm}/$lang/$entity/list",JSONQuery.empty.limit(limit)) def list(kind:String, lang:String, entity:String, query:JSONQuery): Future[Seq[Json]] = client.post[JSONQuery,Seq[Json]](s"/${EntityKind(kind).entityOrForm}/$lang/$entity/list",query) def csv(kind:String, lang:String, entity:String, q:JSONQuery): Future[Seq[Seq[String]]] = client.post[JSONQuery,String](s"/${EntityKind(kind).entityOrForm}/$lang/$entity/csv",q).map{ result => - CSV.split(result) + CSV.read(result) } def count(kind:String, lang:String, entity:String): Future[Int] = client.get[Int](s"/${EntityKind(kind).entityOrForm}/$lang/$entity/count") def keys(kind:String, lang:String, entity:String): Future[Seq[String]] = client.get[Seq[String]](s"/${EntityKind(kind).entityOrForm}/$lang/$entity/keys") @@ -59,7 +59,7 @@ object REST { //export def exportMetadata(name:String,lang:String) = client.get[JSONMetadata](s"/export/$name/metadata/$lang") - def export(name:String,params:Seq[Json],lang:String) = client.post[Seq[Json],String](s"/export/$name/$lang",params).map(CSV.split) + def export(name:String,params:Seq[Json],lang:String):Future[Seq[Seq[String]]] = client.post[Seq[Json],String](s"/export/$name/$lang",params).map(CSV.read) def exports(lang:String) = client.get[Seq[ExportDef]](s"/export/list/$lang") def writeAccess(table:String) = client.get[Boolean](s"/access/table/$table/write") diff --git a/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala b/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala index 5588c33fd..3e7b9c3c1 100755 --- a/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala +++ b/client/src/main/scala/ch/wsl/box/client/views/EntityTableView.scala @@ -317,12 +317,16 @@ case class EntityTablePresenter(model:ModelProperty[EntityTableModel], onSelect: } def downloadCSV() = { - val (kind, modelName) = model.get.metadata.flatMap(_.exportView).getOrElse("") match { - case "" => (EntityKind(model.subProp(_.kind).get).entityOrForm, model.subProp(_.name).get) - case view => ("entity", view) - } - val url = s"api/v1/$kind/${Session.lang()}/$modelName/csv?q=${query().asJson.toString()}".replaceAll("\n","") + val kind = EntityKind(model.subProp(_.kind).get).entityOrForm + val modelName = model.subProp(_.name).get + val exportFields = model.get.metadata.map(_.exportFields).getOrElse(Seq()) + val fields = model.get.metadata.map(_.fields).getOrElse(Seq()) + + val queryWithFK = encodeFk(fields,query()) + + + val url = s"api/v1/$kind/${Session.lang()}/$modelName/csv?fk=${ExportMode.RESOLVE_FK}&fields=${exportFields.mkString(",")}&q=${queryWithFK.asJson.toString()}".replaceAll("\n","") logger.info(s"downloading: $url") dom.window.open(url) } diff --git a/client/src/main/scala/ch/wsl/box/client/views/ExportView.scala b/client/src/main/scala/ch/wsl/box/client/views/ExportView.scala index b2664eea1..670c4957a 100755 --- a/client/src/main/scala/ch/wsl/box/client/views/ExportView.scala +++ b/client/src/main/scala/ch/wsl/box/client/views/ExportView.scala @@ -108,8 +108,8 @@ case class ExportView(model:ModelProperty[ExportModel], presenter:ExportPresente metadata.label ), JSONMetadataRenderer(metadata,model.subProp(_.queryData),Seq()).edit(), - a("load data",onclick :+= ((e:Event) => presenter.query()),GlobalStyles.boxButton), - a("download CSV",onclick :+= ((e:Event) => presenter.csv()),GlobalStyles.boxButton), + button("load data",onclick :+= ((e:Event) => presenter.query()),GlobalStyles.boxButton), + button("download CSV",onclick :+= ((e:Event) => presenter.csv()),GlobalStyles.boxButton), UdashTable()(model.subSeq(_.data))( headerFactory = Some(() => { tr( diff --git a/evolutions/02-exportFields.sql b/evolutions/02-exportFields.sql new file mode 100644 index 000000000..f31f2f8ae --- /dev/null +++ b/evolutions/02-exportFields.sql @@ -0,0 +1,3 @@ +alter table box.form_i18n drop column "exportView"; + +ALTER TABLE box.form ADD exportfields text NULL; \ No newline at end of file diff --git a/project/Settings.scala b/project/Settings.scala index 283a1d97a..77e9bac86 100755 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -69,6 +69,7 @@ object Settings { val udashJQuery = "1.2.0" val scribe = "2.6.0" + val scalaCSV = "1.3.6-SNAPSHOT-scalajs" } @@ -81,7 +82,8 @@ object Settings { "io.circe" %%% "circe-core" % versions.circe, "io.circe" %%% "circe-generic" % versions.circe, "io.circe" %%% "circe-parser" % versions.circe, - "com.outr" %%% "scribe" % versions.scribe + "com.outr" %%% "scribe" % versions.scribe, + "com.github.tototoshi" %%% "scala-csv" % versions.scalaCSV )) val sharedJVMCodegenDependencies = Def.setting(Seq( diff --git a/server/src/main/scala/ch/wsl/box/rest/boxentities/Form.scala b/server/src/main/scala/ch/wsl/box/rest/boxentities/Form.scala index 122b4d418..8a0524daf 100755 --- a/server/src/main/scala/ch/wsl/box/rest/boxentities/Form.scala +++ b/server/src/main/scala/ch/wsl/box/rest/boxentities/Form.scala @@ -22,14 +22,14 @@ object Form { * @param description Database column description SqlType(text), Default(None) * @param layout Database column layout SqlType(text), Default(None) */ case class Form_row(form_id: Option[Int] = None, name: String, entity:String, description: Option[String] = None, layout: Option[String] = None, - tabularFields: Option[String] = None, query: Option[String] = None) + tabularFields: Option[String] = None, query: Option[String] = None,exportFields: Option[String] = None) /** GetResult implicit for fetching Form_row objects using plain SQL queries */ /** Table description of table form. Objects of this class serve as prototypes for rows in queries. */ class Form(_tableTag: Tag) extends profile.api.Table[Form_row](_tableTag, "form") { - def * = (Rep.Some(form_id), name, entity, description, layout, tabularFields, query) <> (Form_row.tupled, Form_row.unapply) + def * = (Rep.Some(form_id), name, entity, description, layout, tabularFields, query,exportFields) <> (Form_row.tupled, Form_row.unapply) /** Maps whole row to an option. Useful for outer joins. */ - def ? = (Rep.Some(form_id), name, entity, description, layout, tabularFields, query).shaped.<>({ r=>import r._; _1.map(_=> Form_row.tupled((_1, _2, _3, _4, _5, _6, _7)))}, (_:Any) => throw new Exception("Inserting into ? projection not supported.")) + def ? = (Rep.Some(form_id), name, entity, description, layout, tabularFields, query,exportFields).shaped.<>({ r=>import r._; _1.map(_=> Form_row.tupled((_1, _2, _3, _4, _5, _6, _7, _8)))}, (_:Any) => throw new Exception("Inserting into ? projection not supported.")) /** Database column id SqlType(serial), AutoInc, PrimaryKey */ val form_id: Rep[Int] = column[Int]("form_id", O.AutoInc, O.PrimaryKey) @@ -43,6 +43,8 @@ object Form { val layout: Rep[Option[String]] = column[Option[String]]("layout", O.Default(None)) val tabularFields: Rep[Option[String]] = column[Option[String]]("tabularFields", O.Default(None)) + + val exportFields: Rep[Option[String]] = column[Option[String]]("exportfields", O.Default(None)) val query: Rep[Option[String]] = column[Option[String]]("query", O.Default(None)) @@ -61,14 +63,14 @@ object Form { * @param hint Database column hint SqlType(text), Default(None)*/ case class Form_i18n_row(id: Option[Int] = None, field_id: Option[Int] = None, lang: Option[String] = None, label: Option[String] = None, - tooltip: Option[String] = None, hint: Option[String] = None, exportView:Option[String] = None) + tooltip: Option[String] = None, hint: Option[String] = None) /** GetResult implicit for fetching Form_i18n_row objects using plain SQL queries */ /** Table description of table form_i18n. Objects of this class serve as prototypes for rows in queries. */ class Form_i18n(_tableTag: Tag) extends profile.api.Table[Form_i18n_row](_tableTag, "form_i18n") { - def * = (Rep.Some(id), form_id, lang, label, tooltip, hint, exportView) <> (Form_i18n_row.tupled, Form_i18n_row.unapply) + def * = (Rep.Some(id), form_id, lang, label, tooltip, hint) <> (Form_i18n_row.tupled, Form_i18n_row.unapply) /** Maps whole row to an option. Useful for outer joins. */ - def ? = (Rep.Some(id), form_id, lang, label, tooltip, hint, exportView).shaped.<>({ r=>import r._; _1.map(_=> Form_i18n_row.tupled((_1, _2, _3, _4, _5, _6, _7)))}, (_:Any) => throw new Exception("Inserting into ? projection not supported.")) + def ? = (Rep.Some(id), form_id, lang, label, tooltip, hint).shaped.<>({ r=>import r._; _1.map(_=> Form_i18n_row.tupled((_1, _2, _3, _4, _5, _6)))}, (_:Any) => throw new Exception("Inserting into ? projection not supported.")) /** Database column id SqlType(serial), AutoInc, PrimaryKey */ val id: Rep[Int] = column[Int]("id", O.AutoInc, O.PrimaryKey) @@ -83,7 +85,6 @@ object Form { /** Database column hint SqlType(text), Default(None) */ val hint: Rep[Option[String]] = column[Option[String]]("hint", O.Default(None)) - val exportView: Rep[Option[String]] = column[Option[String]]("exportView", O.Default(None)) /** Foreign key referencing Field (database name fkey_field) */ lazy val fieldFk = foreignKey("fkey_form", form_id, table)(r => Rep.Some(r.form_id), onUpdate=ForeignKeyAction.NoAction, onDelete=ForeignKeyAction.NoAction) diff --git a/server/src/main/scala/ch/wsl/box/rest/logic/FormActions.scala b/server/src/main/scala/ch/wsl/box/rest/logic/FormActions.scala index 753923e05..59fbef945 100755 --- a/server/src/main/scala/ch/wsl/box/rest/logic/FormActions.scala +++ b/server/src/main/scala/ch/wsl/box/rest/logic/FormActions.scala @@ -7,8 +7,9 @@ import io.circe._ import io.circe.syntax._ import ch.wsl.box.model.shared._ import ch.wsl.box.model.EntityActionsRegistry +import ch.wsl.box.rest.routes.enablers.CSVDownload import ch.wsl.box.rest.utils.{FutureUtils, Timer, UserProfile} -import ch.wsl.box.shared.utils.CSV +import com.github.tototoshi.csv.CSV import io.circe.Json import scribe.Logging import slick.basic.DatabasePublisher @@ -43,15 +44,15 @@ case class FormActions(metadata:JSONMetadata)(implicit up:UserProfile, mat:Mater def extractArray(query:JSONQuery):Source[Json,NotUsed] = extractSeq(query) // todo adapt JSONQuery to select only fields in form def extractOne(query:JSONQuery):Future[Json] = extractSeq(query).runFold(Seq[Json]())(_ ++ Seq(_)).map(x => if(x.length >1) throw new Exception("Multiple rows retrieved with single id") else x.headOption.asJson) - def csv(query:JSONQuery,lookupElements:Option[Map[String,Seq[Json]]]):Source[String,NotUsed] = { + def csv(query:JSONQuery,lookupElements:Option[Map[String,Seq[Json]]],fields:JSONMetadata => Seq[String] = _.tabularFields):Source[String,NotUsed] = { val lookup = Lookup.valueExtractor(lookupElements, metadata) _ extractSeq(query).map { json => - val row = metadata.tabularFields.map { field => + val row = fields(metadata).map { field => lookup(field,json.get(field)) } - CSV.row(row) + CSV.writeRow(row) } } diff --git a/server/src/main/scala/ch/wsl/box/rest/logic/JSONExportMetadataFactory.scala b/server/src/main/scala/ch/wsl/box/rest/logic/JSONExportMetadataFactory.scala index 884e37c49..f0acf9f70 100755 --- a/server/src/main/scala/ch/wsl/box/rest/logic/JSONExportMetadataFactory.scala +++ b/server/src/main/scala/ch/wsl/box/rest/logic/JSONExportMetadataFactory.scala @@ -63,7 +63,7 @@ case class JSONExportMetadataFactory(implicit db:Database, mat:Materializer, ec: val parameters = export.parameters.toSeq.flatMap(_.split(",")) - JSONMetadata(export.export_id.get,export.name,exportI18n.flatMap(_.label).getOrElse(function),jsonFields,layout,exportI18n.flatMap(_.function).getOrElse(export.function),lang,parameters,Seq(),None,None,"") + JSONMetadata(export.export_id.get,export.name,exportI18n.flatMap(_.label).getOrElse(function),jsonFields,layout,exportI18n.flatMap(_.function).getOrElse(export.function),lang,parameters,Seq(),None,Seq(),"") } } diff --git a/server/src/main/scala/ch/wsl/box/rest/logic/JSONFormMetadataFactory.scala b/server/src/main/scala/ch/wsl/box/rest/logic/JSONFormMetadataFactory.scala index 185e58607..f600c2bc0 100755 --- a/server/src/main/scala/ch/wsl/box/rest/logic/JSONFormMetadataFactory.scala +++ b/server/src/main/scala/ch/wsl/box/rest/logic/JSONFormMetadataFactory.scala @@ -147,7 +147,7 @@ case class JSONFormMetadataFactory(implicit up:UserProfile, mat:Materializer, ec - val result = JSONMetadata(form.form_id.get,form.name,formI18n.flatMap(_.label).getOrElse(form.name),jsonFields,layout,form.entity,lang,tableFields,keys,defaultQuery, formI18n.flatMap(_.exportView), form.entity) + val result = JSONMetadata(form.form_id.get,form.name,formI18n.flatMap(_.label).getOrElse(form.name),jsonFields,layout,form.entity,lang,tableFields,keys,defaultQuery, form.exportFields.map(_.split(",").toSeq).getOrElse(tableFields), form.entity) //println(s"resulting form: $result") result } diff --git a/server/src/main/scala/ch/wsl/box/rest/logic/JSONMetadataFactory.scala b/server/src/main/scala/ch/wsl/box/rest/logic/JSONMetadataFactory.scala index e38360319..f0a44d743 100755 --- a/server/src/main/scala/ch/wsl/box/rest/logic/JSONMetadataFactory.scala +++ b/server/src/main/scala/ch/wsl/box/rest/logic/JSONMetadataFactory.scala @@ -137,7 +137,8 @@ object JSONMetadataFactory extends Logging { fields <- Future.sequence(c.map(field2form)) keys <- JSONMetadataFactory.keysOf(table) } yield { - JSONMetadata(1, table, table, fields, Layout.fromFields(fields), table, lang, fields.map(_.name), keys, None, None, table) + val fieldList = fields.map(_.name) + JSONMetadata(1, table, table, fields, Layout.fromFields(fields), table, lang, fieldList, keys, None, fieldList, table) } logger.warn("adding to cache table " + Seq(up.name, table, lang, lookupMaxRows).mkString) diff --git a/server/src/main/scala/ch/wsl/box/rest/logic/Lookup.scala b/server/src/main/scala/ch/wsl/box/rest/logic/Lookup.scala index 0632223f5..30edcbc04 100755 --- a/server/src/main/scala/ch/wsl/box/rest/logic/Lookup.scala +++ b/server/src/main/scala/ch/wsl/box/rest/logic/Lookup.scala @@ -2,7 +2,7 @@ package ch.wsl.box.rest.logic import akka.stream.Materializer import ch.wsl.box.model.EntityActionsRegistry -import ch.wsl.box.model.shared.{JSONMetadata, JSONQuery} +import ch.wsl.box.model.shared.{JSONFieldLookup, JSONMetadata, JSONQuery} import io.circe.Json import slick.driver.PostgresDriver.api._ @@ -24,12 +24,15 @@ object Lookup { }.map(_.toMap) } - def valueExtractor(lookupElements:Option[Map[String,Seq[Json]]],metadata:JSONMetadata)(field:String, value:String) = { - lookupElements.flatMap { le => - def lookup = metadata.fields.find(_.name == field).flatMap(_.lookup) - le(lookup.get.lookupEntity).find(_.get(lookup.get.map.valueProperty) == value).map { lookupRow => - lookupRow.get(lookup.get.map.textProperty) - } - }.getOrElse(value) - } + def valueExtractor(lookupElements:Option[Map[String,Seq[Json]]],metadata:JSONMetadata)(field:String, value:String):String = { + + for{ + elements <- lookupElements + field <- metadata.fields.find(_.name == field) + lookup <- field.lookup + foreignEntity <- elements.get(lookup.lookupEntity) + foreignRow <- foreignEntity.find(_.get(lookup.map.valueProperty) == value) + } yield foreignRow.get(lookup.map.textProperty) + + }.getOrElse(value) } diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/Export.scala b/server/src/main/scala/ch/wsl/box/rest/routes/Export.scala index e22a0ebf5..47f673688 100755 --- a/server/src/main/scala/ch/wsl/box/rest/routes/Export.scala +++ b/server/src/main/scala/ch/wsl/box/rest/routes/Export.scala @@ -11,7 +11,7 @@ import ch.wsl.box.model.shared._ import ch.wsl.box.rest.jdbc.JdbcConnect import ch.wsl.box.rest.logic.{JSONExportMetadataFactory, JSONMetadataFactory} import ch.wsl.box.rest.utils.JSONSupport -import ch.wsl.box.shared.utils.CSV +import com.github.tototoshi.csv.CSV import io.circe.Json import io.circe.parser.parse import scribe.Logging @@ -32,7 +32,7 @@ object Export extends Logging { case Some(fr) => respondWithHeaders(`Content-Disposition`(ContentDispositionTypes.attachment, Map("filename" -> s"$function.csv"))) { { - val csv = CSV.of(Seq(fr.headers) ++ fr.rows.map(_.map(_.string))) + val csv = CSV.writeAll(Seq(fr.headers) ++ fr.rows.map(_.map(_.string))) complete(HttpEntity(ContentTypes.`text/csv(UTF-8)`,ByteString(csv))) } } diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/Form.scala b/server/src/main/scala/ch/wsl/box/rest/routes/Form.scala index f1b60bd5a..4a68065af 100755 --- a/server/src/main/scala/ch/wsl/box/rest/routes/Form.scala +++ b/server/src/main/scala/ch/wsl/box/rest/routes/Form.scala @@ -4,10 +4,10 @@ import akka.http.scaladsl.model.headers.{ContentDispositionTypes, `Content-Dispo import akka.stream.Materializer import akka.stream.scaladsl.Source import ch.wsl.box.model.EntityActionsRegistry -import ch.wsl.box.model.shared.{JSONCount, JSONID, JSONMetadata, JSONQuery} +import ch.wsl.box.model.shared._ import ch.wsl.box.rest.logic.{FormActions, JSONFormMetadataFactory, JSONMetadataFactory, Lookup} import ch.wsl.box.rest.utils.{JSONSupport, Timer, UserProfile} -import ch.wsl.box.shared.utils.CSV +import com.github.tototoshi.csv.CSV import io.circe.Json import io.circe.parser.parse import scribe.Logging @@ -41,8 +41,11 @@ case class Form(name:String,lang:String)(implicit up:UserProfile, ec: ExecutionC val jsonCustomMetadataFactory = JSONFormMetadataFactory() val metadata: Future[JSONMetadata] = jsonCustomMetadataFactory.of(name,lang) - val tabularMetadata = metadata.map{ f => - val filteredFields = f.fields.filter(field => f.tabularFields.contains(field.name)) + def tabularMetadata(fields:Option[Seq[String]] = None) = metadata.map{ f => + val filteredFields = fields match { + case Some(fields) => f.fields.filter(field => fields.contains(field.name)) + case None => f.fields.filter(field => f.tabularFields.contains(field.name)) + } f.copy(fields = filteredFields) } @@ -134,7 +137,7 @@ case class Form(name:String,lang:String)(implicit up:UserProfile, ec: ExecutionC post { entity(as[JSONQuery]) { query => logger.info("list") - complete(actions(tabularMetadata){ fs => + complete(actions(tabularMetadata()){ fs => fs.extractArray(query).map{arr => HttpEntity(ContentTypes.`text/plain(UTF-8)`,arr) } @@ -148,7 +151,7 @@ case class Form(name:String,lang:String)(implicit up:UserProfile, ec: ExecutionC logger.info("csv") complete{ for{ - metadata <- tabularMetadata + metadata <- tabularMetadata() } yield { val formActions = FormActions(metadata) formActions.csv(query,None) @@ -158,19 +161,25 @@ case class Form(name:String,lang:String)(implicit up:UserProfile, ec: ExecutionC } ~ respondWithHeader(`Content-Disposition`(ContentDispositionTypes.attachment,Map("filename" -> s"$name.csv"))) { get { - parameters('q, 'lang.?) { (q,resolveFk) => + parameters('q, 'fk.?,'fields.?) { (q,fk,fields) => val query = parse(q).right.get.as[JSONQuery].right.get + val tabMetadata = tabularMetadata(fields.map(_.split(",").toSeq)) complete{ for { - metadata <- tabularMetadata - fkValues <- resolveFk match { - case None => Future.successful(None) - case Some(_) => Lookup.valuesForEntity(metadata).map(Some(_)) + metadata <- tabMetadata + fkValues <- fk match { + case Some(ExportMode.RESOLVE_FK) => Lookup.valuesForEntity(metadata).map(Some(_)) + case _ => Future.successful(None) } } yield { + + logger.info(s"fk: ${fkValues.toString.take(50)}...") val formActions = FormActions(metadata) - Source.fromFuture(tabularMetadata.map(x => CSV.row(x.tabularFields))) - .concat(formActions.csv(query,fkValues)) + + val headers = metadata.exportFields.map(ef => metadata.fields.find(_.name == ef).map(_.title).getOrElse(ef)) + + Source.fromFuture(Future.successful(CSV.writeRow(headers))) + .concat(formActions.csv(query,fkValues,_.exportFields)) } } } diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/Root.scala b/server/src/main/scala/ch/wsl/box/rest/routes/Root.scala index af923720a..2793d22ba 100755 --- a/server/src/main/scala/ch/wsl/box/rest/routes/Root.scala +++ b/server/src/main/scala/ch/wsl/box/rest/routes/Root.scala @@ -16,7 +16,6 @@ import com.softwaremill.session.SessionDirectives._ import com.softwaremill.session.SessionOptions._ import ch.wsl.box.model.shared.LoginRequest import ch.wsl.box.rest.jdbc.JdbcConnect -import ch.wsl.box.shared.utils.CSV import scribe.Logging import scala.util.{Failure, Success} diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/Table.scala b/server/src/main/scala/ch/wsl/box/rest/routes/Table.scala index e20dbc63a..b494420e8 100755 --- a/server/src/main/scala/ch/wsl/box/rest/routes/Table.scala +++ b/server/src/main/scala/ch/wsl/box/rest/routes/Table.scala @@ -15,7 +15,7 @@ import ch.wsl.box.model.shared.{JSONCount, JSONData, JSONID, JSONQuery} import ch.wsl.box.rest.logic.{DbActions, JSONMetadataFactory} import ch.wsl.box.rest.utils.{JSONSupport, UserProfile} import ch.wsl.box.rest.utils.JSONSupport.jsonContentTypes -import ch.wsl.box.shared.utils.CSV +import com.github.tototoshi.csv.CSV import com.typesafe.config.{Config, ConfigFactory} import scribe.Logging import slick.lifted.TableQuery @@ -161,7 +161,7 @@ case class Table[T <: slick.jdbc.PostgresProfile.api.Table[M],M <: Product](name post { entity(as[JSONQuery]) { query => logger.info("csv") - complete(Source.fromPublisher(dbActions.findStreamed(query).mapResult(x => CSV.row(x.values())))) + complete(Source.fromPublisher(dbActions.findStreamed(query).mapResult(x => CSV.writeRow(x.values())))) } } ~ respondWithHeader(`Content-Disposition`(ContentDispositionTypes.attachment,Map("filename" -> s"$name.csv"))) { @@ -169,8 +169,8 @@ case class Table[T <: slick.jdbc.PostgresProfile.api.Table[M],M <: Product](name parameters('q) { q => val query = parse(q).right.get.as[JSONQuery].right.get val csv = Source.fromFuture(JSONMetadataFactory.of(name,"en", limitLookupFromFk).map{ metadata => - CSV.row(metadata.fields.map(_.name)) - }).concat(Source.fromPublisher(dbActions.findStreamed(query)).map(x => CSV.row(x.values()))) + CSV.writeRow(metadata.fields.map(_.name)) + }).concat(Source.fromPublisher(dbActions.findStreamed(query)).map(x => CSV.writeRow(x.values()))) complete(csv) } } diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/View.scala b/server/src/main/scala/ch/wsl/box/rest/routes/View.scala index 97dc78b78..83cfe7f8b 100755 --- a/server/src/main/scala/ch/wsl/box/rest/routes/View.scala +++ b/server/src/main/scala/ch/wsl/box/rest/routes/View.scala @@ -11,7 +11,7 @@ import ch.wsl.box.model.EntityActionsRegistry import ch.wsl.box.model.shared.{JSONCount, JSONData, JSONQuery} import ch.wsl.box.rest.logic.{DbActions, JSONMetadataFactory, Lookup} import ch.wsl.box.rest.utils.{JSONSupport, UserProfile} -import ch.wsl.box.shared.utils.CSV +import com.github.tototoshi.csv.{CSV, DefaultCSVFormat} import io.circe.{Decoder, Encoder} import io.circe.parser.parse import scribe.Logging @@ -38,6 +38,8 @@ case class View[T <: slick.jdbc.PostgresProfile.api.Table[M],M <: Product](name: ec: ExecutionContext) extends enablers.CSVDownload with Logging { + + View.views = Set(name) ++ View.views import JSONSupport._ @@ -103,7 +105,7 @@ case class View[T <: slick.jdbc.PostgresProfile.api.Table[M],M <: Product](name: entity(as[JSONQuery]) { query => logger.info("csv") //Source - complete(Source.fromPublisher(dbActions.findStreamed(query).mapResult(x => CSV.row(x.values())))) + complete(Source.fromPublisher(dbActions.findStreamed(query).mapResult(x => CSV.writeRow(x.values())))) } } ~ respondWithHeader(`Content-Disposition`(ContentDispositionTypes.attachment,Map("filename" -> s"$name.csv"))) { @@ -122,9 +124,9 @@ case class View[T <: slick.jdbc.PostgresProfile.api.Table[M],M <: Product](name: val lookup = Lookup.valueExtractor(fkValues,metadata) _ Source.fromFuture(Future.successful( - CSV.row(metadata.fields.map(_.name)) + CSV.writeRow(metadata.fields.map(_.name)) )).concat(Source.fromPublisher(dbActions.findStreamed(query).mapResult{x => - CSV.row(x.values()) + CSV.writeRow(x.values()) })) } } diff --git a/server/src/main/scala/ch/wsl/box/rest/routes/enablers/CSVDownload.scala b/server/src/main/scala/ch/wsl/box/rest/routes/enablers/CSVDownload.scala index 53db92537..9d02ada52 100755 --- a/server/src/main/scala/ch/wsl/box/rest/routes/enablers/CSVDownload.scala +++ b/server/src/main/scala/ch/wsl/box/rest/routes/enablers/CSVDownload.scala @@ -3,9 +3,10 @@ package ch.wsl.box.rest.routes.enablers import akka.http.scaladsl.common.EntityStreamingSupport import akka.http.scaladsl.marshalling.{Marshaller, Marshalling} import akka.http.scaladsl.model.ContentTypes +import akka.stream.scaladsl.Flow import akka.util.ByteString import ch.wsl.box.model.shared.JSONData -import ch.wsl.box.shared.utils.CSV +import com.github.tototoshi.csv.{CSV, DefaultCSVFormat} trait CSVDownload { @@ -13,7 +14,7 @@ trait CSVDownload { implicit def asCsv[M <: Product] = Marshaller.strict[M, ByteString] { t => Marshalling.WithFixedContentType(ContentTypes.`text/csv(UTF-8)`, () => { import JSONData._ - ByteString(CSV.row(t.values())) + ByteString(CSV.writeRow(t.values())) }) } @@ -25,4 +26,5 @@ trait CSVDownload { // [2] enable csv streaming: implicit val csvStreaming = EntityStreamingSupport.csv() + .withFramingRenderer(Flow[ByteString].map(bs => bs)) //no new line, let the CSV library manage the new lines } diff --git a/shared/src/main/scala/ch/wsl/box/model/shared/JSONData.scala b/shared/src/main/scala/ch/wsl/box/model/shared/JSONData.scala index 94b52993a..a157c1060 100755 --- a/shared/src/main/scala/ch/wsl/box/model/shared/JSONData.scala +++ b/shared/src/main/scala/ch/wsl/box/model/shared/JSONData.scala @@ -2,7 +2,9 @@ package ch.wsl.box.model.shared import java.text.SimpleDateFormat -import ch.wsl.box.shared.utils.CSV +import com.github.tototoshi.csv.CSV + + /** * Created by andreaminetti on 03/03/16. @@ -10,7 +12,7 @@ import ch.wsl.box.shared.utils.CSV case class JSONData[M <: Product](data:Seq[M], count:Int) { import JSONData._ - def csv:String = CSV.of(data.map(_.values())) + def csv:String = CSV.writeAll(data.map(_.values())) } object JSONData{ diff --git a/shared/src/main/scala/ch/wsl/box/model/shared/JSONMetadata.scala b/shared/src/main/scala/ch/wsl/box/model/shared/JSONMetadata.scala index c79eff4f6..abf93c9d9 100755 --- a/shared/src/main/scala/ch/wsl/box/model/shared/JSONMetadata.scala +++ b/shared/src/main/scala/ch/wsl/box/model/shared/JSONMetadata.scala @@ -20,7 +20,7 @@ case class JSONMetadata( tabularFields:Seq[String], keys:Seq[String], query:Option[JSONQuery], - exportView:Option[String], + exportFields:Seq[String], baseTable:String ) diff --git a/shared/src/main/scala/ch/wsl/box/shared/utils/CSV.scala b/shared/src/main/scala/ch/wsl/box/shared/utils/CSV.scala deleted file mode 100755 index 18467b469..000000000 --- a/shared/src/main/scala/ch/wsl/box/shared/utils/CSV.scala +++ /dev/null @@ -1,22 +0,0 @@ -package ch.wsl.box.shared.utils - - -/** - * Created by andre on 5/22/2017. - */ -object CSV { - - private def escape(str:String):String = { - // str.contains(",") || str.contains("\n") match { - // case true => "\""+str.replaceAll("\"","\\\"")+"\"" - // case false => str - // } - "\""+str.replaceAll("\"","\\\"").replaceAll("\n","\\\\n")+"\"" - } - - def of(data:Seq[Seq[String]]):String = data.map(row).mkString("\n") - - def row(row:Seq[String]):String = row.map(escape).mkString(",") - - def split(data:String):Seq[Seq[String]] = data.split("\n").toSeq.map{x => x.replaceAll("\\\\n","\n").substring(1,x.length-1).split("\",\"").toSeq} -}