From 090438b6b51b24dfc64a6b89343f53e02420034a Mon Sep 17 00:00:00 2001 From: Pekka Helesuo Date: Thu, 27 Jul 2023 14:23:24 +0300 Subject: [PATCH] limit size of json when trying to export to excel and build csv-option in fe --- .../plugin/FeatureDataPluginHandler.js | 68 ++++++++++++++++--- .../featuredata/resources/locale/en.js | 1 + .../featuredata/resources/locale/fi.js | 1 + .../featuredata/resources/locale/sv.js | 1 + .../framework/featuredata/view/ExportData.jsx | 3 +- 5 files changed, 64 insertions(+), 10 deletions(-) diff --git a/bundles/framework/featuredata/plugin/FeatureDataPluginHandler.js b/bundles/framework/featuredata/plugin/FeatureDataPluginHandler.js index 3accc93054..d877db2b39 100644 --- a/bundles/framework/featuredata/plugin/FeatureDataPluginHandler.js +++ b/bundles/framework/featuredata/plugin/FeatureDataPluginHandler.js @@ -1,7 +1,7 @@ import { StateHandler, controllerMixin, Messaging } from 'oskari-ui/util'; import { showFeatureDataFlyout } from '../view/FeatureDataFlyout'; import { FilterTypes, LogicalOperators, showSelectByPropertiesPopup } from '../view/SelectByProperties'; -import { COLUMN_SELECTION, FILETYPES, showExportDataPopup } from '../view/ExportData'; +import { COLUMN_SELECTION, FILETYPES, SEPARATORS, showExportDataPopup } from '../view/ExportData'; import { FEATUREDATA_BUNDLE_ID } from '../view/FeatureDataContainer'; import { FilterSelector } from '../FilterSelector'; @@ -9,6 +9,7 @@ export const FEATUREDATA_DEFAULT_HIDDEN_FIELDS = ['__fid', '__centerX', '__cente export const SELECTION_SERVICE_CLASSNAME = 'Oskari.mapframework.service.VectorFeatureSelectionService'; const EXPORT_FEATUREDATA_ROUTE = 'ExportTableFile'; +const EXPORT_FEATUREDATA_EXCEL_MAX_LENGTH = 120000; class FeatureDataPluginUIHandler extends StateHandler { constructor (mapModule) { @@ -409,19 +410,70 @@ class FeatureDataPluginUIHandler extends StateHandler { const layer = layers?.find(layer => layer.getId() === activeLayerId); params.filename = layer.getName(); - params.data = JSON.stringify(this.gatherExportData(exportOnlySelected, columns === COLUMN_SELECTION.opened)); - params.additionalData = JSON.stringify( - this.gatherAdditionalInfo( - exportDataSource, - exportMetadataLink, - layer)); + params.data = this.gatherExportData(exportOnlySelected, columns === COLUMN_SELECTION.opened); + params.additionalData = this.gatherAdditionalInfo( + exportDataSource, + exportMetadataLink, + layer + ); + + if (format === FILETYPES.excel) { + this.exportAsExcel(params); + return; + } + + this.exportAsCsv(params); + } + + exportAsExcel (params) { + params.data = JSON.stringify(params.data); + params.additionalData = JSON.stringify(params.additionalData); + let contentLength = 0; + Object.keys(params).forEach((key) => { + if (params[key]?.length) { + contentLength += params[key].length; + } + }); + + if (contentLength >= EXPORT_FEATUREDATA_EXCEL_MAX_LENGTH) { + Messaging.error(Oskari.getMsg(FEATUREDATA_BUNDLE_ID, 'exportDataPopup.datasetTooLargeForExcel')); + return; + } const payload = new URLSearchParams(); Object.keys(params).forEach(key => payload.append(key, params[key])); - const extension = format === FILETYPES.csv ? '.csv' : '.xlsx'; + const extension = '.xlsx'; this.sendData(payload, params.filename + extension); } + exportAsCsv (params) { + if (params?.delimiter === SEPARATORS.tabulator) { + params.delimiter = '\t'; + } + const replacer = (value) => !value ? '' : value; + const headers = params.data.splice(0, 1)[0]; + const headerRow = headers.join(params.delimiter); + const dataRows = params.data + .map((values) => values.map((value) => replacer(value)).join(params.delimiter)) + .join('\r\n'); + + const additionalDataRows = params.additionalData.map((value) => { + // TODO: case metadata - we should dig up the metadata service url from someplace + // not currently available in fe + return [value.name, value.value].join(params.delimiter); + }).join('\r\n'); + const emptyRow = null; + const csv = [headerRow, dataRows, emptyRow, additionalDataRows].join('\r\n'); + + const link = document.createElement('a'); + link.setAttribute('href', 'data:text/csv;charset=utf-8,%EF%BB%BF' + encodeURIComponent(csv)); + link.setAttribute('download', params.filename + '.csv'); + link.style.visibility = 'hidden'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + } + sendData (payload, filename) { try { fetch(Oskari.urls.getRoute(EXPORT_FEATUREDATA_ROUTE), { diff --git a/bundles/framework/featuredata/resources/locale/en.js b/bundles/framework/featuredata/resources/locale/en.js index fbd5716f20..85ca3e21de 100644 --- a/bundles/framework/featuredata/resources/locale/en.js +++ b/bundles/framework/featuredata/resources/locale/en.js @@ -36,6 +36,7 @@ Oskari.registerLocalization( "openButtonLabel": "Export data", "exportButtonLabel": "Export", "exportFailed": "Exporting the data failed.", + "datasetTooLargeForExcel": "Dataset is too large for generating an excel file. Limit the number of features or choose csv as format.", "fileFormat": { "title": "File format", "excel": "Excel", diff --git a/bundles/framework/featuredata/resources/locale/fi.js b/bundles/framework/featuredata/resources/locale/fi.js index 12c45daa3e..288c21fac8 100644 --- a/bundles/framework/featuredata/resources/locale/fi.js +++ b/bundles/framework/featuredata/resources/locale/fi.js @@ -36,6 +36,7 @@ Oskari.registerLocalization( "openButtonLabel": "Aineiston vienti", "exportButtonLabel": "Vie", "exportFailed": "Aineiston vienti epäonnistui.", + "datasetTooLargeForExcel": "Dataset is too large for generating an excel file. Limit the number of features or choose csv as format.", "fileFormat": { "title": "Tiedostomuoto", "excel": "Excel", diff --git a/bundles/framework/featuredata/resources/locale/sv.js b/bundles/framework/featuredata/resources/locale/sv.js index 707fea768c..879031ee1d 100644 --- a/bundles/framework/featuredata/resources/locale/sv.js +++ b/bundles/framework/featuredata/resources/locale/sv.js @@ -36,6 +36,7 @@ Oskari.registerLocalization( "openButtonLabel": "Exportera data", "exportButtonLabel": "Exportera", "exportFailed": "Exporting the data failed.", + "datasetTooLargeForExcel": "Dataset is too large for generating an excel file. Limit the number of features or choose csv as format.", "fileFormat": { "title": "Filformat", "excel": "Excel", diff --git a/bundles/framework/featuredata/view/ExportData.jsx b/bundles/framework/featuredata/view/ExportData.jsx index 949c59f864..61e2f0f379 100644 --- a/bundles/framework/featuredata/view/ExportData.jsx +++ b/bundles/framework/featuredata/view/ExportData.jsx @@ -45,8 +45,7 @@ const ColumnHeader = styled('h3')` padding-bottom: 0.25em; `; - -const SEPARATORS = { +export const SEPARATORS = { semicolon: ';', comma: ',', tabulator: 'tab'