From 3bfd970aa7a513ebad30d31acf24adef11b6cf32 Mon Sep 17 00:00:00 2001 From: Meriem Ben Ismail <161816721+Meriem-BenIsmail@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:47:59 +0100 Subject: [PATCH] Speed-up GeoTIFF file handling (#262) * speed-up geotiff loading with caching geotiff cache added. pre loading file method added. indexedDB added indexedDB for caching remove console log * lint * move database functions to tools * use caching in mainview --- .../types/SingleBandPseudoColor.tsx | 62 +++++------ packages/base/src/mainview/mainView.tsx | 17 ++- packages/base/src/tools.ts | 105 ++++++++++++++++++ 3 files changed, 147 insertions(+), 37 deletions(-) diff --git a/packages/base/src/dialogs/symbology/tiff_layer/types/SingleBandPseudoColor.tsx b/packages/base/src/dialogs/symbology/tiff_layer/types/SingleBandPseudoColor.tsx index 73b58c19..1956017e 100644 --- a/packages/base/src/dialogs/symbology/tiff_layer/types/SingleBandPseudoColor.tsx +++ b/packages/base/src/dialogs/symbology/tiff_layer/types/SingleBandPseudoColor.tsx @@ -12,8 +12,8 @@ import ColorRamp, { } from '../../components/color_ramp/ColorRamp'; import StopRow from '../../components/color_stops/StopRow'; import { Utils } from '../../symbologyUtils'; -import { getGdal } from '../../../../gdal'; import { Spinner } from '../../../../mainview/spinner'; +import { loadGeoTIFFWithCache } from '../../../../tools'; export interface IBandRow { band: number; @@ -129,50 +129,42 @@ const SingleBandPseudoColor = ({ setSelectedFunction(interpolation); }; + const preloadGeoTiffFile = async (sourceInfo: { + url?: string | undefined; + }) => { + return await loadGeoTIFFWithCache(sourceInfo); + }; + const getBandInfo = async () => { const bandsArr: IBandRow[] = []; const source = context.model.getSource(layer?.parameters?.source); const sourceInfo = source?.parameters?.urls[0]; - if (!sourceInfo.url) { + if (!sourceInfo?.url) { return; } - let tifData; - - if (layerState && layerState.tifData) { - tifData = JSON.parse(layerState.tifData as string); - } else { - const Gdal = await getGdal(); - - const fileData = await fetch(sourceInfo.url); - const file = new File([await fileData.blob()], 'loaded.tif'); - - const result = await Gdal.open(file); - const tifDataset = result.datasets[0]; - tifData = await Gdal.gdalinfo(tifDataset, ['-stats']); - Gdal.close(tifDataset); - - await stateDb?.save(`jupytergis:${layerId}`, { - tifData: JSON.stringify(tifData) + // Preload the file only once + const preloadedFile = await preloadGeoTiffFile(sourceInfo); + const { file, metadata, sourceUrl } = { ...preloadedFile }; + + if (file && metadata && sourceUrl === sourceInfo.url) { + metadata['bands'].forEach((bandData: TifBandData) => { + bandsArr.push({ + band: bandData.band, + colorInterpretation: bandData.colorInterpretation, + stats: { + minimum: sourceInfo.min ?? bandData.minimum, + maximum: sourceInfo.max ?? bandData.maximum, + mean: bandData.mean, + stdDev: bandData.stdDev + }, + metadata: bandData.metadata, + histogram: bandData.histogram + }); }); + setBandRows(bandsArr); } - - tifData['bands'].forEach((bandData: TifBandData) => { - bandsArr.push({ - band: bandData.band, - colorInterpretation: bandData.colorInterpretation, - stats: { - minimum: sourceInfo.min ?? bandData.minimum, - maximum: sourceInfo.max ?? bandData.maximum, - mean: bandData.mean, - stdDev: bandData.stdDev - }, - metadata: bandData.metadata, - histogram: bandData.histogram - }); - }); - setBandRows(bandsArr); }; const buildColorInfo = () => { diff --git a/packages/base/src/mainview/mainView.tsx b/packages/base/src/mainview/mainView.tsx index 8f0ecef4..0c98836e 100644 --- a/packages/base/src/mainview/mainView.tsx +++ b/packages/base/src/mainview/mainView.tsx @@ -60,7 +60,7 @@ import { Rule } from 'ol/style/flat'; import proj4 from 'proj4'; import * as React from 'react'; import shp from 'shpjs'; -import { isLightTheme } from '../tools'; +import { isLightTheme, loadGeoTIFFWithCache } from '../tools'; import { MainViewModel } from './mainviewmodel'; import { Spinner } from './spinner'; //@ts-expect-error no types for proj4-list @@ -313,6 +313,13 @@ export class MainView extends React.Component { } } + private async _loadGeoTIFFWithCache(sourceInfo: { + url?: string | undefined; + }) { + const result = await loadGeoTIFFWithCache(sourceInfo); + return result?.file; + } + /** * Add a source in the map. * @@ -473,9 +480,15 @@ export class MainView extends React.Component { const addNoData = (url: (typeof sourceParameters.urls)[0]) => { return { ...url, nodata: 0 }; }; + const sourcesWithBlobs = await Promise.all( + sourceParameters.urls.map(async sourceInfo => { + const blob = await this._loadGeoTIFFWithCache(sourceInfo); + return { ...addNoData(sourceInfo), blob }; + }) + ); newSource = new GeoTIFFSource({ - sources: sourceParameters.urls.map(addNoData), + sources: sourcesWithBlobs, normalize: sourceParameters.normalize, wrapX: sourceParameters.wrapX }); diff --git a/packages/base/src/tools.ts b/packages/base/src/tools.ts index 436df859..e8e2a6a3 100644 --- a/packages/base/src/tools.ts +++ b/packages/base/src/tools.ts @@ -13,6 +13,7 @@ import { IRasterLayerGalleryEntry } from '@jupytergis/schema'; import RASTER_LAYER_GALLERY from '../rasterlayer_gallery/raster_layer_gallery.json'; +import { getGdal } from './gdal'; export const debounce = ( func: CallableFunction, @@ -347,3 +348,107 @@ export function parseColor(type: string, style: any) { return parsedStyle; } + +/** + * Open or create an IndexedDB database for caching GeoTIFF files. + * + * @returns A promise that resolves to the opened IndexedDB database instance. + */ +export const openDatabase = () => { + return new Promise((resolve, reject) => { + const request = indexedDB.open('GeoTIFFCache', 1); + + request.onupgradeneeded = event => { + const db = request.result; + if (!db.objectStoreNames.contains('files')) { + db.createObjectStore('files', { keyPath: 'url' }); + } + }; + + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); +}; + +/** + * Save a file and its metadata to the IndexedDB database. + * + * @param key file ID (sourceUrl). + * @param file Blob object representing the file content. + * @param metadata metadata of file. + * @returns A promise that resolves once the data is successfully saved. + */ +export const saveToIndexedDB = async ( + key: string, + file: Blob, + metadata: any +) => { + const db = await openDatabase(); + return new Promise((resolve, reject) => { + const transaction = db.transaction('files', 'readwrite'); + const store = transaction.objectStore('files'); + store.put({ url: key, file, metadata }); + + transaction.oncomplete = () => resolve(); + transaction.onerror = () => reject(transaction.error); + }); +}; + +/** + * Retrieve a file and its metadata from the IndexedDB database. + * + * @param key fileID (sourceUrl). + * @returns A promise that resolves to the stored data object or undefined. + */ +export const getFromIndexedDB = async (key: string) => { + const db = await openDatabase(); + return new Promise((resolve, reject) => { + const transaction = db.transaction('files', 'readonly'); + const store = transaction.objectStore('files'); + const request = store.get(key); + + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); +}; + +/** + * Load a GeoTIFF file from IndexedDB database cache or fetch it . + * + * @param sourceInfo object containing the URL of the GeoTIFF file. + * @returns A promise that resolves to the file as a Blob, or undefined . + */ +export const loadGeoTIFFWithCache = async (sourceInfo: { + url?: string | undefined; +}) => { + if (!sourceInfo?.url) { + return null; + } + + const cachedData = await getFromIndexedDB(sourceInfo.url); + if (cachedData) { + return { + file: new Blob([cachedData.file]), + metadata: cachedData.metadata, + sourceUrl: sourceInfo.url + }; + } + + const response = await fetch(sourceInfo.url); + const fileBlob = await response.blob(); + const file = new File([fileBlob], 'loaded.tif'); + + const Gdal = await getGdal(); + const result = await Gdal.open(file); + const tifDataset = result.datasets[0]; + const metadata = await Gdal.gdalinfo(tifDataset, ['-stats']); + Gdal.close(tifDataset); + + await saveToIndexedDB(sourceInfo.url, fileBlob, metadata); + + return { + file: fileBlob, + metadata, + sourceUrl: sourceInfo.url + }; +};