Speed-up GeoTIFF file handling (geojupyter#262)

* speed-up geotiff loading with caching

geotiff cache added.

pre loading file method added.

indexedDB

added indexedDB for caching

remove console log

* lint

* move database functions to tools

* use caching in mainview

packages/base/src/dialogs/symbology/tiff_layer/types/SingleBandPseudoColor.tsx

@@ -12,8 +12,8 @@ import ColorRamp, {
 } from '../../components/color_ramp/ColorRamp';
 import StopRow from '../../components/color_stops/StopRow';
 import { Utils } from '../../symbologyUtils';
-import { getGdal } from '../../../../gdal';
 import { Spinner } from '../../../../mainview/spinner';
+import { loadGeoTIFFWithCache } from '../../../../tools';
 
 export interface IBandRow {
   band: number;
@@ -129,50 +129,42 @@ const SingleBandPseudoColor = ({
     setSelectedFunction(interpolation);
   };
 
+  const preloadGeoTiffFile = async (sourceInfo: {
+    url?: string | undefined;
+  }) => {
+    return await loadGeoTIFFWithCache(sourceInfo);
+  };
+
   const getBandInfo = async () => {
     const bandsArr: IBandRow[] = [];
     const source = context.model.getSource(layer?.parameters?.source);
     const sourceInfo = source?.parameters?.urls[0];
 
-    if (!sourceInfo.url) {
+    if (!sourceInfo?.url) {
       return;
     }
 
-    let tifData;
-
-    if (layerState && layerState.tifData) {
-      tifData = JSON.parse(layerState.tifData as string);
-    } else {
-      const Gdal = await getGdal();
-
-      const fileData = await fetch(sourceInfo.url);
-      const file = new File([await fileData.blob()], 'loaded.tif');
-
-      const result = await Gdal.open(file);
-      const tifDataset = result.datasets[0];
-      tifData = await Gdal.gdalinfo(tifDataset, ['-stats']);
-      Gdal.close(tifDataset);
-
-      await stateDb?.save(`jupytergis:${layerId}`, {
-        tifData: JSON.stringify(tifData)
+    // Preload the file only once
+    const preloadedFile = await preloadGeoTiffFile(sourceInfo);
+    const { file, metadata, sourceUrl } = { ...preloadedFile };
+
+    if (file && metadata && sourceUrl === sourceInfo.url) {
+      metadata['bands'].forEach((bandData: TifBandData) => {
+        bandsArr.push({
+          band: bandData.band,
+          colorInterpretation: bandData.colorInterpretation,
+          stats: {
+            minimum: sourceInfo.min ?? bandData.minimum,
+            maximum: sourceInfo.max ?? bandData.maximum,
+            mean: bandData.mean,
+            stdDev: bandData.stdDev
+          },
+          metadata: bandData.metadata,
+          histogram: bandData.histogram
+        });
       });
+      setBandRows(bandsArr);
     }
-
-    tifData['bands'].forEach((bandData: TifBandData) => {
-      bandsArr.push({
-        band: bandData.band,
-        colorInterpretation: bandData.colorInterpretation,
-        stats: {
-          minimum: sourceInfo.min ?? bandData.minimum,
-          maximum: sourceInfo.max ?? bandData.maximum,
-          mean: bandData.mean,
-          stdDev: bandData.stdDev
-        },
-        metadata: bandData.metadata,
-        histogram: bandData.histogram
-      });
-    });
-    setBandRows(bandsArr);
   };
 
   const buildColorInfo = () => {

packages/base/src/mainview/mainView.tsx

@@ -60,7 +60,7 @@ import { Rule } from 'ol/style/flat';
 import proj4 from 'proj4';
 import * as React from 'react';
 import shp from 'shpjs';
-import { isLightTheme } from '../tools';
+import { isLightTheme, loadGeoTIFFWithCache } from '../tools';
 import { MainViewModel } from './mainviewmodel';
 import { Spinner } from './spinner';
 //@ts-expect-error no types for proj4-list
@@ -313,6 +313,13 @@ export class MainView extends React.Component<IProps, IStates> {
     }
   }
 
+  private async _loadGeoTIFFWithCache(sourceInfo: {
+    url?: string | undefined;
+  }) {
+    const result = await loadGeoTIFFWithCache(sourceInfo);
+    return result?.file;
+  }
+
   /**
    * Add a source in the map.
    *
@@ -473,9 +480,15 @@ export class MainView extends React.Component<IProps, IStates> {
         const addNoData = (url: (typeof sourceParameters.urls)[0]) => {
           return { ...url, nodata: 0 };
         };
+        const sourcesWithBlobs = await Promise.all(
+          sourceParameters.urls.map(async sourceInfo => {
+            const blob = await this._loadGeoTIFFWithCache(sourceInfo);
+            return { ...addNoData(sourceInfo), blob };
+          })
+        );
 
         newSource = new GeoTIFFSource({
-          sources: sourceParameters.urls.map(addNoData),
+          sources: sourcesWithBlobs,
           normalize: sourceParameters.normalize,
           wrapX: sourceParameters.wrapX
         });

packages/base/src/tools.ts

@@ -13,6 +13,7 @@ import {
   IRasterLayerGalleryEntry
 } from '@jupytergis/schema';
 import RASTER_LAYER_GALLERY from '../rasterlayer_gallery/raster_layer_gallery.json';
+import { getGdal } from './gdal';
 
 export const debounce = (
   func: CallableFunction,
@@ -347,3 +348,107 @@ export function parseColor(type: string, style: any) {
 
   return parsedStyle;
 }
+
+/**
+ * Open or create an IndexedDB database for caching GeoTIFF files.
+ *
+ * @returns A promise that resolves to the opened IndexedDB database instance.
+ */
+export const openDatabase = () => {
+  return new Promise<IDBDatabase>((resolve, reject) => {
+    const request = indexedDB.open('GeoTIFFCache', 1);
+
+    request.onupgradeneeded = event => {
+      const db = request.result;
+      if (!db.objectStoreNames.contains('files')) {
+        db.createObjectStore('files', { keyPath: 'url' });
+      }
+    };
+
+    request.onsuccess = () => resolve(request.result);
+    request.onerror = () => reject(request.error);
+  });
+};
+
+/**
+ * Save a file and its metadata to the IndexedDB database.
+ *
+ * @param key file ID (sourceUrl).
+ * @param file Blob object representing the file content.
+ * @param metadata metadata of file.
+ * @returns A promise that resolves once the data is successfully saved.
+ */
+export const saveToIndexedDB = async (
+  key: string,
+  file: Blob,
+  metadata: any
+) => {
+  const db = await openDatabase();
+  return new Promise<void>((resolve, reject) => {
+    const transaction = db.transaction('files', 'readwrite');
+    const store = transaction.objectStore('files');
+    store.put({ url: key, file, metadata });
+
+    transaction.oncomplete = () => resolve();
+    transaction.onerror = () => reject(transaction.error);
+  });
+};
+
+/**
+ * Retrieve a file and its metadata from the IndexedDB database.
+ *
+ * @param key fileID (sourceUrl).
+ * @returns A promise that resolves to the stored data object or undefined.
+ */
+export const getFromIndexedDB = async (key: string) => {
+  const db = await openDatabase();
+  return new Promise<any>((resolve, reject) => {
+    const transaction = db.transaction('files', 'readonly');
+    const store = transaction.objectStore('files');
+    const request = store.get(key);
+
+    request.onsuccess = () => resolve(request.result);
+    request.onerror = () => reject(request.error);
+  });
+};
+
+/**
+ * Load a GeoTIFF file from IndexedDB database cache or fetch it .
+ *
+ * @param sourceInfo object containing the URL of the GeoTIFF file.
+ * @returns A promise that resolves to the file as a Blob, or undefined .
+ */
+export const loadGeoTIFFWithCache = async (sourceInfo: {
+  url?: string | undefined;
+}) => {
+  if (!sourceInfo?.url) {
+    return null;
+  }
+
+  const cachedData = await getFromIndexedDB(sourceInfo.url);
+  if (cachedData) {
+    return {
+      file: new Blob([cachedData.file]),
+      metadata: cachedData.metadata,
+      sourceUrl: sourceInfo.url
+    };
+  }
+
+  const response = await fetch(sourceInfo.url);
+  const fileBlob = await response.blob();
+  const file = new File([fileBlob], 'loaded.tif');
+
+  const Gdal = await getGdal();
+  const result = await Gdal.open(file);
+  const tifDataset = result.datasets[0];
+  const metadata = await Gdal.gdalinfo(tifDataset, ['-stats']);
+  Gdal.close(tifDataset);
+
+  await saveToIndexedDB(sourceInfo.url, fileBlob, metadata);
+
+  return {
+    file: fileBlob,
+    metadata,
+    sourceUrl: sourceInfo.url
+  };
+};