Enable multi-ZIM support by removing SW timer #1146 (#1164)

kiwix · Nov 17, 2023 · ddf4305 · ddf4305
1 parent 7d6e4dc
commit ddf4305
Show file tree

Hide file tree

Showing 12 changed files with 196 additions and 154 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -94,7 +94,7 @@ jobs:
           local-testing: start
           local-identifier: random
 
-      - name: Run App locally in background
+      - name: Start server locally in background
         if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
         env:
           BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }}

diff --git a/i18n/en.jsonp.js b/i18n/en.jsonp.js
diff --git a/i18n/es.jsonp.js b/i18n/es.jsonp.js
diff --git a/i18n/fr.jsonp.js b/i18n/fr.jsonp.js
diff --git a/service-worker.js b/service-worker.js
@@ -69,22 +69,22 @@ var useAppCache = true;
  * Add any further Content-Types you wish to cache to the regexp, separated by '|'
  * @type {RegExp}
  */
-var regexpCachedContentTypes = /text\/css|text\/javascript|application\/javascript/i;
+const regexpCachedContentTypes = /text\/css|\/javascript|application\/javascript/i;
 
 /**
  * A regular expression that excludes listed schemata from caching attempts
  * As of 08-2019 the chrome-extension: schema is incompatible with the Cache API
  * 'example-extension' is included to show how to add another schema if necessary
  * @type {RegExp}
  */
-var regexpExcludedURLSchema = /^(?:file|chrome-extension|example-extension):/i;
+const regexpExcludedURLSchema = /^(?:file|chrome-extension|example-extension):/i;
 
 /**
  * Pattern for ZIM file namespace: see https://wiki.openzim.org/wiki/ZIM_file_format#Namespaces
  * In our case, there is also the ZIM file name used as a prefix in the URL
  * @type {RegExp}
  */
-const regexpZIMUrlWithNamespace = /(?:^|\/)([^/]+\/)([-ABCIJMUVWX])\/(.+)/;
+const regexpZIMUrlWithNamespace = /(?:^|\/)([^/]+\/)([-ABCHIJMUVWX])\/(.+)/;
 
 /**
  * Pattern to parse the first offset of a "range" request header
@@ -201,6 +201,7 @@ self.addEventListener('install', function (event) {
 
 // Allow sw to control current page
 self.addEventListener('activate', function (event) {
+    console.debug('[SW] Activate Event processing');
     // "Claiming" the ServiceWorker is necessary to make it work right away,
     // without the need to reload the page.
     // See https://developer.mozilla.org/en-US/docs/Web/API/Clients/claim
@@ -222,16 +223,18 @@ self.addEventListener('activate', function (event) {
     );
 });
 
-let outgoingMessagePort = null;
-let fetchCaptureEnabled = false;
+// For PWA functionality, this should be true unless explicitly disabled, and in fact currently it is never disabled
+let fetchCaptureEnabled = true;
 
 /**
  * Intercept selected Fetch requests from the browser window
  */
 self.addEventListener('fetch', function (event) {
-    // Only cache GET requests
-    if (event.request.method !== 'GET') return;
+    // Only handle GET or POST requests (POST is intended to handle video in Zimit ZIMs)
+    if (!/GET|POST/.test(event.request.method)) return;
     var rqUrl = event.request.url;
+    // Filter out requests that do not match the scope of the Service Worker
+    if (/\/dist\/(www|[^/]+?\.zim)\//.test(rqUrl) && !/\/dist\//.test(self.registration.scope)) return;
     var urlObject = new URL(rqUrl);
     // Test the URL with parameters removed
     var strippedUrl = urlObject.pathname;
@@ -241,15 +244,16 @@ self.addEventListener('fetch', function (event) {
     // For APP_CACHE assets, we should ignore any querystring (whereas it should be conserved for ZIM assets,
     // especially .js assets, where it may be significant). Anchor targets are irreleveant in this context.
     if (cache === APP_CACHE) rqUrl = strippedUrl;
-    event.respondWith(
+    return event.respondWith(
         // First see if the content is in the cache
         fromCache(cache, rqUrl).then(function (response) {
             // The response was found in the cache so we respond with it
             return response;
         }, function () {
             // The response was not found in the cache so we look for it in the ZIM
             // and add it to the cache if it is an asset type (css or js)
-            if (cache === ASSETS_CACHE && regexpZIMUrlWithNamespace.test(strippedUrl)) {
+            // YouTube links from Zimit archives are dealt with specially
+            if (/youtubei.*player/.test(strippedUrl) || cache === ASSETS_CACHE && regexpZIMUrlWithNamespace.test(strippedUrl)) {
                 const range = event.request.headers.get('range');
                 return fetchUrlFromZIM(urlObject, range).then(function (response) {
                     // Add css or js assets to ASSETS_CACHE (or update their cache entries) unless the URL schema is not supported
@@ -284,12 +288,18 @@ self.addEventListener('fetch', function (event) {
 self.addEventListener('message', function (event) {
     if (event.data.action) {
         if (event.data.action === 'init') {
-            // On 'init' message, we initialize the outgoingMessagePort and enable the fetchEventListener
-            outgoingMessagePort = event.ports[0];
+            // On 'init' message, we enable the fetchEventListener
             fetchCaptureEnabled = true;
+            // Acdknowledge the init message to all clients
+            self.clients.matchAll().then(function (clientList) {
+                clientList.forEach(function (client) {
+                    client.postMessage({ action: 'acknowledge' });
+                });
+            });
         } else if (event.data.action === 'disable') {
-            // On 'disable' message, we delete the outgoingMessagePort and disable the fetchEventListener
-            outgoingMessagePort = null;
+            // On 'disable' message, we disable the fetchEventListener
+            // Note that this code doesn't currently run because the app currently never sends a 'disable' message
+            // This is because the app may be running as a PWA, and still needs to be able to fetch assets even in jQuery mode
             fetchCaptureEnabled = false;
         }
         var oldValue;
@@ -330,15 +340,15 @@ function fetchUrlFromZIM (urlObject, range) {
         // Be sure that you haven't encoded any querystring along with the URL.
         var barePathname = decodeURIComponent(urlObject.pathname);
         var partsOfZIMUrl = regexpZIMUrlWithNamespace.exec(barePathname);
-        var prefix = partsOfZIMUrl[1];
-        var nameSpace = partsOfZIMUrl[2];
-        var title = partsOfZIMUrl[3];
-
+        var prefix = partsOfZIMUrl ? partsOfZIMUrl[1] : '';
+        var nameSpace = partsOfZIMUrl ? partsOfZIMUrl[2] : '';
+        var title = partsOfZIMUrl ? partsOfZIMUrl[3] : barePathname;
+        var anchorTarget = urlObject.hash.replace(/^#/, '');
+        var uriComponent = urlObject.search.replace(/\?kiwix-display/, '');
         var titleWithNameSpace = nameSpace + '/' + title;
+        var zimName = prefix.replace(/\/$/, '');
 
-        // Let's instantiate a new messageChannel, to allow app.js to give us the content
-        var messageChannel = new MessageChannel();
-        messageChannel.port1.onmessage = function (msgPortEvent) {
+        var messageListener = function (msgPortEvent) {
             if (msgPortEvent.data.action === 'giveContent') {
                 // Content received from app.js
                 var contentLength = msgPortEvent.data.content ? (msgPortEvent.data.content.byteLength || msgPortEvent.data.content.length) : null;
@@ -379,7 +389,7 @@ function fetchUrlFromZIM (urlObject, range) {
                     // HTTP status is usually 200, but has to bee 206 when partial content (range) is sent
                     status: range ? 206 : 200,
                     statusText: 'OK',
-                    headers
+                    headers: headers
                 };
 
                 var httpResponse = new Response(slicedData, responseInit);
@@ -392,10 +402,22 @@ function fetchUrlFromZIM (urlObject, range) {
                 reject(msgPortEvent.data, titleWithNameSpace);
             }
         };
-        outgoingMessagePort.postMessage({
-            action: 'askForContent',
-            title: titleWithNameSpace
-        }, [messageChannel.port2]);
+        // Get all the clients currently being controlled and send them a message
+        self.clients.matchAll().then(function (clientList) {
+            clientList.forEach(function (client) {
+                if (client.frameType !== 'top-level') return;
+                // Let's instantiate a new messageChannel, to allow app.js to give us the content
+                var messageChannel = new MessageChannel();
+                messageChannel.port1.onmessage = messageListener;
+                client.postMessage({
+                    action: 'askForContent',
+                    title: titleWithNameSpace,
+                    search: uriComponent,
+                    anchorTarget: anchorTarget,
+                    zimFileName: zimName
+                }, [messageChannel.port2]);
+            });
+        });
     });
 }
 
@@ -408,7 +430,7 @@ function fetchUrlFromZIM (urlObject, range) {
 function fromCache (cache, requestUrl) {
     // Prevents use of Cache API if user has disabled it
     if (!(useAppCache && cache === APP_CACHE || useAssetsCache && cache === ASSETS_CACHE)) {
-        return Promise.reject(new Error('disabled'));
+        return Promise.reject(new Error('Cache disabled'));
     }
     return caches.open(cache).then(function (cacheObj) {
         return cacheObj.match(requestUrl).then(function (matching) {
@@ -434,8 +456,9 @@ function updateCache (cache, request, response) {
         return Promise.resolve();
     }
     return caches.open(cache).then(function (cacheObj) {
-        console.debug('[SW] Adding ' + (request.url || request) + ' to ' + cache + '...');
-        return cacheObj.put(request, response);
+        var reqKey = request.url || request;
+        console.debug('[SW] Adding ' + reqKey + ' to ' + cache + '...');
+        return cacheObj.put(reqKey, response);
     });
 }
 

diff --git a/tests/e2e/runners/chrome/chrome60.bs.runner.js b/tests/e2e/runners/chrome/chrome60.bs.runner.js
@@ -37,7 +37,6 @@ async function loadChromeDriver () {
 // Maximize the window so that full browser state is visible in the screenshots
 // await driver_chrome.manage().window().maximize(); // Not supported in this version / Selenium
 
-console.log(' ');
 console.log('\x1b[33m%s\x1b[0m', 'Running Gutenberg tests only for this browser version');
 console.log(' ');
 

diff --git a/tests/e2e/runners/edge/ieMode.e2e.runner.js b/tests/e2e/runners/edge/ieMode.e2e.runner.js
@@ -16,5 +16,5 @@ async function loadIEModeDriver () {
     return driver;
 };
 
-await legacyRayCharles.runTests(await loadIEModeDriver());
+await legacyRayCharles.runTests(await loadIEModeDriver(), ['jquery']);
 await gutenbergRo.runTests(await loadIEModeDriver(), ['jquery']);
diff --git a/tests/e2e/runners/firefox/firefox70.bs.runner.js b/tests/e2e/runners/firefox/firefox70.bs.runner.js
@@ -33,7 +33,6 @@ async function loadFirefoxDriver () {
 const driver_gutenberg_fx = await loadFirefoxDriver();
 
 // Run test in SW mode only
-console.log(' ');
 console.log('\x1b[33m%s\x1b[0m', 'Running Gutenberg tests in ServiceWorker mode only for this browser version');
 console.log(' ');
 

diff --git a/tests/e2e/spec/gutenberg_ro.e2e.spec.js b/tests/e2e/spec/gutenberg_ro.e2e.spec.js
@@ -178,7 +178,7 @@ function runTests (driver, modes) {
             // Loads the ZIM archive for the mode if the mode is not skipped
             it('Load Modern zim file', async function () {
                 if (!serviceWorkerAPI) {
-                    console.log('\x1b[33m%s\x1b[0m', '    X Following test skipped:');
+                    console.log('\x1b[33m%s\x1b[0m', '    - Following test skipped:');
                     return;
                 }
                 // Wait until files have loaded
@@ -219,7 +219,7 @@ function runTests (driver, modes) {
 
             it('Sorting books by popularity', async function () {
                 if (isJqueryMode) {
-                    console.log('\x1b[33m%s\x1b[0m', '    X Following test skipped:');
+                    console.log('\x1b[33m%s\x1b[0m', '    - Following test skipped:');
                     return;
                 }
                 await driver.switchTo().frame('articleContent');
@@ -233,7 +233,7 @@ function runTests (driver, modes) {
 
             it('Sorting books by name', async function () {
                 if (isJqueryMode) {
-                    console.log('\x1b[33m%s\x1b[0m', '    X Following test skipped:');
+                    console.log('\x1b[33m%s\x1b[0m', '    - Following test skipped:');
                     return;
                 }
                 // We switch to default Content and back to Iframe because the If we are retrying the test
@@ -252,7 +252,7 @@ function runTests (driver, modes) {
 
             it('Change Language', async function () {
                 if (isJqueryMode) {
-                    console.log('\x1b[33m%s\x1b[0m', '    X Following test skipped:');
+                    console.log('\x1b[33m%s\x1b[0m', '    - Following test skipped:');
                     return;
                 }
                 // click on the language dropdown and select option French
@@ -325,7 +325,7 @@ function runTests (driver, modes) {
 
             it('Author search Autocomplete', async function () {
                 if (isJqueryMode) {
-                    console.log('\x1b[33m%s\x1b[0m', '    X Following test skipped:');
+                    console.log('\x1b[33m%s\x1b[0m', '    - Following test skipped:');
                     return;
                 }
                 const filter = await driver.wait(until.elementIsVisible(driver.findElement(By.id('author_filter'))), 1500);
@@ -336,7 +336,7 @@ function runTests (driver, modes) {
 
             it('Author search Results', async function () {
                 if (isJqueryMode) {
-                    console.log('\x1b[33m%s\x1b[0m', '    X Following test skipped:');
+                    console.log('\x1b[33m%s\x1b[0m', '    - Following test skipped:');
                     return;
                 }
                 // search by author name and press enter to apply the filter

diff --git a/tests/e2e/spec/legacy-ray_charles.e2e.spec.js b/tests/e2e/spec/legacy-ray_charles.e2e.spec.js
@@ -192,7 +192,7 @@ function runTests (driver, modes) {
 
             it('Load legacy Ray Charles and check index contains specified article', async function () {
                 if (!serviceWorkerAPI) {
-                    console.log('\x1b[33m%s\x1b[0m', '    X Following test skipped:');
+                    console.log('\x1b[33m%s\x1b[0m', '    - Following test skipped:');
                     return;
                 }
                 const archiveFiles = await driver.findElement(By.id('archiveFiles'));
@@ -221,7 +221,7 @@ function runTests (driver, modes) {
 
             it('Navigate to "This Little Girl of Mine"', async function () {
                 if (!serviceWorkerAPI) {
-                    console.log('\x1b[33m%s\x1b[0m', '    X Following test skipped:');
+                    console.log('\x1b[33m%s\x1b[0m', '    - Following test skipped:');
                     return;
                 }
                 // console.log('FilesLength outer: ' + filesLength);
@@ -259,7 +259,7 @@ function runTests (driver, modes) {
 
             it('Search for Ray Charles in title index and go to article', async function () {
                 if (!serviceWorkerAPI) {
-                    console.log('\x1b[33m%s\x1b[0m', '    X Following test skipped:');
+                    console.log('\x1b[33m%s\x1b[0m', '    - Following test skipped:');
                     return;
                 }
                 await driver.switchTo().defaultContent();