diff --git a/.eslintrc.cjs b/.eslintrc.cjs index e3c53898c..c00a10fef 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -19,6 +19,8 @@ module.exports = { 'no-unused-vars': 1, 'n/no-callback-literal': 0, 'object-shorthand': 0, - 'multiline-ternary': 0 + 'multiline-ternary': 0, + 'no-extend-native': 0, + 'no-global-assign': 0 } } diff --git a/nightwatch.js b/nightwatch.js index 56c1628b4..9d8035003 100644 --- a/nightwatch.js +++ b/nightwatch.js @@ -1,119 +1,123 @@ /** * nightwatch.js : Configuration of nightwatch. * Global settings of NightWatch. - * + * * Copyright 2017 Mossroy and contributors * License GPL v3: - * + * * This file is part of Kiwix. - * + * * Kiwix is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + * * Kiwix is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + * * You should have received a copy of the GNU General Public License * along with Kiwix (file LICENSE-GPLv3.txt). If not, see */ + 'use strict'; + +/* eslint-disable no-template-curly-in-string */ + const build = `${process.env.GITHUB_REPOSITORY} run #${process.env.GITHUB_RUN_ID}`; module.exports = { - "src_folders" : ["browser-tests"], - "output_folder" : "reports", - "custom_commands_path" : "", - "custom_assertions_path" : "", - "page_objects_path" : "", - "globals_path" : "", + src_folders: ['browser-tests'], + output_folder: 'reports', + custom_commands_path: '', + custom_assertions_path: '', + page_objects_path: '', + globals_path: '', - "test_settings" : { - "default" : { - "launch_url": "http://ondemand.saucelabs.com:80", - "selenium_port": 80, - "selenium_host": "ondemand.saucelabs.com", - "silent": true, - "username": "${SAUCE_USERNAME}", - "access_key": "${SAUCE_ACCESS_KEY}", - "screenshots" : { - "enabled" : false - }, - "globals": { - "waitForConditionTimeout": 600 - } - }, - "firefox52" : { - "desiredCapabilities": { - "browserName": "firefox", - "version": "52.0", - "javascriptEnabled": true, - "acceptSslCerts": true, - "build": build - } - }, - "firefox" : { - "desiredCapabilities": { - "browserName": "firefox", - "version": "latest", - "javascriptEnabled": true, - "acceptSslCerts": true, - "build": build - } - }, - "chrome58" : { - "desiredCapabilities": { - "browserName": "chrome", - "version": "58.0", - "javascriptEnabled": true, - "acceptSslCerts": true, - "build": build - } - }, - "chrome" : { - "desiredCapabilities": { - "browserName": "chrome", - "javascriptEnabled": true, - "acceptSslCerts": true, - "build": build - } - }, - "edge" : { - "desiredCapabilities": { - "browserName": "MicrosoftEdge", - "javascriptEnabled": true, - "acceptSslCerts": true, - "build": build - } - }, - "edge40" : { - "desiredCapabilities": { - "browserName": "MicrosoftEdge", - "version": "15.15063", - "javascriptEnabled": true, - "acceptSslCerts": true, - "build": build - } - }, - "edge44" : { - "desiredCapabilities": { - "browserName": "MicrosoftEdge", - "version": "18.17763", - "javascriptEnabled": true, - "acceptSslCerts": true, - "build": build - } - }, - "ie11" : { - "desiredCapabilities": { - "browserName": "internet explorer", - "javascriptEnabled": true, - "acceptSslCerts": true, - "build": build - } + test_settings: { + default: { + launch_url: 'http://ondemand.saucelabs.com:80', + selenium_port: 80, + selenium_host: 'ondemand.saucelabs.com', + silent: true, + username: '${SAUCE_USERNAME}', + access_key: '${SAUCE_ACCESS_KEY}', + screenshots: { + enabled: false + }, + globals: { + waitForConditionTimeout: 600 + } + }, + firefox52: { + desiredCapabilities: { + browserName: 'firefox', + version: '52.0', + javascriptEnabled: true, + acceptSslCerts: true, + build: build + } + }, + firefox: { + desiredCapabilities: { + browserName: 'firefox', + version: 'latest', + javascriptEnabled: true, + acceptSslCerts: true, + build: build + } + }, + chrome58: { + desiredCapabilities: { + browserName: 'chrome', + version: '58.0', + javascriptEnabled: true, + acceptSslCerts: true, + build: build + } + }, + chrome: { + desiredCapabilities: { + browserName: 'chrome', + javascriptEnabled: true, + acceptSslCerts: true, + build: build + } + }, + edge: { + desiredCapabilities: { + browserName: 'MicrosoftEdge', + javascriptEnabled: true, + acceptSslCerts: true, + build: build + } + }, + edge40: { + desiredCapabilities: { + browserName: 'MicrosoftEdge', + version: '15.15063', + javascriptEnabled: true, + acceptSslCerts: true, + build: build + } + }, + edge44: { + desiredCapabilities: { + browserName: 'MicrosoftEdge', + version: '18.17763', + javascriptEnabled: true, + acceptSslCerts: true, + build: build + } + }, + ie11: { + desiredCapabilities: { + browserName: 'internet explorer', + javascriptEnabled: true, + acceptSslCerts: true, + build: build + } + } } - } }; diff --git a/tests/init.js b/tests/init.js index 0cca17e04..cc3366bd1 100644 --- a/tests/init.js +++ b/tests/init.js @@ -1,27 +1,30 @@ /** * init.js : Configuration for the library require.js * This file handles the dependencies between javascript libraries, for the unit tests - * + * * Copyright 2013-2014 Mossroy and contributors * License GPL v3: - * + * * This file is part of Kiwix. - * + * * Kiwix is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + * * Kiwix is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + * * You should have received a copy of the GNU General Public License * along with Kiwix (file LICENSE-GPLv3.txt). If not, see */ + 'use strict'; +/* global requirejs */ + // Define global params needed for tests to run on existing app code var params = {}; var webpMachine = true; @@ -29,11 +32,11 @@ var webpMachine = true; require.config({ baseUrl: (window.__karma__ ? 'base/' : '') + 'www/js/lib/', paths: { - 'jquery': 'jquery-3.7.0.slim.min', - 'webpHeroBundle': 'webpHeroBundle_0.0.2' + jquery: 'jquery-3.7.0.slim.min', + webpHeroBundle: 'webpHeroBundle_0.0.2' }, shim: { - 'webpHeroBundle': '' + webpHeroBundle: '' } }); diff --git a/tests/karma.conf.local.js b/tests/karma.conf.local.js index ad4d27cb0..ab9ff59a6 100644 --- a/tests/karma.conf.local.js +++ b/tests/karma.conf.local.js @@ -1,36 +1,36 @@ module.exports = function (config) { - config.set({ - basePath: '../', - // https://karma-runner.github.io/5.2/config/browsers.html - browsers: [ - 'FirefoxHeadless', - // During local development, consider Chrome and Chromium to be similar enough - // and pick whichever the developer is most likely to have. - // In general, Linux distros provide and update Chromium only, - // whereas Windows and macOS users tend to have auto-updating Google Chrome. - // - // See package.json for commands to run tests in a single browser only. - // - // The CHROME_BIN check is to temporarily accomodate GitHub Actions - // which oddly uses Ubuntu but replaces the standard Chromium distribution - // with a custom install of Google Chrome. Being fixed in the next release: - // https://github.com/actions/virtual-environments/issues/2388 - process.platform === 'linux' && !process.env.CHROME_BIN ? 'ChromiumHeadless' : 'ChromeHeadless' - ], - frameworks: ['qunit'], - client: { - qunit: { - autostart: false - } - }, - // logLevel: 'DEBUG', - files: [ - 'www/js/lib/require.js', - 'tests/init.js', - { pattern: 'www/**/*', included: false }, - { pattern: 'tests/**/*', included: false } - ], - singleRun: true, - autoWatch: false - }); + config.set({ + basePath: '../', + // https://karma-runner.github.io/5.2/config/browsers.html + browsers: [ + 'FirefoxHeadless', + // During local development, consider Chrome and Chromium to be similar enough + // and pick whichever the developer is most likely to have. + // In general, Linux distros provide and update Chromium only, + // whereas Windows and macOS users tend to have auto-updating Google Chrome. + // + // See package.json for commands to run tests in a single browser only. + // + // The CHROME_BIN check is to temporarily accomodate GitHub Actions + // which oddly uses Ubuntu but replaces the standard Chromium distribution + // with a custom install of Google Chrome. Being fixed in the next release: + // https://github.com/actions/virtual-environments/issues/2388 + process.platform === 'linux' && !process.env.CHROME_BIN ? 'ChromiumHeadless' : 'ChromeHeadless' + ], + frameworks: ['qunit'], + client: { + qunit: { + autostart: false + } + }, + // logLevel: 'DEBUG', + files: [ + 'www/js/lib/require.js', + 'tests/init.js', + { pattern: 'www/**/*', included: false }, + { pattern: 'tests/**/*', included: false } + ], + singleRun: true, + autoWatch: false + }); }; diff --git a/tests/karma.conf.saucelabs.js b/tests/karma.conf.saucelabs.js index 6e25ce66c..e87df588c 100644 --- a/tests/karma.conf.saucelabs.js +++ b/tests/karma.conf.saucelabs.js @@ -1,85 +1,85 @@ module.exports = function (config) { - config.set({ - basePath: '../', - // https://karma-runner.github.io/5.2/config/browsers.html - // https://github.com/karma-runner/karma-sauce-launcher - sauceLabs: { - build: process.env.GITHUB_RUN_ID ? `${process.env.GITHUB_REPOSITORY} run #${process.env.GITHUB_RUN_ID}` : null, - startConnect: true, - username: process.env.SAUCE_USERNAME, - accessKey: process.env.SAUCE_ACCESS_KEY - }, - customLaunchers: { - firefox52: { - base: 'SauceLabs', - browserName: 'firefox', - version: '52.0' - }, - firefox: { - base: 'SauceLabs', - browserName: 'firefox', - version: 'latest' - }, - chrome58: { - base: 'SauceLabs', - browserName: 'chrome', - version: '58.0' - }, - chrome: { - base: 'SauceLabs', - browserName: 'chrome', - version: 'latest' - }, - edge: { - base: 'SauceLabs', - browserName: 'MicrosoftEdge', - version: 'latest' - }, - edge40: { - base: 'SauceLabs', - browserName: 'MicrosoftEdge', - version: '15.15063' - }, - edge44: { - base: 'SauceLabs', - browserName: 'MicrosoftEdge', - version: '18.17763' - }, - ie11: { - base: 'SauceLabs', - browserName: 'internet explorer', - version: 'latest' - } - }, - // The free account on Sauce does not allow more than 5 concurrent sessions - concurrency: 4, + config.set({ + basePath: '../', + // https://karma-runner.github.io/5.2/config/browsers.html + // https://github.com/karma-runner/karma-sauce-launcher + sauceLabs: { + build: process.env.GITHUB_RUN_ID ? `${process.env.GITHUB_REPOSITORY} run #${process.env.GITHUB_RUN_ID}` : null, + startConnect: true, + username: process.env.SAUCE_USERNAME, + accessKey: process.env.SAUCE_ACCESS_KEY + }, + customLaunchers: { + firefox52: { + base: 'SauceLabs', + browserName: 'firefox', + version: '52.0' + }, + firefox: { + base: 'SauceLabs', + browserName: 'firefox', + version: 'latest' + }, + chrome58: { + base: 'SauceLabs', + browserName: 'chrome', + version: '58.0' + }, + chrome: { + base: 'SauceLabs', + browserName: 'chrome', + version: 'latest' + }, + edge: { + base: 'SauceLabs', + browserName: 'MicrosoftEdge', + version: 'latest' + }, + edge40: { + base: 'SauceLabs', + browserName: 'MicrosoftEdge', + version: '15.15063' + }, + edge44: { + base: 'SauceLabs', + browserName: 'MicrosoftEdge', + version: '18.17763' + }, + ie11: { + base: 'SauceLabs', + browserName: 'internet explorer', + version: 'latest' + } + }, + // The free account on Sauce does not allow more than 5 concurrent sessions + concurrency: 4, - // REMINDER: Keep this list in sync with the UI tests, in .github/workflows/CI.yml. - browsers: [ - // 'firefox', // Disable latest Firefox due to #894 - 'chrome', - 'edge', - 'edge40', - 'edge44', - 'firefox52', - 'chrome58' - // 'ie11' // Disable IE11 due to Promise error - ], - frameworks: ['qunit'], - client: { - qunit: { - autostart: false - } - }, - reporters: ['dots'], - logLevel: 'WARN', - files: [ - 'www/js/lib/require.js', - 'tests/init.js', - { pattern: 'www/**/*', included: false }, - { pattern: 'tests/**/*', included: false } - ], - singleRun: true, - autoWatch: false - }); + // REMINDER: Keep this list in sync with the UI tests, in .github/workflows/CI.yml. + browsers: [ + // 'firefox', // Disable latest Firefox due to #894 + 'chrome', + 'edge', + 'edge40', + 'edge44', + 'firefox52', + 'chrome58' + // 'ie11' // Disable IE11 due to Promise error + ], + frameworks: ['qunit'], + client: { + qunit: { + autostart: false + } + }, + reporters: ['dots'], + logLevel: 'WARN', + files: [ + 'www/js/lib/require.js', + 'tests/init.js', + { pattern: 'www/**/*', included: false }, + { pattern: 'tests/**/*', included: false } + ], + singleRun: true, + autoWatch: false + }); }; diff --git a/tests/tests.js b/tests/tests.js index fca69ab90..bd491652c 100644 --- a/tests/tests.js +++ b/tests/tests.js @@ -19,374 +19,373 @@ * You should have received a copy of the GNU General Public License * along with Kiwix (file LICENSE-GPLv3.txt). If not, see */ -define(['jquery', 'zimArchive', 'zimDirEntry', 'util', 'uiUtil', 'utf8'], - function($, zimArchive, zimDirEntry, util, uiUtil, utf8) { - var localZimArchive; +/* global define, require, QUnit, Promise */ - /** - * Make an HTTP request for a Blob and return a Promise - * - * @param {String} url URL to download from - * @param {String} name Name to give to the Blob instance - * @returns {Promise} A Promise for the Blob - */ - function makeBlobRequest(url, name) { - return new Promise(function (resolve, reject) { - var xhr = new XMLHttpRequest(); - xhr.open('GET', url); - xhr.onreadystatechange = function () { - if (xhr.readyState === XMLHttpRequest.DONE) { - if ((xhr.status >= 200 && xhr.status < 300) || xhr.status === 0 ) { - var blob = new Blob([xhr.response], {type: 'application/octet-stream'}); - blob.name = name; - resolve(blob); - } - else { - console.error("Error reading file " + url + " status:" + xhr.status + ", statusText:" + xhr.statusText); - reject({ - status: xhr.status, - statusText: xhr.statusText - }); +define(['jquery', 'zimArchive', 'zimDirEntry', 'util', 'uiUtil', 'utf8'], + function ($, zimArchive, zimDirEntry, util, uiUtil, utf8) { + var localZimArchive; + + /** + * Make an HTTP request for a Blob and return a Promise + * + * @param {String} url URL to download from + * @param {String} name Name to give to the Blob instance + * @returns {Promise} A Promise for the Blob + */ + function makeBlobRequest (url, name) { + return new Promise(function (resolve, reject) { + var xhr = new XMLHttpRequest(); + xhr.open('GET', url); + xhr.onreadystatechange = function () { + if (xhr.readyState === XMLHttpRequest.DONE) { + if ((xhr.status >= 200 && xhr.status < 300) || xhr.status === 0) { + var blob = new Blob([xhr.response], { type: 'application/octet-stream' }); + blob.name = name; + resolve(blob); + } else { + console.error('Error reading file ' + url + ' status:' + xhr.status + ', statusText:' + xhr.statusText); + reject(new Error({ + status: xhr.status, + statusText: xhr.statusText + })); + } } - } - }; - xhr.onerror = function () { - console.error("Error reading file " + url + " status:" + xhr.status + ", statusText:" + xhr.statusText); - reject({ - status: xhr.status, - statusText: xhr.statusText - }); - }; - xhr.responseType = 'blob'; - xhr.send(); - }); - } + }; + xhr.onerror = function () { + console.error('Error reading file ' + url + ' status:' + xhr.status + ', statusText:' + xhr.statusText); + reject(new Error({ + status: xhr.status, + statusText: xhr.statusText + })); + }; + xhr.responseType = 'blob'; + xhr.send(); + }); + } - // Let's try to download the ZIM files - var zimArchiveFiles = new Array(); + // Let's try to download the ZIM files + var zimArchiveFiles = []; - var splitBlobs = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'].map(function(c) { - var filename = 'wikipedia_en_ray_charles_2015-06.zima' + c; - return makeBlobRequest(require.toUrl('../../../tests/' + filename), filename); - }); - Promise.all(splitBlobs) - .then(function(values) { - zimArchiveFiles = values; - }).then(function() { - // Create a localZimArchive from selected files, in order to run the following tests - localZimArchive = new zimArchive.ZIMArchive(zimArchiveFiles, null, function (zimArchive) { - runTests(); + var splitBlobs = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'].map(function (c) { + var filename = 'wikipedia_en_ray_charles_2015-06.zima' + c; + return makeBlobRequest(require.toUrl('../../../tests/' + filename), filename); }); - }); - - var runTests = function() { + Promise.all(splitBlobs) + .then(function (values) { + zimArchiveFiles = values; + }).then(function () { + // Create a localZimArchive from selected files, in order to run the following tests + localZimArchive = new zimArchive.ZIMArchive(zimArchiveFiles, null, function (zimArchive) { + runTests(); + }); + }); - QUnit.module("environment"); - QUnit.test("qunit test", function(assert) { - assert.equal("test", "test", "QUnit is properly configured"); - }); + var runTests = function () { + QUnit.module('environment'); + QUnit.test('qunit test', function (assert) { + assert.equal('test', 'test', 'QUnit is properly configured'); + }); - QUnit.test("check archive files are read", function(assert) { - assert.ok(zimArchiveFiles && zimArchiveFiles[0] && zimArchiveFiles[0].size > 0, "ZIM file read and not empty"); - }); + QUnit.test('check archive files are read', function (assert) { + assert.ok(zimArchiveFiles && zimArchiveFiles[0] && zimArchiveFiles[0].size > 0, 'ZIM file read and not empty'); + }); - QUnit.module("utils"); - QUnit.test("check reading an IEEE_754 float from 4 bytes" ,function(assert) { - var byteArray = new Uint8Array(4); - // This example is taken from https://fr.wikipedia.org/wiki/IEEE_754#Un_exemple_plus_complexe - // 1100 0010 1110 1101 0100 0000 0000 0000 - byteArray[0] = 194; - byteArray[1] = 237; - byteArray[2] = 64; - byteArray[3] = 0; - var float = util.readFloatFrom4Bytes(byteArray, 0); - assert.equal(float, -118.625, "the IEEE_754 float should be converted as -118.625"); - }); - QUnit.test("check upper/lower case variations", function (assert) { - var testString1 = "téléphone"; - var testString2 = "Paris"; - var testString3 = "le Couvre-chef Est sur le porte-manteaux"; - var testString4 = "épée"; - var testString5 = '$￥€“«xριστός» †¡Ἀνέστη!”'; - var testString6 = "Καλά Νερά Μαγνησία žižek"; - assert.equal(util.allCaseFirstLetters(testString1).indexOf("Téléphone") >= 0, true, "The first letter should be uppercase"); - assert.equal(util.allCaseFirstLetters(testString2).indexOf("paris") >= 0, true, "The first letter should be lowercase"); - assert.equal(util.allCaseFirstLetters(testString3).indexOf("Le Couvre-Chef Est Sur Le Porte-Manteaux") >= 0, true, "The first letter of every word should be uppercase"); - assert.equal(util.allCaseFirstLetters(testString4).indexOf("Épée") >= 0, true, "The first letter should be uppercase (with accent)"); - assert.equal(util.allCaseFirstLetters(testString5).indexOf('$￥€“«Xριστός» †¡ἀνέστη!”') >= 0, true, "First non-punctuation/non-currency Unicode letter should be uppercase, second (with breath mark) lowercase"); - assert.equal(util.allCaseFirstLetters(testString6, "full").indexOf("ΚΑΛΆ ΝΕΡΆ ΜΑΓΝΗΣΊΑ ŽIŽEK") >= 0, true, "All Unicode letters should be uppercase"); - }); - QUnit.test("check removal of parameters in URL", function(assert) { - var baseUrl = "A/Che cosa è l'amore?.html"; - var testUrls = [ - "A/Che%20cosa%20%C3%A8%20l'amore%3F.html?param1=toto¶m2=titi", - "A/Che%20cosa%20%C3%A8%20l'amore%3F.html?param1=toto¶m2=titi#anchor", - "A/Che%20cosa%20%C3%A8%20l'amore%3F.html#anchor" - ]; - testUrls.forEach(function (testUrl) { - assert.equal(decodeURIComponent( - uiUtil.removeUrlParameters(testUrl) - ), baseUrl); + QUnit.module('utils'); + QUnit.test('check reading an IEEE_754 float from 4 bytes', function (assert) { + var byteArray = new Uint8Array(4); + // This example is taken from https://fr.wikipedia.org/wiki/IEEE_754#Un_exemple_plus_complexe + // 1100 0010 1110 1101 0100 0000 0000 0000 + byteArray[0] = 194; + byteArray[1] = 237; + byteArray[2] = 64; + byteArray[3] = 0; + var float = util.readFloatFrom4Bytes(byteArray, 0); + assert.equal(float, -118.625, 'the IEEE_754 float should be converted as -118.625'); + }); + QUnit.test('check upper/lower case variations', function (assert) { + var testString1 = 'téléphone'; + var testString2 = 'Paris'; + var testString3 = 'le Couvre-chef Est sur le porte-manteaux'; + var testString4 = 'épée'; + var testString5 = '$￥€“«xριστός» †¡Ἀνέστη!”'; + var testString6 = 'Καλά Νερά Μαγνησία žižek'; + assert.equal(util.allCaseFirstLetters(testString1).indexOf('Téléphone') >= 0, true, 'The first letter should be uppercase'); + assert.equal(util.allCaseFirstLetters(testString2).indexOf('paris') >= 0, true, 'The first letter should be lowercase'); + assert.equal(util.allCaseFirstLetters(testString3).indexOf('Le Couvre-Chef Est Sur Le Porte-Manteaux') >= 0, true, 'The first letter of every word should be uppercase'); + assert.equal(util.allCaseFirstLetters(testString4).indexOf('Épée') >= 0, true, 'The first letter should be uppercase (with accent)'); + assert.equal(util.allCaseFirstLetters(testString5).indexOf('$￥€“«Xριστός» †¡ἀνέστη!”') >= 0, true, 'First non-punctuation/non-currency Unicode letter should be uppercase, second (with breath mark) lowercase'); + assert.equal(util.allCaseFirstLetters(testString6, 'full').indexOf('ΚΑΛΆ ΝΕΡΆ ΜΑΓΝΗΣΊΑ ŽIŽEK') >= 0, true, 'All Unicode letters should be uppercase'); + }); + QUnit.test('check removal of parameters in URL', function (assert) { + var baseUrl = "A/Che cosa è l'amore?.html"; + var testUrls = [ + "A/Che%20cosa%20%C3%A8%20l'amore%3F.html?param1=toto¶m2=titi", + "A/Che%20cosa%20%C3%A8%20l'amore%3F.html?param1=toto¶m2=titi#anchor", + "A/Che%20cosa%20%C3%A8%20l'amore%3F.html#anchor" + ]; + testUrls.forEach(function (testUrl) { + assert.equal(decodeURIComponent( + uiUtil.removeUrlParameters(testUrl) + ), baseUrl); + }); }); - }); - QUnit.module("ZIM initialisation"); - QUnit.test("ZIM archive is ready", function(assert) { - assert.ok(localZimArchive.isReady() === true, "ZIM archive should be set as ready"); - }); + QUnit.module('ZIM initialisation'); + QUnit.test('ZIM archive is ready', function (assert) { + assert.ok(localZimArchive.isReady() === true, 'ZIM archive should be set as ready'); + }); - QUnit.module("ZIM metadata"); - QUnit.test("read ZIM language", function(assert) { - var done = assert.async(); - assert.expect(1); - var callbackFunction = function(language) { - assert.equal(language , 'eng', 'The language read inside the Metadata should be "eng" for "English"'); - done(); - }; - localZimArchive.getMetadata("Language", callbackFunction); - }); - QUnit.test("try to read a missing metadata", function(assert) { - var done = assert.async(); - assert.expect(1); - var callbackFunction = function(string) { - assert.equal(string, undefined, 'The metadata zzz should not be found inside the ZIM'); - done(); - }; - localZimArchive.getMetadata("zzz", callbackFunction); - }); + QUnit.module('ZIM metadata'); + QUnit.test('read ZIM language', function (assert) { + var done = assert.async(); + assert.expect(1); + var callbackFunction = function (language) { + assert.equal(language, 'eng', 'The language read inside the Metadata should be "eng" for "English"'); + done(); + }; + localZimArchive.getMetadata('Language', callbackFunction); + }); + QUnit.test('try to read a missing metadata', function (assert) { + var done = assert.async(); + assert.expect(1); + var callbackFunction = function (string) { + assert.equal(string, undefined, 'The metadata zzz should not be found inside the ZIM'); + done(); + }; + localZimArchive.getMetadata('zzz', callbackFunction); + }); - QUnit.module("zim_direntry_search_and_read"); - QUnit.test("check DirEntry.fromStringId 'A Fool for You'", function(assert) { - var done = assert.async(); - var aFoolForYouDirEntry = zimDirEntry.DirEntry.fromStringId(localZimArchive._file, "5856|7|A|0|2|A_Fool_for_You.html|A Fool for You|false|undefined"); + QUnit.module('zim_direntry_search_and_read'); + QUnit.test("check DirEntry.fromStringId 'A Fool for You'", function (assert) { + var done = assert.async(); + var aFoolForYouDirEntry = zimDirEntry.DirEntry.fromStringId(localZimArchive._file, '5856|7|A|0|2|A_Fool_for_You.html|A Fool for You|false|undefined'); - assert.expect(2); - var callbackFunction = function(dirEntry, htmlArticle) { - assert.ok(htmlArticle && htmlArticle.length > 0, "Article not empty"); - // Remove new lines - htmlArticle = htmlArticle.replace(/[\r\n]/g, " "); - assert.ok(htmlArticle.match("^.*]*>A Fool for You"), "'A Fool for You' title somewhere in the article"); - done(); - }; - localZimArchive.readUtf8File(aFoolForYouDirEntry, callbackFunction); - }); - QUnit.test("check findDirEntriesWithPrefix 'A'", function(assert) { - var done = assert.async(); - assert.expect(2); - var callbackFunction = function(dirEntryList) { - assert.ok(dirEntryList && dirEntryList.length === 5, "Article list with 5 results"); - var firstDirEntry = dirEntryList[0]; - assert.equal(firstDirEntry.getTitleOrUrl() , 'A Fool for You', 'First result should be "A Fool for You"'); - done(); - }; - localZimArchive.findDirEntriesWithPrefix({prefix: 'A', size: 5}, callbackFunction, true); - }); - QUnit.test("check findDirEntriesWithPrefix 'a'", function(assert) { - var done = assert.async(); - assert.expect(2); - var callbackFunction = function(dirEntryList) { - assert.ok(dirEntryList && dirEntryList.length === 5, "Article list with 5 results"); - var firstDirEntry = dirEntryList[0]; - assert.equal(firstDirEntry.getTitleOrUrl() , 'A Fool for You', 'First result should be "A Fool for You"'); - done(); - }; - localZimArchive.findDirEntriesWithPrefix({prefix: 'a', size: 5}, callbackFunction, true); - }); - QUnit.test("check findDirEntriesWithPrefix 'blues brothers'", function(assert) { - var done = assert.async(); - assert.expect(2); - var callbackFunction = function(dirEntryList) { - assert.ok(dirEntryList && dirEntryList.length === 3, "Article list with 3 result"); - var firstDirEntry = dirEntryList[0]; - assert.equal(firstDirEntry.getTitleOrUrl() , 'Blues Brothers (film)', 'First result should be "Blues Brothers (film)"'); - done(); - }; - localZimArchive.findDirEntriesWithPrefix({prefix: 'blues brothers', size: 5}, callbackFunction, true); - }); - QUnit.test("article '(The Night Time Is) The Right Time' correctly redirects to 'Night Time Is the Right Time'", function(assert) { - var done = assert.async(); - assert.expect(6); - localZimArchive.getDirEntryByPath("A/(The_Night_Time_Is)_The_Right_Time.html").then(function(dirEntry) { - assert.ok(dirEntry !== null, "DirEntry found"); - if (dirEntry !== null) { - assert.ok(dirEntry.isRedirect(), "DirEntry is a redirect."); - assert.equal(dirEntry.getTitleOrUrl(), "(The Night Time Is) The Right Time", "Correct redirect title name."); - localZimArchive.resolveRedirect(dirEntry, function(dirEntry) { - assert.ok(dirEntry !== null, "DirEntry found"); - assert.ok(!dirEntry.isRedirect(), "DirEntry is not a redirect."); - assert.equal(dirEntry.getTitleOrUrl(), "Night Time Is the Right Time", "Correct redirected title name."); - done(); - }); - } else { + assert.expect(2); + var callbackFunction = function (dirEntry, htmlArticle) { + assert.ok(htmlArticle && htmlArticle.length > 0, 'Article not empty'); + // Remove new lines + htmlArticle = htmlArticle.replace(/[\r\n]/g, ' '); + assert.ok(htmlArticle.match('^.*]*>A Fool for You'), "'A Fool for You' title somewhere in the article"); done(); - } + }; + localZimArchive.readUtf8File(aFoolForYouDirEntry, callbackFunction); }); - }); - QUnit.test("article 'Raelettes' correctly redirects to 'The Raelettes'", function(assert) { - var done = assert.async(); - assert.expect(6); - localZimArchive.getDirEntryByPath("A/Raelettes.html").then(function(dirEntry) { - assert.ok(dirEntry !== null, "DirEntry found"); - if (dirEntry !== null) { - assert.ok(dirEntry.isRedirect(), "DirEntry is a redirect."); - assert.equal(dirEntry.getTitleOrUrl(), "Raelettes", "Correct redirect title name."); - localZimArchive.resolveRedirect(dirEntry, function(dirEntry) { - assert.ok(dirEntry !== null, "DirEntry found"); - assert.ok(!dirEntry.isRedirect(), "DirEntry is not a redirect."); - assert.equal(dirEntry.getTitleOrUrl(), "The Raelettes", "Correct redirected title name."); - done(); - }); - } else { + QUnit.test("check findDirEntriesWithPrefix 'A'", function (assert) { + var done = assert.async(); + assert.expect(2); + var callbackFunction = function (dirEntryList) { + assert.ok(dirEntryList && dirEntryList.length === 5, 'Article list with 5 results'); + var firstDirEntry = dirEntryList[0]; + assert.equal(firstDirEntry.getTitleOrUrl(), 'A Fool for You', 'First result should be "A Fool for You"'); done(); - } + }; + localZimArchive.findDirEntriesWithPrefix({ prefix: 'A', size: 5 }, callbackFunction, true); }); - }); - QUnit.test("article 'Bein Green' correctly redirects to 'Bein' Green", function(assert) { - var done = assert.async(); - assert.expect(6); - localZimArchive.getDirEntryByPath("A/Bein_Green.html").then(function(dirEntry) { - assert.ok(dirEntry !== null, "DirEntry found"); - if (dirEntry !== null) { - assert.ok(dirEntry.isRedirect(), "DirEntry is a redirect."); - assert.equal(dirEntry.getTitleOrUrl(), "Bein Green", "Correct redirect title name."); - localZimArchive.resolveRedirect(dirEntry, function(dirEntry) { - assert.ok(dirEntry !== null, "DirEntry found"); - assert.ok(!dirEntry.isRedirect(), "DirEntry is not a redirect."); - assert.equal(dirEntry.getTitleOrUrl(), "Bein' Green", "Correct redirected title name."); - done(); - }); - } else { + QUnit.test("check findDirEntriesWithPrefix 'a'", function (assert) { + var done = assert.async(); + assert.expect(2); + var callbackFunction = function (dirEntryList) { + assert.ok(dirEntryList && dirEntryList.length === 5, 'Article list with 5 results'); + var firstDirEntry = dirEntryList[0]; + assert.equal(firstDirEntry.getTitleOrUrl(), 'A Fool for You', 'First result should be "A Fool for You"'); done(); - } + }; + localZimArchive.findDirEntriesWithPrefix({ prefix: 'a', size: 5 }, callbackFunction, true); }); - }); - QUnit.test("article 'America, the Beautiful' correctly redirects to 'America the Beautiful'", function(assert) { - var done = assert.async(); - assert.expect(6); - localZimArchive.getDirEntryByPath("A/America,_the_Beautiful.html").then(function(dirEntry) { - assert.ok(dirEntry !== null, "DirEntry found"); - if (dirEntry !== null) { - assert.ok(dirEntry.isRedirect(), "DirEntry is a redirect."); - assert.equal(dirEntry.getTitleOrUrl(), "America, the Beautiful", "Correct redirect title name."); - localZimArchive.resolveRedirect(dirEntry, function(dirEntry) { - assert.ok(dirEntry !== null, "DirEntry found"); - assert.ok(!dirEntry.isRedirect(), "DirEntry is not a redirect."); - assert.equal(dirEntry.getTitleOrUrl(), "America the Beautiful", "Correct redirected title name."); - done(); - }); - } else { + QUnit.test("check findDirEntriesWithPrefix 'blues brothers'", function (assert) { + var done = assert.async(); + assert.expect(2); + var callbackFunction = function (dirEntryList) { + assert.ok(dirEntryList && dirEntryList.length === 3, 'Article list with 3 result'); + var firstDirEntry = dirEntryList[0]; + assert.equal(firstDirEntry.getTitleOrUrl(), 'Blues Brothers (film)', 'First result should be "Blues Brothers (film)"'); done(); - } + }; + localZimArchive.findDirEntriesWithPrefix({ prefix: 'blues brothers', size: 5 }, callbackFunction, true); }); - }); - QUnit.test("Image 'm/RayCharles_AManAndHisSoul.jpg' can be loaded", function(assert) { - var done = assert.async(); - assert.expect(5); - localZimArchive.getDirEntryByPath("I/m/RayCharles_AManAndHisSoul.jpg").then(function(dirEntry) { - assert.ok(dirEntry !== null, "DirEntry found"); - if (dirEntry !== null) { - assert.equal(dirEntry.namespace +"/"+ dirEntry.url, "I/m/RayCharles_AManAndHisSoul.jpg", "URL is correct."); - assert.equal(dirEntry.getMimetype(), "image/jpeg", "MIME type is correct."); - localZimArchive.readBinaryFile(dirEntry, function(title, data) { - assert.equal(data.length, 4951, "Data length is correct."); - var beginning = new Uint8Array([255, 216, 255, 224, 0, 16, 74, 70, - 73, 70, 0, 1, 1, 0, 0, 1]); - assert.equal(data.slice(0, beginning.length).toString(), beginning.toString(), "Data beginning is correct."); + QUnit.test("article '(The Night Time Is) The Right Time' correctly redirects to 'Night Time Is the Right Time'", function (assert) { + var done = assert.async(); + assert.expect(6); + localZimArchive.getDirEntryByPath('A/(The_Night_Time_Is)_The_Right_Time.html').then(function (dirEntry) { + assert.ok(dirEntry !== null, 'DirEntry found'); + if (dirEntry !== null) { + assert.ok(dirEntry.isRedirect(), 'DirEntry is a redirect.'); + assert.equal(dirEntry.getTitleOrUrl(), '(The Night Time Is) The Right Time', 'Correct redirect title name.'); + localZimArchive.resolveRedirect(dirEntry, function (dirEntry) { + assert.ok(dirEntry !== null, 'DirEntry found'); + assert.ok(!dirEntry.isRedirect(), 'DirEntry is not a redirect.'); + assert.equal(dirEntry.getTitleOrUrl(), 'Night Time Is the Right Time', 'Correct redirected title name.'); + done(); + }); + } else { done(); - }); - } else { - done(); - } + } + }); }); - }); - QUnit.test("Stylesheet '-/s/style.css' can be loaded", function(assert) { - var done = assert.async(); + QUnit.test("article 'Raelettes' correctly redirects to 'The Raelettes'", function (assert) { + var done = assert.async(); + assert.expect(6); + localZimArchive.getDirEntryByPath('A/Raelettes.html').then(function (dirEntry) { + assert.ok(dirEntry !== null, 'DirEntry found'); + if (dirEntry !== null) { + assert.ok(dirEntry.isRedirect(), 'DirEntry is a redirect.'); + assert.equal(dirEntry.getTitleOrUrl(), 'Raelettes', 'Correct redirect title name.'); + localZimArchive.resolveRedirect(dirEntry, function (dirEntry) { + assert.ok(dirEntry !== null, 'DirEntry found'); + assert.ok(!dirEntry.isRedirect(), 'DirEntry is not a redirect.'); + assert.equal(dirEntry.getTitleOrUrl(), 'The Raelettes', 'Correct redirected title name.'); + done(); + }); + } else { + done(); + } + }); + }); + QUnit.test("article 'Bein Green' correctly redirects to 'Bein' Green", function (assert) { + var done = assert.async(); + assert.expect(6); + localZimArchive.getDirEntryByPath('A/Bein_Green.html').then(function (dirEntry) { + assert.ok(dirEntry !== null, 'DirEntry found'); + if (dirEntry !== null) { + assert.ok(dirEntry.isRedirect(), 'DirEntry is a redirect.'); + assert.equal(dirEntry.getTitleOrUrl(), 'Bein Green', 'Correct redirect title name.'); + localZimArchive.resolveRedirect(dirEntry, function (dirEntry) { + assert.ok(dirEntry !== null, 'DirEntry found'); + assert.ok(!dirEntry.isRedirect(), 'DirEntry is not a redirect.'); + assert.equal(dirEntry.getTitleOrUrl(), "Bein' Green", 'Correct redirected title name.'); + done(); + }); + } else { + done(); + } + }); + }); + QUnit.test("article 'America, the Beautiful' correctly redirects to 'America the Beautiful'", function (assert) { + var done = assert.async(); + assert.expect(6); + localZimArchive.getDirEntryByPath('A/America,_the_Beautiful.html').then(function (dirEntry) { + assert.ok(dirEntry !== null, 'DirEntry found'); + if (dirEntry !== null) { + assert.ok(dirEntry.isRedirect(), 'DirEntry is a redirect.'); + assert.equal(dirEntry.getTitleOrUrl(), 'America, the Beautiful', 'Correct redirect title name.'); + localZimArchive.resolveRedirect(dirEntry, function (dirEntry) { + assert.ok(dirEntry !== null, 'DirEntry found'); + assert.ok(!dirEntry.isRedirect(), 'DirEntry is not a redirect.'); + assert.equal(dirEntry.getTitleOrUrl(), 'America the Beautiful', 'Correct redirected title name.'); + done(); + }); + } else { + done(); + } + }); + }); + QUnit.test("Image 'm/RayCharles_AManAndHisSoul.jpg' can be loaded", function (assert) { + var done = assert.async(); + assert.expect(5); + localZimArchive.getDirEntryByPath('I/m/RayCharles_AManAndHisSoul.jpg').then(function (dirEntry) { + assert.ok(dirEntry !== null, 'DirEntry found'); + if (dirEntry !== null) { + assert.equal(dirEntry.namespace + '/' + dirEntry.url, 'I/m/RayCharles_AManAndHisSoul.jpg', 'URL is correct.'); + assert.equal(dirEntry.getMimetype(), 'image/jpeg', 'MIME type is correct.'); + localZimArchive.readBinaryFile(dirEntry, function (title, data) { + assert.equal(data.length, 4951, 'Data length is correct.'); + var beginning = new Uint8Array([255, 216, 255, 224, 0, 16, 74, 70, + 73, 70, 0, 1, 1, 0, 0, 1]); + assert.equal(data.slice(0, beginning.length).toString(), beginning.toString(), 'Data beginning is correct.'); + done(); + }); + } else { + done(); + } + }); + }); + QUnit.test("Stylesheet '-/s/style.css' can be loaded", function (assert) { + var done = assert.async(); - assert.expect(5); - localZimArchive.getDirEntryByPath("-/s/style.css").then(function(dirEntry) { - assert.ok(dirEntry !== null, "DirEntry found"); - if (dirEntry !== null) { - assert.equal(dirEntry.namespace +"/"+ dirEntry.url, "-/s/style.css", "URL is correct."); - assert.equal(dirEntry.getMimetype(), "text/css", "MIME type is correct."); - localZimArchive.readBinaryFile(dirEntry, function(dirEntry, data) { - assert.equal(data.length, 104495, "Data length is correct."); - data = utf8.parse(data); - var beginning = "\n/* start http://en.wikipedia.org/w/load.php?debug=false&lang=en&modules=site&only=styles&skin=vector"; - assert.equal(data.slice(0, beginning.length), beginning, "Content starts correctly."); + assert.expect(5); + localZimArchive.getDirEntryByPath('-/s/style.css').then(function (dirEntry) { + assert.ok(dirEntry !== null, 'DirEntry found'); + if (dirEntry !== null) { + assert.equal(dirEntry.namespace + '/' + dirEntry.url, '-/s/style.css', 'URL is correct.'); + assert.equal(dirEntry.getMimetype(), 'text/css', 'MIME type is correct.'); + localZimArchive.readBinaryFile(dirEntry, function (dirEntry, data) { + assert.equal(data.length, 104495, 'Data length is correct.'); + data = utf8.parse(data); + var beginning = '\n/* start http://en.wikipedia.org/w/load.php?debug=false&lang=en&modules=site&only=styles&skin=vector'; + assert.equal(data.slice(0, beginning.length), beginning, 'Content starts correctly.'); + done(); + }); + } else { done(); - }); - } else { - done(); - } + } + }); }); - }); - QUnit.test("Javascript '-/j/local.js' can be loaded", function(assert) { - var done = assert.async(); - assert.expect(5); - localZimArchive.getDirEntryByPath("-/j/local.js").then(function(dirEntry) { - assert.ok(dirEntry !== null, "DirEntry found"); - if (dirEntry !== null) { - assert.equal(dirEntry.namespace +"/"+ dirEntry.url, "-/j/local.js", "URL is correct."); - assert.equal(dirEntry.getMimetype(), "application/javascript", "MIME type is correct."); - localZimArchive.readBinaryFile(dirEntry, function(dirEntry, data) { - assert.equal(data.length, 41, "Data length is correct."); - data = utf8.parse(data); - var beginning = "console.log( \"mw.loader"; - assert.equal(data.slice(0, beginning.length), beginning, "Content starts correctly."); + QUnit.test("Javascript '-/j/local.js' can be loaded", function (assert) { + var done = assert.async(); + assert.expect(5); + localZimArchive.getDirEntryByPath('-/j/local.js').then(function (dirEntry) { + assert.ok(dirEntry !== null, 'DirEntry found'); + if (dirEntry !== null) { + assert.equal(dirEntry.namespace + '/' + dirEntry.url, '-/j/local.js', 'URL is correct.'); + assert.equal(dirEntry.getMimetype(), 'application/javascript', 'MIME type is correct.'); + localZimArchive.readBinaryFile(dirEntry, function (dirEntry, data) { + assert.equal(data.length, 41, 'Data length is correct.'); + data = utf8.parse(data); + var beginning = 'console.log( "mw.loader'; + assert.equal(data.slice(0, beginning.length), beginning, 'Content starts correctly.'); + done(); + }); + } else { done(); - }); - } - else { - done(); - } + } + }); }); - }); - QUnit.test("Split article 'A/Ray_Charles.html' can be loaded", function(assert) { - var done = assert.async(); - assert.expect(7); - localZimArchive.getDirEntryByPath("A/Ray_Charles.html").then(function(dirEntry) { - assert.ok(dirEntry !== null, "Title found"); - if (dirEntry !== null) { - assert.equal(dirEntry.namespace +"/"+ dirEntry.url, "A/Ray_Charles.html", "URL is correct."); - assert.equal(dirEntry.getMimetype(), "text/html", "MIME type is correct."); - localZimArchive.readUtf8File(dirEntry, function(dirEntry2, data) { - assert.equal(dirEntry2.getTitleOrUrl(), "Ray Charles", "Title is correct."); - assert.equal(data.length, 157186, "Data length is correct."); - assert.equal(data.indexOf("the only true genius in show business"), 5535, "Specific substring at beginning found."); - assert.equal(data.indexOf("Random Access Memories"), 154107, "Specific substring at end found."); + QUnit.test("Split article 'A/Ray_Charles.html' can be loaded", function (assert) { + var done = assert.async(); + assert.expect(7); + localZimArchive.getDirEntryByPath('A/Ray_Charles.html').then(function (dirEntry) { + assert.ok(dirEntry !== null, 'Title found'); + if (dirEntry !== null) { + assert.equal(dirEntry.namespace + '/' + dirEntry.url, 'A/Ray_Charles.html', 'URL is correct.'); + assert.equal(dirEntry.getMimetype(), 'text/html', 'MIME type is correct.'); + localZimArchive.readUtf8File(dirEntry, function (dirEntry2, data) { + assert.equal(dirEntry2.getTitleOrUrl(), 'Ray Charles', 'Title is correct.'); + assert.equal(data.length, 157186, 'Data length is correct.'); + assert.equal(data.indexOf('the only true genius in show business'), 5535, 'Specific substring at beginning found.'); + assert.equal(data.indexOf('Random Access Memories'), 154107, 'Specific substring at end found.'); + done(); + }); + } else { done(); - }); - } else { - done(); - } + } + }); }); - }); - QUnit.module("zim_random_and_main_article"); - QUnit.test("check that a random article is found", function(assert) { - var done = assert.async(); - assert.expect(2); - var callbackRandomArticleFound = function(dirEntry) { - assert.ok(dirEntry !== null, "One DirEntry should be found"); - assert.ok(dirEntry.getTitleOrUrl() !== null, "The random DirEntry should have a title" ); + QUnit.module('zim_random_and_main_article'); + QUnit.test('check that a random article is found', function (assert) { + var done = assert.async(); + assert.expect(2); + var callbackRandomArticleFound = function (dirEntry) { + assert.ok(dirEntry !== null, 'One DirEntry should be found'); + assert.ok(dirEntry.getTitleOrUrl() !== null, 'The random DirEntry should have a title'); - done(); - }; - localZimArchive.getRandomDirEntry(callbackRandomArticleFound); - }); - QUnit.test("check that the main article is found", function(assert) { - var done = assert.async(); - assert.expect(2); - var callbackMainPageArticleFound = function(dirEntry) { - assert.ok(dirEntry !== null, "Main DirEntry should be found"); - assert.equal(dirEntry.getTitleOrUrl(), "Summary", "The main DirEntry should be called Summary" ); + done(); + }; + localZimArchive.getRandomDirEntry(callbackRandomArticleFound); + }); + QUnit.test('check that the main article is found', function (assert) { + var done = assert.async(); + assert.expect(2); + var callbackMainPageArticleFound = function (dirEntry) { + assert.ok(dirEntry !== null, 'Main DirEntry should be found'); + assert.equal(dirEntry.getTitleOrUrl(), 'Summary', 'The main DirEntry should be called Summary'); - done(); - }; - localZimArchive.getMainPageDirEntry(callbackMainPageArticleFound); - }); + done(); + }; + localZimArchive.getMainPageDirEntry(callbackMainPageArticleFound); + }); - QUnit.start(); - }; -}); + QUnit.start(); + }; + }); diff --git a/www/js/app.js b/www/js/app.js index 84020f1e8..8a4650813 100644 --- a/www/js/app.js +++ b/www/js/app.js @@ -1,4 +1,3 @@ -/* eslint-disable indent */ /** * app.js : User Interface implementation * This file handles the interaction between the application and the user @@ -26,6 +25,7 @@ // The global parameters object is defined in init.js /* global params, define, webpMachine */ +/* eslint-disable indent */ // This uses require.js to structure javascript: // http://requirejs.org/docs/api.html#define diff --git a/www/js/lib/abstractFilesystemAccess.js b/www/js/lib/abstractFilesystemAccess.js index b48eb010e..f0396209b 100644 --- a/www/js/lib/abstractFilesystemAccess.js +++ b/www/js/lib/abstractFilesystemAccess.js @@ -5,41 +5,44 @@ * filesystem. * It is unfortunately not possible to do that inside a standard browser * (even inside an extension). - * + * * Copyright 2014 Kiwix developers * License GPL v3: - * + * * This file is part of Kiwix. - * + * * Kiwix is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + * * Kiwix is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + * * You should have received a copy of the GNU General Public License * along with Kiwix (file LICENSE-GPLv3.txt). If not, see */ + 'use strict'; -define([], function() { - + +/* global define */ + +define([], function () { /** * Storage implemented by Firefox OS - * + * * @typedef StorageFirefoxOS * @property {DeviceStorage} _storage DeviceStorage * @property {String} storageName Name of the storage */ - + /** * Creates an abstraction layer around the FirefoxOS storage. * @param storage FirefoxOS DeviceStorage object */ - function StorageFirefoxOS(storage) { + function StorageFirefoxOS (storage) { this._storage = storage; this.storageName = storage.storageName; }; @@ -49,27 +52,27 @@ define([], function() { * @return {Promise} Promise which is resolved with a HTML5 file object and * rejected with an error message. */ - StorageFirefoxOS.prototype.get = function(path) { + StorageFirefoxOS.prototype.get = function (path) { var that = this; - return new Promise(function (resolve, reject){ + return new Promise(function (resolve, reject) { var request = that._storage.get(path); - request.onsuccess = function() { resolve(this.result); }; - request.onerror = function() { reject(this.error.name); }; + request.onsuccess = function () { resolve(this.result); }; + request.onerror = function () { reject(this.error.name); }; }); }; - + // We try to match both a standalone ZIM file (.zim) or // the first file of a split ZIM files collection (.zimaa) var regexpZIMFileName = /\.zim(aa)?$/i; - + /** * Searches for archive files or directories. * @return {Promise} Promise which is resolved with an array of * paths and rejected with an error message. */ - StorageFirefoxOS.prototype.scanForArchives = function() { + StorageFirefoxOS.prototype.scanForArchives = function () { var that = this; - return new Promise(function (resolve, reject){ + return new Promise(function (resolve, reject) { var directories = []; var cursor = that._storage.enumerate(); cursor.onerror = function () { @@ -90,17 +93,16 @@ define([], function() { }; }); }; - + /** * Browse a path through DeviceStorage API * @param path Path where to look for files * @return {DOMCursor} Cursor of files found in given path */ - StorageFirefoxOS.prototype.enumerate = function(path) { + StorageFirefoxOS.prototype.enumerate = function (path) { return this._storage.enumerate(); }; - return { StorageFirefoxOS: StorageFirefoxOS }; diff --git a/www/js/lib/filecache.js b/www/js/lib/filecache.js index 040d3bf27..247506282 100644 --- a/www/js/lib/filecache.js +++ b/www/js/lib/filecache.js @@ -21,8 +21,11 @@ * You should have received a copy of the GNU General Public License * along with Kiwix JS (file LICENSE). If not, see */ + 'use strict'; +/* global define */ + define([], function () { /** * Set maximum number of cache blocks of BLOCK_SIZE bytes each @@ -42,7 +45,7 @@ define([], function () { /** * A Block Cache employing a Least Recently Used caching strategy * @typedef {Object} BlockCache - * @property {Number} capacity The maximum number of entries in the cache + * @property {Number} capacity The maximum number of entries in the cache * @property {Map} cache A map to store the cache keys and data */ @@ -50,7 +53,7 @@ define([], function () { * Creates a new cache with max size limit of MAX_CACHE_SIZE blocks * LRUCache implemnentation with Map adapted from https://markmurray.co/blog/lru-cache/ */ - function LRUCache() { + function LRUCache () { /** CACHE TUNING **/ // console.log('Creating cache of size ' + MAX_CACHE_SIZE + ' * ' + BLOCK_SIZE + ' bytes'); // Initialize persistent Cache properties @@ -62,7 +65,7 @@ define([], function () { * Tries to retrieve an element by its id. If it is not present in the cache, returns undefined; if it is present, * then the value is returned and the entry is moved to the bottom of the cache * @param {String} key The block cache entry key (file.id + ':' + byte offset) - * @returns {Uint8Array | undefined} The requested cache data or undefined + * @returns {Uint8Array | undefined} The requested cache data or undefined */ LRUCache.prototype.get = function (key) { var entry = this.cache.get(key); @@ -78,7 +81,7 @@ define([], function () { /** * Stores a value in the cache by id and prunes the least recently used entry if the cache is larger than MAX_CACHE_SIZE * @param {String} key The key under which to store the value (file.id + ':' + byte offset from start of ZIM archive) - * @param {Uint8Array} value The value to store in the cache + * @param {Uint8Array} value The value to store in the cache */ LRUCache.prototype.store = function (key, value) { // We get the existing entry's object for memory-management purposes; if it exists, it will contain identical data @@ -90,7 +93,7 @@ define([], function () { if (entry) this.cache.delete(key); else entry = value; this.cache.set(key, entry); - // If we've exceeded the cache capacity, then delete the least recently accessed value, + // If we've exceeded the cache capacity, then delete the least recently accessed value, // which will be the item at the top of the Map, i.e the first position if (this.cache.size > this.capacity) { if (this.cache.keys) { @@ -117,7 +120,7 @@ define([], function () { */ var cache = new LRUCache(); - /** CACHE TUNING **/ + /** CACHE TUNING **/ // DEV: Uncomment this block and blocks below marked 'CACHE TUNING' to measure Cache hit and miss rates for different Cache sizes // var hits = 0; // var misses = 0; @@ -128,7 +131,7 @@ define([], function () { * @param {Object} file The requested ZIM archive to read from * @param {Number} begin The byte from which to start reading * @param {Number} end The byte at which to stop reading (end will not be read) - * @return {Promise} A Promise that resolves to the correctly concatenated data from the cache + * @return {Promise} A Promise that resolves to the correctly concatenated data from the cache * or from the ZIM archive */ var read = function (file, begin, end) { @@ -144,7 +147,7 @@ define([], function () { // Data not in cache, so read from archive /** CACHE TUNING **/ // misses++; - // DEV: This is a self-calling function, i.e. the function is called with an argument of which then + // DEV: This is a self-calling function, i.e. the function is called with an argument of which then // becomes the parameter readRequests.push(function (offset) { return file._readSplitSlice(offset, offset + BLOCK_SIZE).then(function (result) { @@ -160,7 +163,7 @@ define([], function () { } /** CACHE TUNING **/ // if (misses + hits > 2000) { - // console.log('** Block cache hit rate: ' + Math.round(hits / (hits + misses) * 1000) / 10 + '% [ hits:' + hits + + // console.log('** Block cache hit rate: ' + Math.round(hits / (hits + misses) * 1000) / 10 + '% [ hits:' + hits + // ' / misses:' + misses + ' ] Size: ' + cache.cache.size); // hits = 0; // misses = 0; @@ -183,4 +186,4 @@ define([], function () { return { read: read }; -}); \ No newline at end of file +}); diff --git a/www/js/lib/settingsStore.js b/www/js/lib/settingsStore.js index c7391c741..5d82d1551 100644 --- a/www/js/lib/settingsStore.js +++ b/www/js/lib/settingsStore.js @@ -1,304 +1,309 @@ 'use strict'; + +/* global define, params */ + define([], function () { - /** - * settingsStore.js - * - * A reader/writer framework for cookies or localStorage with full unicode support based on the Mozilla cookies framework. - * The Mozilla code has been adapted to test for the availability of the localStorage API, and to use it in preference to cookies. - * - * Mozilla version information: - * - * Revision #1 - September 4, 2014 - * - * https://developer.mozilla.org/en-US/docs/Web/API/document.cookie - * https://developer.mozilla.org/User:fusionchess - * - * This framework is released under the GNU Public License, version 3 or later. - * http://www.gnu.org/licenses/gpl-3.0-standalone.html - * - * Syntaxes: - * - * * settingsStore.setItem(name, value[, end[, path[, domain[, secure]]]]) - * * settingsStore.getItem(name) - * * settingsStore.removeItem(name[, path[, domain]]) - * * settingsStore.hasItem(name) - * - */ + /** + * settingsStore.js + * + * A reader/writer framework for cookies or localStorage with full unicode support based on the Mozilla cookies framework. + * The Mozilla code has been adapted to test for the availability of the localStorage API, and to use it in preference to cookies. + * + * Mozilla version information: + * + * Revision #1 - September 4, 2014 + * + * https://developer.mozilla.org/en-US/docs/Web/API/document.cookie + * https://developer.mozilla.org/User:fusionchess + * + * This framework is released under the GNU Public License, version 3 or later. + * http://www.gnu.org/licenses/gpl-3.0-standalone.html + * + * Syntaxes: + * + * * settingsStore.setItem(name, value[, end[, path[, domain[, secure]]]]) + * * settingsStore.getItem(name) + * * settingsStore.removeItem(name[, path[, domain]]) + * * settingsStore.hasItem(name) + * + */ - /** - * A RegExp of the settings keys used in the cookie that should be migrated to localStorage if the API is available - * DEV: It should not be necessary to keep this list up-to-date because any keys added after this list was created - * (April 2020) will already be stored in localStorage if it is available to the client's browser or platform and - * will not need to be migrated - * @type {RegExp} - */ + /** + * A RegExp of the settings keys used in the cookie that should be migrated to localStorage if the API is available + * DEV: It should not be necessary to keep this list up-to-date because any keys added after this list was created + * (April 2020) will already be stored in localStorage if it is available to the client's browser or platform and + * will not need to be migrated + * @type {RegExp} + */ - var regexpCookieKeysToMigrate = new RegExp([ - 'hideActiveContentWarning', 'showUIAnimations', 'appTheme', 'useCache', - 'contentInjectionMode', 'listOfArchives', 'lastSelectedArchive' - ].join('|')); + var regexpCookieKeysToMigrate = new RegExp([ + 'hideActiveContentWarning', 'showUIAnimations', 'appTheme', 'useCache', + 'contentInjectionMode', 'listOfArchives', 'lastSelectedArchive' + ].join('|')); - /** - * A list of deprecated keys that should be removed. Add any further keys to the list of strings separated by a comma. - * @type {Array} - */ - var deprecatedKeys = [ - 'lastContentInjectionMode', - 'useCache' - ]; + /** + * A list of deprecated keys that should be removed. Add any further keys to the list of strings separated by a comma. + * @type {Array} + */ + var deprecatedKeys = [ + 'lastContentInjectionMode', + 'useCache' + ]; - /** - * The prefix that will be added to keys when stored in localStorage: this is used to prevent - * potential collision of key names with localStorage keys used by code inside ZIM archives - * It is set in init.js because it is needed early in app loading - * @type {String} - */ - var keyPrefix = params.keyPrefix; + /** + * The prefix that will be added to keys when stored in localStorage: this is used to prevent + * potential collision of key names with localStorage keys used by code inside ZIM archives + * It is set in init.js because it is needed early in app loading + * @type {String} + */ + var keyPrefix = params.keyPrefix; - // Tests for available Storage APIs (document.cookie or localStorage) and returns the best available of these - function getBestAvailableStorageAPI() { - // DEV: In FF extensions, cookies are blocked since at least FF 68.6 but possibly since FF 55 [kiwix-js #612] - var type = 'none'; - // First test for localStorage API support - var localStorageTest; - try { - localStorageTest = 'localStorage' in window && window['localStorage'] !== null; - // DEV: Above test returns true in IE11 running from file:// protocol, but attempting to write a key to - // localStorage causes an exception; so to test fully, we must now attempt to write and remove a test key - if (localStorageTest) { - localStorage.setItem('tempKiwixStorageTest', ''); - localStorage.removeItem('tempKiwixStorageTest'); - } - } catch (e) { - localStorageTest = false; + // Tests for available Storage APIs (document.cookie or localStorage) and returns the best available of these + function getBestAvailableStorageAPI () { + // DEV: In FF extensions, cookies are blocked since at least FF 68.6 but possibly since FF 55 [kiwix-js #612] + var type = 'none'; + // First test for localStorage API support + var localStorageTest; + try { + localStorageTest = 'localStorage' in window && window['localStorage'] !== null; + // DEV: Above test returns true in IE11 running from file:// protocol, but attempting to write a key to + // localStorage causes an exception; so to test fully, we must now attempt to write and remove a test key + if (localStorageTest) { + localStorage.setItem('tempKiwixStorageTest', ''); + localStorage.removeItem('tempKiwixStorageTest'); + } + } catch (e) { + localStorageTest = false; + } + // Now test for document.cookie API support + document.cookie = 'tempKiwixCookieTest=working; expires=Fri, 31 Dec 9999 23:59:59 GMT; SameSite=Strict'; + var kiwixCookieTest = /tempKiwixCookieTest=working/.test(document.cookie); + // Remove test value by expiring the key + document.cookie = 'tempKiwixCookieTest=; expires=Thu, 01 Jan 1970 00:00:00 GMT; SameSite=Strict'; + if (kiwixCookieTest) type = 'cookie'; + // Prefer localStorage if supported due to some platforms removing cookies once the session ends in some contexts + if (localStorageTest) type = 'local_storage'; + // If both cookies and localStorage are supported, and document.cookie contains keys to migrate, + // migrate settings to use localStorage + if (kiwixCookieTest && localStorageTest && regexpCookieKeysToMigrate.test(document.cookie)) _migrateStorageSettings(); + // Remove any deprecated keys + deprecatedKeys.forEach(function (key) { + if (localStorageTest) localStorage.removeItem(keyPrefix + key); + settingsStore.removeItem(key); // Because this runs before we have returned a store type, this will remove from cookie too + }); + // Note that if this function returns 'none', the cookie implementations below will run anyway. This is because storing a cookie + // does not cause an exception even if cookies are blocked in some contexts, whereas accessing localStorage may cause an exception + return type; } - // Now test for document.cookie API support - document.cookie = 'tempKiwixCookieTest=working; expires=Fri, 31 Dec 9999 23:59:59 GMT; SameSite=Strict'; - var kiwixCookieTest = /tempKiwixCookieTest=working/.test(document.cookie); - // Remove test value by expiring the key - document.cookie = 'tempKiwixCookieTest=; expires=Thu, 01 Jan 1970 00:00:00 GMT; SameSite=Strict'; - if (kiwixCookieTest) type = 'cookie'; - // Prefer localStorage if supported due to some platforms removing cookies once the session ends in some contexts - if (localStorageTest) type = 'local_storage'; - // If both cookies and localStorage are supported, and document.cookie contains keys to migrate, - // migrate settings to use localStorage - if (kiwixCookieTest && localStorageTest && regexpCookieKeysToMigrate.test(document.cookie)) _migrateStorageSettings(); - // Remove any deprecated keys - deprecatedKeys.forEach(function (key) { - if (localStorageTest) localStorage.removeItem(keyPrefix + key); - settingsStore.removeItem(key); // Because this runs before we have returned a store type, this will remove from cookie too - }); - // Note that if this function returns 'none', the cookie implementations below will run anyway. This is because storing a cookie - // does not cause an exception even if cookies are blocked in some contexts, whereas accessing localStorage may cause an exception - return type; - } - /** - * Performs a full app reset, deleting all caches and settings - * Or, if a parameter is supplied, deletes or disables the object - * @param {String} object Optional name of the object to disable or delete ('cookie', 'localStorage', 'cacheAPI') - */ - function reset(object) { - // 1. Clear any cookie entries - if (!object || object === 'cookie') { - var regexpCookieKeys = /(?:^|;)\s*([^=]+)=([^;]*)/ig; - var currentCookie = document.cookie; - var foundCrumb = false; - var cookieCrumb = regexpCookieKeys.exec(currentCookie); - while (cookieCrumb !== null) { - // DEV: Note that we don't use the keyPrefix in legacy cookie support - foundCrumb = true; - // This expiry date will cause the browser to delete the cookie crumb on next page refresh - document.cookie = cookieCrumb[1] + '=;expires=Thu, 21 Sep 1979 00:00:01 UTC;'; - cookieCrumb = regexpCookieKeys.exec(currentCookie); - } - if (foundCrumb) console.debug('All cookie keys were expired...'); - } + /** + * Performs a full app reset, deleting all caches and settings + * Or, if a parameter is supplied, deletes or disables the object + * @param {String} object Optional name of the object to disable or delete ('cookie', 'localStorage', 'cacheAPI') + */ + function reset (object) { + // 1. Clear any cookie entries + if (!object || object === 'cookie') { + var regexpCookieKeys = /(?:^|;)\s*([^=]+)=([^;]*)/ig; + var currentCookie = document.cookie; + var foundCrumb = false; + var cookieCrumb = regexpCookieKeys.exec(currentCookie); + while (cookieCrumb !== null) { + // DEV: Note that we don't use the keyPrefix in legacy cookie support + foundCrumb = true; + // This expiry date will cause the browser to delete the cookie crumb on next page refresh + document.cookie = cookieCrumb[1] + '=;expires=Thu, 21 Sep 1979 00:00:01 UTC;'; + cookieCrumb = regexpCookieKeys.exec(currentCookie); + } + if (foundCrumb) console.debug('All cookie keys were expired...'); + } - // 2. Clear any localStorage settings - if (!object || object === 'localStorage') { - if (params.storeType === 'local_storage') { - localStorage.clear(); - console.debug('All Local Storage settings were deleted...'); - } - } + // 2. Clear any localStorage settings + if (!object || object === 'localStorage') { + if (params.storeType === 'local_storage') { + localStorage.clear(); + console.debug('All Local Storage settings were deleted...'); + } + } - // 3. Clear any Cache API caches - if (!object || object === 'cacheAPI') { - getCacheNames(function (cacheNames) { - if (cacheNames && !cacheNames.error) { - var cnt = 0; - for (var cacheName in cacheNames) { - cnt++; - caches.delete(cacheNames[cacheName]).then(function () { - cnt--; - if (!cnt) { - // All caches deleted - console.debug('All Cache API caches were deleted...'); - // Reload if user performed full reset or if appCache is needed - if (!object || params.appCache) _reloadApp(); - } + // 3. Clear any Cache API caches + if (!object || object === 'cacheAPI') { + getCacheNames(function (cacheNames) { + if (cacheNames && !cacheNames.error) { + var cnt = 0; + for (var cacheName in cacheNames) { + cnt++; + caches.delete(cacheNames[cacheName]).then(function () { + cnt--; + if (!cnt) { + // All caches deleted + console.debug('All Cache API caches were deleted...'); + // Reload if user performed full reset or if appCache is needed + if (!object || params.appCache) _reloadApp(); + } + }); + } + } else { + console.debug('No Cache API caches were in use (or we do not have access to the names).'); + // All operations complete, reload if user performed full reset or if appCache is needed + if (!object || params.appCache) _reloadApp(); + } }); - } - } else { - console.debug('No Cache API caches were in use (or we do not have access to the names).'); - // All operations complete, reload if user performed full reset or if appCache is needed - if (!object || params.appCache) _reloadApp(); } - }); - } - } - - // Gets cache names from Service Worker, as we cannot rely on having them in params.cacheNames - function getCacheNames(callback) { - if (navigator.serviceWorker && navigator.serviceWorker.controller) { - var channel = new MessageChannel(); - channel.port1.onmessage = function (event) { - var names = event.data; - callback(names); - }; - navigator.serviceWorker.controller.postMessage({ - action: 'getCacheNames' - }, [channel.port2]); - } else { - callback(null); } - } - // Deregisters all Service Workers and reboots the app - function _reloadApp() { - var reboot = function () { - console.debug('Performing app reload...'); - setTimeout(function () { - window.location.href = location.origin + location.pathname + uriParams - }, 300); - }; - // Blank the querystring, so that parameters are not set on reload - var uriParams = ''; - if (~window.location.href.indexOf(params.PWAServer) && params.referrerExtensionURL) { - // However, if we're in a PWA that was called from local code, then by definition we must remain in SW mode and we need to - // ensure the user still has access to the referrerExtensionURL (so they can get back to local code from the UI) - uriParams = '?allowInternetAccess=truee&contentInjectionMode=serviceworker'; - uriParams += '&referrerExtensionURL=' + encodeURIComponent(params.referrerExtensionURL); - } - if (navigator && navigator.serviceWorker) { - console.debug('Deregistering Service Workers...'); - var cnt = 0; - navigator.serviceWorker.getRegistrations().then(function (registrations) { - if (!registrations.length) { - reboot(); - return; + // Gets cache names from Service Worker, as we cannot rely on having them in params.cacheNames + function getCacheNames (callback) { + if (navigator.serviceWorker && navigator.serviceWorker.controller) { + var channel = new MessageChannel(); + channel.port1.onmessage = function (event) { + var names = event.data; + callback(names); + }; + navigator.serviceWorker.controller.postMessage({ + action: 'getCacheNames' + }, [channel.port2]); + } else { + callback(null); } - cnt++; - registrations.forEach(function (registration) { - registration.unregister().then(function () { - cnt--; - if (!cnt) { - console.debug('All Service Workers unregistered...'); - reboot(); - } - }); - }); - }).catch(function (err) { - console.error(err); - reboot(); - }); - } else { - console.debug('Performing app reload...'); - reboot(); } - } - var settingsStore = { - getItem: function (sKey) { - if (!sKey) { - return null; - } - if (params.storeType !== 'local_storage') { - return decodeURIComponent(document.cookie.replace(new RegExp("(?:(?:^|.*;)\\s*" + encodeURIComponent(sKey).replace(/[-.+*]/g, "\\$&") + "\\s*\\=\\s*([^;]*).*$)|^.*$"), "$1")) || null; - } else { - return localStorage.getItem(keyPrefix + sKey); - } - }, - setItem: function (sKey, sValue, vEnd, sPath, sDomain, bSecure) { - if (params.storeType !== 'local_storage') { - if (!sKey || /^(?:expires|max-age|path|domain|secure)$/i.test(sKey)) { - return false; + // Deregisters all Service Workers and reboots the app + function _reloadApp () { + var reboot = function () { + console.debug('Performing app reload...'); + setTimeout(function () { + window.location.href = location.origin + location.pathname + uriParams + }, 300); + }; + // Blank the querystring, so that parameters are not set on reload + var uriParams = ''; + if (~window.location.href.indexOf(params.PWAServer) && params.referrerExtensionURL) { + // However, if we're in a PWA that was called from local code, then by definition we must remain in SW mode and we need to + // ensure the user still has access to the referrerExtensionURL (so they can get back to local code from the UI) + uriParams = '?allowInternetAccess=truee&contentInjectionMode=serviceworker'; + uriParams += '&referrerExtensionURL=' + encodeURIComponent(params.referrerExtensionURL); } - var sExpires = ""; - if (vEnd) { - switch (vEnd.constructor) { - case Number: - sExpires = vEnd === Infinity ? "; expires=Fri, 31 Dec 9999 23:59:59 GMT" : "; max-age=" + vEnd; - break; - case String: - sExpires = "; expires=" + vEnd; - break; - case Date: - sExpires = "; expires=" + vEnd.toUTCString(); - break; - } + if (navigator && navigator.serviceWorker) { + console.debug('Deregistering Service Workers...'); + var cnt = 0; + navigator.serviceWorker.getRegistrations().then(function (registrations) { + if (!registrations.length) { + reboot(); + return; + } + cnt++; + registrations.forEach(function (registration) { + registration.unregister().then(function () { + cnt--; + if (!cnt) { + console.debug('All Service Workers unregistered...'); + reboot(); + } + }); + }); + }).catch(function (err) { + console.error(err); + reboot(); + }); + } else { + console.debug('Performing app reload...'); + reboot(); } - document.cookie = encodeURIComponent(sKey) + "=" + encodeURIComponent(sValue) + sExpires + (sDomain ? "; domain=" + sDomain : "") + (sPath ? "; path=" + sPath : "") + (bSecure ? "; secure" : ""); - } else { - localStorage.setItem(keyPrefix + sKey, sValue); - } - return true; - }, - removeItem: function (sKey, sPath, sDomain) { - if (!this.hasItem(sKey)) { - return false; - } - if (params.storeType !== 'local_storage') { - document.cookie = encodeURIComponent(sKey) + "=; expires=Thu, 01 Jan 1970 00:00:00 GMT" + (sDomain ? "; domain=" + sDomain : "") + (sPath ? "; path=" + sPath : ""); - } else { - localStorage.removeItem(keyPrefix + sKey); - } - return true; - }, - hasItem: function (sKey) { - if (!sKey) { - return false; - } - if (params.storeType !== 'local_storage') { - return (new RegExp("(?:^|;\\s*)" + encodeURIComponent(sKey).replace(/[-.+*]/g, "\\$&") + "\\s*\\=")).test(document.cookie); - } else { - return localStorage.getItem(keyPrefix + sKey) === null ? false : true; - } - }, - _cookieKeys: function () { - var aKeys = document.cookie.replace(/((?:^|\s*;)[^=]+)(?=;|$)|^\s*|\s*(?:=[^;]*)?(?:\1|$)/g, "").split(/\s*(?:=[^;]*)?;\s*/); - for (var nLen = aKeys.length, nIdx = 0; nIdx < nLen; nIdx++) { - aKeys[nIdx] = decodeURIComponent(aKeys[nIdx]); - } - return aKeys; } - }; - // One-off migration of storage settings from cookies to localStorage - function _migrateStorageSettings() { - console.log('Migrating Settings Store from cookies to localStorage...'); - var cookieKeys = settingsStore._cookieKeys(); - // Note that because migration occurs before setting params.storeType, settingsStore.getItem() will get the item from - // document.cookie instead of localStorage, which is the intended behaviour - for (var i = 0; i < cookieKeys.length; i++) { - if (regexpCookieKeysToMigrate.test(cookieKeys[i])) { - var migratedKey = keyPrefix + cookieKeys[i]; - localStorage.setItem(migratedKey, settingsStore.getItem(cookieKeys[i])); - settingsStore.removeItem(cookieKeys[i]); - console.log('- ' + migratedKey); - } + var settingsStore = { + getItem: function (sKey) { + if (!sKey) { + return null; + } + if (params.storeType !== 'local_storage') { + return decodeURIComponent(document.cookie.replace(new RegExp('(?:(?:^|.*;)\\s*' + encodeURIComponent(sKey).replace(/[-.+*\\]/g, '\\$&') + '\\s*\\=\\s*([^;]*).*$)|^.*$'), '$1')) || null; + } else { + return localStorage.getItem(keyPrefix + sKey); + } + }, + setItem: function (sKey, sValue, vEnd, sPath, sDomain, bSecure) { + if (params.storeType !== 'local_storage') { + if (!sKey || /^(?:expires|max-age|path|domain|secure)$/i.test(sKey)) { + return false; + } + var sExpires = ''; + if (vEnd) { + switch (vEnd.constructor) { + case Number: + sExpires = vEnd === Infinity ? '; expires=Fri, 31 Dec 9999 23:59:59 GMT' : '; max-age=' + vEnd; + break; + case String: + sExpires = '; expires=' + vEnd; + break; + case Date: + sExpires = '; expires=' + vEnd.toUTCString(); + break; + } + } + document.cookie = encodeURIComponent(sKey) + '=' + encodeURIComponent(sValue) + sExpires + (sDomain ? '; domain=' + sDomain : '') + (sPath ? '; path=' + sPath : '') + (bSecure ? '; secure' : ''); + } else { + localStorage.setItem(keyPrefix + sKey, sValue); + } + return true; + }, + removeItem: function (sKey, sPath, sDomain) { + if (!this.hasItem(sKey)) { + return false; + } + if (params.storeType !== 'local_storage') { + document.cookie = encodeURIComponent(sKey) + '=; expires=Thu, 01 Jan 1970 00:00:00 GMT' + (sDomain ? '; domain=' + sDomain : '') + (sPath ? '; path=' + sPath : ''); + } else { + localStorage.removeItem(keyPrefix + sKey); + } + return true; + }, + hasItem: function (sKey) { + if (!sKey) { + return false; + } + if (params.storeType !== 'local_storage') { + return (new RegExp('(?:^|;\\s*)' + encodeURIComponent(sKey).replace(/[-.+*\\]/g, '\\$&') + '\\s*\\=')).test(document.cookie); + } else { + return localStorage.getItem(keyPrefix + sKey) !== null; + } + }, + _cookieKeys: function () { + // Disabling linter check because this is library code + // eslint-disable-next-line no-useless-backreference + var aKeys = document.cookie.replace(/((?:^|\s*;)[^=]+)(?=;|$)|^\s*|\s*(?:=[^;]*)?(?:\1|$)/g, '').split(/\s*(?:=[^;]*)?;\s*/); + for (var nLen = aKeys.length, nIdx = 0; nIdx < nLen; nIdx++) { + aKeys[nIdx] = decodeURIComponent(aKeys[nIdx]); + } + return aKeys; + } + }; + + // One-off migration of storage settings from cookies to localStorage + function _migrateStorageSettings () { + console.log('Migrating Settings Store from cookies to localStorage...'); + var cookieKeys = settingsStore._cookieKeys(); + // Note that because migration occurs before setting params.storeType, settingsStore.getItem() will get the item from + // document.cookie instead of localStorage, which is the intended behaviour + for (var i = 0; i < cookieKeys.length; i++) { + if (regexpCookieKeysToMigrate.test(cookieKeys[i])) { + var migratedKey = keyPrefix + cookieKeys[i]; + localStorage.setItem(migratedKey, settingsStore.getItem(cookieKeys[i])); + settingsStore.removeItem(cookieKeys[i]); + console.log('- ' + migratedKey); + } + } + console.log('Migration done.'); } - console.log('Migration done.'); - } - return { - getItem: settingsStore.getItem, - setItem: settingsStore.setItem, - removeItem: settingsStore.removeItem, - hasItem: settingsStore.hasItem, - getCacheNames: getCacheNames, - reset: reset, - getBestAvailableStorageAPI: getBestAvailableStorageAPI - }; + return { + getItem: settingsStore.getItem, + setItem: settingsStore.setItem, + removeItem: settingsStore.removeItem, + hasItem: settingsStore.hasItem, + getCacheNames: getCacheNames, + reset: reset, + getBestAvailableStorageAPI: getBestAvailableStorageAPI + }; }); diff --git a/www/js/lib/utf8.js b/www/js/lib/utf8.js index 0f5ca8189..dfc9cee56 100644 --- a/www/js/lib/utf8.js +++ b/www/js/lib/utf8.js @@ -2,28 +2,32 @@ * utf8.js : UTF8 conversion functions * Original code from http://stackoverflow.com/users/553542/kadm , taken from * http://stackoverflow.com/questions/1240408/reading-bytes-from-a-javascript-string - * + * * Copyright 2013-2014 Mossroy and contributors * License GPL v3: - * + * * This file is part of Kiwix. - * + * * Kiwix is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + * * Kiwix is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + * * You should have received a copy of the GNU General Public License * along with Kiwix (file LICENSE-GPLv3.txt). If not, see */ + 'use strict'; -define([], function() { +/* global define */ +/* eslint-disable eqeqeq */ + +define([], function () { var utf8 = {}; /** @@ -31,16 +35,18 @@ define([], function() { * @param {String} str * @returns {Array} */ - utf8.toByteArray = function(str) { + utf8.toByteArray = function (str) { var byteArray = []; - for (var i = 0; i < str.length; i++) - if (str.charCodeAt(i) <= 0x7F) + for (var i = 0; i < str.length; i++) { + if (str.charCodeAt(i) <= 0x7F) { byteArray.push(str.charCodeAt(i)); - else { + } else { var h = encodeURIComponent(str.charAt(i)).substr(1).split('%'); - for (var j = 0; j < h.length; j++) + for (var j = 0; j < h.length; j++) { byteArray.push(parseInt(h[j], 16)); + } } + } return byteArray; }; @@ -50,50 +56,44 @@ define([], function() { * @param {Boolean} zeroTerminated * @returns {String} */ - utf8.parse = function(data, zeroTerminated) - { + utf8.parse = function (data, zeroTerminated) { var u0, u1, u2, u3, u4, u5; var str = ''; - for (var idx = 0; idx < data.length; ) { + for (var idx = 0; idx < data.length;) { u0 = data[idx++]; - if (u0 === 0 && zeroTerminated) + if (u0 === 0 && zeroTerminated) { break; - if (!(u0 & 0x80)) - { + } + if (!(u0 & 0x80)) { str += String.fromCharCode(u0); continue; } u1 = data[idx++] & 63; - if ((u0 & 0xe0) == 0xc0) - { + if ((u0 & 0xe0) == 0xc0) { str += String.fromCharCode(((u0 & 31) << 6) | u1); continue; } u2 = data[idx++] & 63; - if ((u0 & 0xf0) == 0xe0) + if ((u0 & 0xf0) == 0xe0) { u0 = ((u0 & 15) << 12) | (u1 << 6) | u2; - else - { + } else { u3 = data[idx++] & 63; - if ((u0 & 0xF8) == 0xF0) + if ((u0 & 0xF8) == 0xF0) { u0 = ((u0 & 7) << 18) | (u1 << 12) | (u2 << 6) | u3; - else - { + } else { u4 = data[idx++] & 63; - if ((u0 & 0xFC) == 0xF8) + if ((u0 & 0xFC) == 0xF8) { u0 = ((u0 & 3) << 24) | (u1 << 18) | (u2 << 12) | (u3 << 6) | u4; - else - { + } else { u5 = data[idx++] & 63; u0 = ((u0 & 1) << 30) | (u1 << 24) | (u2 << 18) | (u3 << 12) | (u4 << 6) | u5; } } } - if (u0 < 0x10000) + if (u0 < 0x10000) { str += String.fromCharCode(u0); - else - { + } else { var ch = u0 - 0x10000; str += String.fromCharCode(0xD800 | (ch >> 10), 0xDC00 | (ch & 0x3FF)); } diff --git a/www/js/lib/util.js b/www/js/lib/util.js index c388a9568..4edd83181 100644 --- a/www/js/lib/util.js +++ b/www/js/lib/util.js @@ -20,15 +20,17 @@ * along with Kiwix (file LICENSE-GPLv3.txt). If not, see */ 'use strict'; -define([], function() { +/* global define */ + +define([], function () { /** * A Regular Expression to match the first letter of a word even if preceded by Unicode punctuation * Includes currency signs and mathematical symbols: see https://stackoverflow.com/a/21396529/9727685 * DEV: To maintain the list below, see https://github.com/slevithan/xregexp/blob/master/tools/output/categories.js * where all the different Unicode punctuation categories can be found (simplify double backspacing before using below) * Note that the XRegExp punctuation categories begin at !-# in list below - * @type {RegExp} + * @type {RegExp} */ var regExpFindStringParts = /(?:^|.+?)(?:[\s$£€\uFFE5^+=`~<>{}[\]|\u3000-\u303F!-#%-\x2A,-/:;\x3F@\x5B-\x5D_\x7B}\u00A1\u00A7\u00AB\u00B6\u00B7\u00BB\u00BF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u0AF0\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166D\u166E\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2010-\u2027\u2030-\u2043\u2045-\u2051\u2053-\u205E\u207D\u207E\u208D\u208E\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E3B\u3001-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65]+|$)/g; @@ -38,10 +40,10 @@ define([], function() { * If caseMatchType is 'full', then all-uppercase combinations of each word are added to the variations array * NB may produce duplicate strings if string begins with punctuation or if it is in a language with no case * @param {String} string The string to be converted - * @param {String} caseMatchType ('basic'|'full') The type (complexity) of case variations to calculate + * @param {String} caseMatchType ('basic'|'full') The type (complexity) of case variations to calculate * @return {Array} An array containing strings with all possible combinations of case types */ - function allCaseFirstLetters(string, caseMatchType) { + function allCaseFirstLetters (string, caseMatchType) { if (string) { var comboArray = []; // Split string into parts beginning with first word letters @@ -93,12 +95,13 @@ define([], function() { * @param {Array} array of String * @returns {Array} same array of Strings, without duplicates */ - function removeDuplicateStringsInSmallArray(array) { + function removeDuplicateStringsInSmallArray (array) { var unique = []; for (var i = 0; i < array.length; i++) { var current = array[i]; - if (unique.indexOf(current) < 0) + if (unique.indexOf(current) < 0) { unique.push(current); + } } return unique; } @@ -109,7 +112,7 @@ define([], function() { * @param {String} suffix * @returns {Boolean} */ - function endsWith(str, suffix) { + function endsWith (str, suffix) { return str.indexOf(suffix, str.length - suffix.length) !== -1; } @@ -120,7 +123,7 @@ define([], function() { * @param {Boolean} littleEndian (optional) * @returns {Float} */ - function readFloatFrom4Bytes(byteArray, firstIndex, littleEndian) { + function readFloatFrom4Bytes (byteArray, firstIndex, littleEndian) { var dataView = new DataView(byteArray.buffer, firstIndex, 4); var float = dataView.getFloat32(0, littleEndian); return float; @@ -131,9 +134,9 @@ define([], function() { * @param {File} file The file object to be read * @param {Integer} begin The offset in at which to begin reading * @param {Integer} end The byte at whcih to stop reading (reads up to and including end - 1) - * @returns {Promise} A Promise for an array buffer with the read data + * @returns {Promise} A Promise for an array buffer with the read data */ - function readFileSlice(file, begin, end) { + function readFileSlice (file, begin, end) { if ('arrayBuffer' in Blob.prototype) { // DEV: This method uses the native arrayBuffer method of Blob, if available, as it eliminates // the need to use FileReader and set up event listeners; it also uses the method's native Promise @@ -165,18 +168,19 @@ define([], function() { * @param {Boolean} lowerBound Determines the type of search * @returns {Promise} Promise for the lowest dirEntry that fulfils (or fails to fulfil) the query */ - function binarySearch(begin, end, query, lowerBound) { - if (end <= begin) + function binarySearch (begin, end, query, lowerBound) { + if (end <= begin) { return lowerBound ? begin : null; + } var mid = Math.floor((begin + end) / 2); - return query(mid).then(function(decision) - { - if (decision < 0) + return query(mid).then(function (decision) { + if (decision < 0) { return binarySearch(begin, mid, query, lowerBound); - else if (decision > 0) + } else if (decision > 0) { return binarySearch(mid + 1, end, query, lowerBound); - else + } else { return mid; + } }); } @@ -189,23 +193,23 @@ define([], function() { * @param {Integer} bits * @returns {Integer} */ - function leftShift(int, bits) { + function leftShift (int, bits) { return int * Math.pow(2, bits); } /** * Queues Promise Factories* to be resolved or rejected sequentially. This helps to avoid overlapping Promise functions. * Primarily used by uiUtil.systemAlert, to prevent alerts showing while others are being displayed. - * + * * *A Promise Factory is merely a Promise wrapped in a function to prevent it from executing immediately. E.g. to use * this function with a Promise, call it like this (or, more likely, use your own pre-wrapped Promise): - * + * * return util.PromiseQueue.enqueue(function () { * return new Promise(function (resolve, reject) { ... }); * }); - * + * * Adapted from https://medium.com/@karenmarkosyan/how-to-manage-promises-into-dynamic-queue-with-vanilla-javascript-9d0d1f8d4df5 - * + * * @type {Object} PromiseQueue * @property {Function} enqueue Queues a Promise Factory. Call this function repeatedly to queue Promises sequentially * @param {Function} promiseFactory A Promise wrapped in an ordinary function @@ -217,7 +221,7 @@ define([], function() { enqueue: function (promiseFactory) { var that = this; return new Promise(function (resolve, reject) { - that._queue.push({promise: promiseFactory, resolve: resolve, reject: reject}); + that._queue.push({ promise: promiseFactory, resolve: resolve, reject: reject }); if (!that._working) that._dequeue(); }); }, diff --git a/www/js/lib/xzdec_wrapper.js b/www/js/lib/xzdec_wrapper.js index 292ba7bae..956ae8e8d 100644 --- a/www/js/lib/xzdec_wrapper.js +++ b/www/js/lib/xzdec_wrapper.js @@ -19,8 +19,11 @@ * You should have received a copy of the GNU General Public License * along with Kiwix (file LICENSE-GPLv3.txt). If not, see */ + 'use strict'; +/* global params, define, XZ */ + // DEV: Put your RequireJS definition in the rqDefXZ array below, and any function exports in the function parenthesis of the define statement // We need to do it this way in order to load the wasm or asm versions of xzdec conditionally. Older browsers can only use the asm version // because they cannot interpret WebAssembly. @@ -40,7 +43,7 @@ if ('WebAssembly' in self) { rqDefXZ.push('xzdec-asm'); } -define(rqDefXZ, function(uiUtil) { +define(rqDefXZ, function (uiUtil) { // DEV: xzdec.js has been compiled with `-s EXPORT_NAME="XZ" -s MODULARIZE=1` to avoid a clash with zstddec.js // Note that we include xzdec-asm or xzdec-wasm above in requireJS definition, but we cannot change the name in the function list // There is no longer any need to load it in index.html @@ -78,19 +81,19 @@ define(rqDefXZ, function(uiUtil) { }); } }); - + /** * Number of milliseconds to wait for the decompressor to be available for another chunk * @type Integer */ var DELAY_WAITING_IDLE_DECOMPRESSOR = 50; - + /** * Is the decompressor already working? * @type Boolean */ var busy = false; - + /** * @typedef Decompressor * @property {Integer} _chunkSize @@ -100,14 +103,14 @@ define(rqDefXZ, function(uiUtil) { * @property {Integer} _outStreamPos * @property {Array} _outBuffer */ - + /** * @constructor * @param {FileReader} reader * @param {Integer} chunkSize * @returns {Decompressor} */ - function Decompressor(reader, chunkSize) { + function Decompressor (reader, chunkSize) { params.decompressorAPI.decompressorLastUsed = 'XZ'; this._chunkSize = chunkSize || 1024 * 5; this._reader = reader; @@ -119,7 +122,7 @@ define(rqDefXZ, function(uiUtil) { * @param {Integer} offset * @param {Integer} length */ - Decompressor.prototype.readSlice = function(offset, length) { + Decompressor.prototype.readSlice = function (offset, length) { busy = true; var that = this; this._inStreamPos = 0; @@ -127,13 +130,13 @@ define(rqDefXZ, function(uiUtil) { this._decHandle = xzdec._init_decompression(this._chunkSize); this._outBuffer = new Int8Array(new ArrayBuffer(length)); this._outBufferPos = 0; - return this._readLoop(offset, length).then(function(data) { + return this._readLoop(offset, length).then(function (data) { xzdec._release(that._decHandle); busy = false; return data; }); }; - + /** * Reads stream of data from file offset for length of bytes to send to the decompresor * This function ensures that only one decompression runs at a time @@ -159,14 +162,14 @@ define(rqDefXZ, function(uiUtil) { }; /** - * + * * @param {Integer} offset * @param {Integer} length * @returns {Array} */ - Decompressor.prototype._readLoop = function(offset, length) { + Decompressor.prototype._readLoop = function (offset, length) { var that = this; - return this._fillInBufferIfNeeded().then(function() { + return this._fillInBufferIfNeeded().then(function () { var ret = xzdec._decompress(that._decHandle); var finished = false; if (ret === 0) { @@ -180,37 +183,41 @@ define(rqDefXZ, function(uiUtil) { } var outPos = xzdec._get_out_pos(that._decHandle); - if (outPos > 0 && that._outStreamPos + outPos >= offset) - { + if (outPos > 0 && that._outStreamPos + outPos >= offset) { var outBuffer = xzdec._get_out_buffer(that._decHandle); var copyStart = offset - that._outStreamPos; - if (copyStart < 0) + if (copyStart < 0) { copyStart = 0; - for (var i = copyStart; i < outPos && that._outBufferPos < that._outBuffer.length; i++) + } + for (var i = copyStart; i < outPos && that._outBufferPos < that._outBuffer.length; i++) { that._outBuffer[that._outBufferPos++] = xzdec.HEAP8[outBuffer + i]; + } } that._outStreamPos += outPos; - if (outPos > 0) + if (outPos > 0) { xzdec._out_buffer_cleared(that._decHandle); - if (finished || that._outStreamPos >= offset + length) + } + if (finished || that._outStreamPos >= offset + length) { return that._outBuffer; - else + } else { return that._readLoop(offset, length); + } }); }; - + /** - * + * * @returns {Promise} */ - Decompressor.prototype._fillInBufferIfNeeded = function() { + Decompressor.prototype._fillInBufferIfNeeded = function () { if (!xzdec._input_empty(this._decHandle)) { return Promise.resolve(0); } var that = this; - return this._reader(this._inStreamPos, this._chunkSize).then(function(data) { - if (data.length > that._chunkSize) + return this._reader(this._inStreamPos, this._chunkSize).then(function (data) { + if (data.length > that._chunkSize) { data = data.slice(0, that._chunkSize); + } // For some reason, xzdec.writeArrayToMemory does not seem to be available, and is equivalent to xzdec.HEAP8.set xzdec.HEAP8.set(data, xzdec._get_in_buffer(that._decHandle)); that._inStreamPos += data.length; diff --git a/www/js/lib/zimArchive.js b/www/js/lib/zimArchive.js index 72a14dc11..6476017bf 100644 --- a/www/js/lib/zimArchive.js +++ b/www/js/lib/zimArchive.js @@ -19,563 +19,571 @@ * You should have received a copy of the GNU General Public License * along with Kiwix (file LICENSE-GPLv3.txt). If not, see */ + 'use strict'; + +/* global params, define */ + define(['zimfile', 'zimDirEntry', 'util', 'uiUtil', 'utf8'], - function(zimfile, zimDirEntry, util, uiUtil, utf8) { - - /** - * ZIM Archive - * - * - * @typedef ZIMArchive - * @property {ZIMFile} _file The ZIM file (instance of ZIMFile, that might physically be split into several actual files) - * @property {String} _language Language of the content - */ - - /** - * @callback callbackZIMArchive - * @param {ZIMArchive} zimArchive Ready-to-use ZIMArchive - */ - - /** - * @callback callbackMetadata - * @param {String} data metadata string - */ + function (zimfile, zimDirEntry, util, uiUtil, utf8) { + /** + * ZIM Archive + * + * + * @typedef ZIMArchive + * @property {ZIMFile} _file The ZIM file (instance of ZIMFile, that might physically be split into several actual files) + * @property {String} _language Language of the content + */ - /** - * @param {Worker} LZ A Web Worker to run the libzim Web Assembly binary - */ - var LZ; - - /** - * Creates a ZIM archive object to access the ZIM file at the given path in the given storage. - * This constructor can also be used with a single File parameter. - * - * @param {StorageFirefoxOS|Array} storage Storage (in this case, the path must be given) or Array of Files (path parameter must be omitted) - * @param {String} path The Storage path for an OS that requires this to be specified - * @param {callbackZIMArchive} callbackReady The function to call when the archive is ready to use - * @param {callbackZIMArchive} callbackError The function to call when an error occurs - */ - function ZIMArchive(storage, path, callbackReady, callbackError) { - var that = this; - that._file = null; - that._language = ""; //@TODO - var createZimfile = function (fileArray) { - zimfile.fromFileArray(fileArray).then(function (file) { - that._file = file; - // Clear the previous libzimWoker - LZ = null; - // Set a global parameter to report the search provider type - params.searchProvider = 'title'; - // File has been created, but we need to add any Listings which extend the archive metadata - that._file.setListings([ - // Provide here any Listings for which we need to extract metadata as key:value obects to be added to the file - // 'ptrName' and 'countName' contain the key names to be set in the archive file object - { - // This defines the standard v0 (legacy) title index that contains listings for every entry in the ZIM (not just articles) - // It represents the same index that is referenced in the ZIM archive header - path: 'X/listing/titleOrdered/v0', - ptrName: 'titlePtrPos', - countName: 'entryCount' - }, - { - // This defines a new version 1 index that is present in no-namespace ZIMs, and contains a title-ordered list of articles - path: 'X/listing/titleOrdered/v1', - ptrName: 'articlePtrPos', - countName: 'articleCount' - }, - { - // This tests for and specifies the existence of any Xapian Full Text Index - path: 'X/fulltext/xapian', - ptrName: 'fullTextIndex', - countName: 'fullTextIndexSize' - } - ]).then(function () { - // There is currently an exception thrown in the libzim wasm if we attempt to load a split ZIM archive, so we work around - var isSplitZim = /\.zima.$/i.test(that._file._files[0].name); - if (params.debugLibzimASM || that._file.fullTextIndex && !isSplitZim && typeof Atomics !== 'undefined' - // Note that Android and NWJS currently throw due to problems with Web Worker context - && !/Android/.test(params.appType) && !(window.nw && that._file._files[0].readMode === 'electron')) { - var libzimReaderType = params.debugLibzimASM || ('WebAssembly' in self ? 'wasm' : 'asm'); - console.log('Instantiating libzim ' + libzimReaderType + ' Web Worker...'); - LZ = new Worker('js/lib/libzim-' + libzimReaderType + '.js'); - that.callLibzimWorker({action: "init", files: that._file._files}).then(function (msg) { - // console.debug(msg); - params.searchProvider = 'fulltext: ' + libzimReaderType; - // Update the API panel + /** + * @callback callbackZIMArchive + * @param {ZIMArchive} zimArchive Ready-to-use ZIMArchive + */ + + /** + * @callback callbackMetadata + * @param {String} data metadata string + */ + + /** + * @param {Worker} LZ A Web Worker to run the libzim Web Assembly binary + */ + var LZ; + + /** + * Creates a ZIM archive object to access the ZIM file at the given path in the given storage. + * This constructor can also be used with a single File parameter. + * + * @param {StorageFirefoxOS|Array} storage Storage (in this case, the path must be given) or Array of Files (path parameter must be omitted) + * @param {String} path The Storage path for an OS that requires this to be specified + * @param {callbackZIMArchive} callbackReady The function to call when the archive is ready to use + * @param {callbackZIMArchive} callbackError The function to call when an error occurs + */ + function ZIMArchive (storage, path, callbackReady, callbackError) { + var that = this; + that._file = null; + that._language = ''; // @TODO + var createZimfile = function (fileArray) { + zimfile.fromFileArray(fileArray).then(function (file) { + that._file = file; + // Clear the previous libzimWoker + LZ = null; + // Set a global parameter to report the search provider type + params.searchProvider = 'title'; + // File has been created, but we need to add any Listings which extend the archive metadata + that._file.setListings([ + // Provide here any Listings for which we need to extract metadata as key:value obects to be added to the file + // 'ptrName' and 'countName' contain the key names to be set in the archive file object + { + // This defines the standard v0 (legacy) title index that contains listings for every entry in the ZIM (not just articles) + // It represents the same index that is referenced in the ZIM archive header + path: 'X/listing/titleOrdered/v0', + ptrName: 'titlePtrPos', + countName: 'entryCount' + }, + { + // This defines a new version 1 index that is present in no-namespace ZIMs, and contains a title-ordered list of articles + path: 'X/listing/titleOrdered/v1', + ptrName: 'articlePtrPos', + countName: 'articleCount' + }, + { + // This tests for and specifies the existence of any Xapian Full Text Index + path: 'X/fulltext/xapian', + ptrName: 'fullTextIndex', + countName: 'fullTextIndexSize' + } + ]).then(function () { + // There is currently an exception thrown in the libzim wasm if we attempt to load a split ZIM archive, so we work around + var isSplitZim = /\.zima.$/i.test(that._file._files[0].name); + if (params.debugLibzimASM || that._file.fullTextIndex && !isSplitZim && typeof Atomics !== 'undefined' && + // Note that Android and NWJS currently throw due to problems with Web Worker context + !/Android/.test(params.appType) && !(window.nw && that._file._files[0].readMode === 'electron')) { + var libzimReaderType = params.debugLibzimASM || ('WebAssembly' in self ? 'wasm' : 'asm'); + console.log('Instantiating libzim ' + libzimReaderType + ' Web Worker...'); + LZ = new Worker('js/lib/libzim-' + libzimReaderType + '.js'); + that.callLibzimWorker({ action: 'init', files: that._file._files }).then(function (msg) { + // console.debug(msg); + params.searchProvider = 'fulltext: ' + libzimReaderType; + // Update the API panel + uiUtil.reportSearchProviderToAPIStatusPanel(params.searchProvider); + }).catch(function (err) { + uiUtil.reportSearchProviderToAPIStatusPanel(params.searchProvider + ': ERROR'); + console.error('The libzim worker could not be instantiated!', err); + }); + } else { + // var message = 'Full text searching is not available because '; + if (!that._file.fullTextIndex) { + params.searchProvider += ': no_fulltext'; // message += 'this ZIM does not have a full-text index.'; + } else if (isSplitZim) { + params.searchProvider += ': split_zim'; // message += 'the ZIM archive is split.'; + } else if (typeof Atomics === 'undefined') { + params.searchProvider += ': no_atomics'; // message += 'this browser does not support Atomic operations.'; + } else if (/Android/.test(params.appType)) { + params.searchProvider += ': no_sharedArrayBuffer'; + } uiUtil.reportSearchProviderToAPIStatusPanel(params.searchProvider); - }).catch(function (err) { - uiUtil.reportSearchProviderToAPIStatusPanel(params.searchProvider + ': ERROR'); - console.error('The libzim worker could not be instantiated!', err); - }); - } else { - // var message = 'Full text searching is not available because '; - if (!that._file.fullTextIndex) { - params.searchProvider += ': no_fulltext'; // message += 'this ZIM does not have a full-text index.'; - } else if (isSplitZim) { - params.searchProvider += ': split_zim'; // message += 'the ZIM archive is split.'; - } else if (typeof Atomics === 'undefined') { - params.searchProvider += ': no_atomics'; // message += 'this browser does not support Atomic operations.'; - } else if (/Android/.test(params.appType)) { - params.searchProvider += ': no_sharedArrayBuffer'; + // uiUtil.systemAlert(message); } - uiUtil.reportSearchProviderToAPIStatusPanel(params.searchProvider); - // uiUtil.systemAlert(message); - } - }); - // Set the archive file type ('open' or 'zimit') - params.zimType = that.setZimType(); - // DEV: Currently, extended listings are only used for title (=article) listings when the user searches - // for an article or uses the Random button, by which time the listings will have been extracted. - // If, in the future, listings are used in a more time-critical manner, consider forcing a wait before - // declaring the archive to be ready, by chaining the following callback in a .then() function of setListings. - callbackReady(that); - }); - }; - if (storage && !path) { - var fileList = storage; - // We need to convert the FileList into an Array - var fileArray = [].slice.call(fileList); - // The constructor has been called with an array of File/Blob parameter - createZimfile(fileArray); - } else { - if (/.*zim..$/.test(path)) { - // split archive - that._searchArchiveParts(storage, path.slice(0, -2)).then(function (fileArray) { - createZimfile(fileArray); - }).catch(function (error) { - callbackError("Error reading files in split archive " + path + ": " + error, "Error reading archive files"); + }); + // Set the archive file type ('open' or 'zimit') + params.zimType = that.setZimType(); + // DEV: Currently, extended listings are only used for title (=article) listings when the user searches + // for an article or uses the Random button, by which time the listings will have been extracted. + // If, in the future, listings are used in a more time-critical manner, consider forcing a wait before + // declaring the archive to be ready, by chaining the following callback in a .then() function of setListings. + callbackReady(that); }); + }; + if (storage && !path) { + var fileList = storage; + // We need to convert the FileList into an Array + var fileArray = [].slice.call(fileList); + // The constructor has been called with an array of File/Blob parameter + createZimfile(fileArray); } else { - storage.get(path).then(function (file) { - createZimfile([file]); - }).catch(function (error) { - callbackError("Error reading ZIM file " + path + " : " + error, "Error reading archive file"); - }); + if (/.*zim..$/.test(path)) { + // split archive + that._searchArchiveParts(storage, path.slice(0, -2)).then(function (fileArray) { + createZimfile(fileArray); + }).catch(function (error) { + callbackError('Error reading files in split archive ' + path + ': ' + error, 'Error reading archive files'); + }); + } else { + storage.get(path).then(function (file) { + createZimfile([file]); + }).catch(function (error) { + callbackError('Error reading ZIM file ' + path + ' : ' + error, 'Error reading archive file'); + }); + } } } - } - /** - * Searches the directory for all parts of a split archive. - * @param {Storage} storage storage interface - * @param {String} prefixPath path to the split files, missing the "aa" / "ab" / ... suffix. - * @returns {Promise} that resolves to the array of file objects found. - */ - ZIMArchive.prototype._searchArchiveParts = function(storage, prefixPath) { - var fileArray = []; - var nextFile = function(part) { - var suffix = String.fromCharCode(0x61 + Math.floor(part / 26)) + String.fromCharCode(0x61 + part % 26); - return storage.get(prefixPath + suffix) - .then(function(file) { - fileArray.push(file); - return nextFile(part + 1); - }, function(error) { - return fileArray; - }); + /** + * Searches the directory for all parts of a split archive. + * @param {Storage} storage storage interface + * @param {String} prefixPath path to the split files, missing the "aa" / "ab" / ... suffix. + * @returns {Promise} that resolves to the array of file objects found. + */ + ZIMArchive.prototype._searchArchiveParts = function (storage, prefixPath) { + var fileArray = []; + var nextFile = function (part) { + var suffix = String.fromCharCode(0x61 + Math.floor(part / 26)) + String.fromCharCode(0x61 + part % 26); + return storage.get(prefixPath + suffix) + .then(function (file) { + fileArray.push(file); + return nextFile(part + 1); + }, function (error) { + console.error('Error reading split archive file ' + prefixPath + suffix + ': ', error); + return fileArray; + }); + }; + return nextFile(0); }; - return nextFile(0); - }; - /** - * - * @returns {Boolean} - */ - ZIMArchive.prototype.isReady = function() { - return this._file !== null; - }; + /** + * + * @returns {Boolean} + */ + ZIMArchive.prototype.isReady = function () { + return this._file !== null; + }; - /** - * Detects whether the supplied archive is a Zimit-style archive or an OpenZIM archive and - * sets a _file.zimType property accordingly; also returns the detected type. Extends ZIMFile. - * @returns {String} Either 'zimit' for a Zimit archive, or 'open' for an OpenZIM archive - */ - ZIMArchive.prototype.setZimType = function () { - var fileType = null; - if (this.isReady()) { - fileType = 'open'; - this._file.mimeTypes.forEach(function (v) { - if (/warc-headers/i.test(v)) fileType = 'zimit'; - }); - this._file.zimType = fileType; - console.debug('Archive type set to: ' + fileType); - } else { - console.error('ZIMArchive is not ready! Cannot set ZIM type.'); - } - return fileType; - }; + /** + * Detects whether the supplied archive is a Zimit-style archive or an OpenZIM archive and + * sets a _file.zimType property accordingly; also returns the detected type. Extends ZIMFile. + * @returns {String} Either 'zimit' for a Zimit archive, or 'open' for an OpenZIM archive + */ + ZIMArchive.prototype.setZimType = function () { + var fileType = null; + if (this.isReady()) { + fileType = 'open'; + this._file.mimeTypes.forEach(function (v) { + if (/warc-headers/i.test(v)) fileType = 'zimit'; + }); + this._file.zimType = fileType; + console.debug('Archive type set to: ' + fileType); + } else { + console.error('ZIMArchive is not ready! Cannot set ZIM type.'); + } + return fileType; + }; - /** - * Looks for the DirEntry of the main page - * @param {callbackDirEntry} callback - * @returns {Promise} that resolves to the DirEntry - */ - ZIMArchive.prototype.getMainPageDirEntry = function(callback) { - if (this.isReady()) { - var mainPageUrlIndex = this._file.mainPage; - this._file.dirEntryByUrlIndex(mainPageUrlIndex).then(callback); - } - }; + /** + * Looks for the DirEntry of the main page + * @param {callbackDirEntry} callback + * @returns {Promise} that resolves to the DirEntry + */ + ZIMArchive.prototype.getMainPageDirEntry = function (callback) { + if (this.isReady()) { + var mainPageUrlIndex = this._file.mainPage; + this._file.dirEntryByUrlIndex(mainPageUrlIndex).then(callback); + } + }; - /** - * - * @param {String} dirEntryId - * @returns {DirEntry} - */ - ZIMArchive.prototype.parseDirEntryId = function(dirEntryId) { - return zimDirEntry.DirEntry.fromStringId(this._file, dirEntryId); - }; - - /** - * @callback callbackDirEntryList - * @param {Array.} dirEntryArray Array of DirEntries found - */ + /** + * + * @param {String} dirEntryId + * @returns {DirEntry} + */ + ZIMArchive.prototype.parseDirEntryId = function (dirEntryId) { + return zimDirEntry.DirEntry.fromStringId(this._file, dirEntryId); + }; - /** - * Look for DirEntries with title starting with the prefix of the current search object. - * For now, ZIM titles are case sensitive. - * So, as workaround, we try several variants of the prefix to find more results. - * This should be enhanced when the ZIM format will be modified to store normalized titles - * See https://phabricator.wikimedia.org/T108536 - * - * @param {Object} search The current appstate.search object - * @param {callbackDirEntryList} callback The function to call with the result - * @param {Boolean} noInterim A flag to prevent callback until all results are ready (used in testing) - */ - ZIMArchive.prototype.findDirEntriesWithPrefix = function (search, callback, noInterim) { - var that = this; - // Establish array of initial values that must be searched first. All of these patterns are generated by the full - // search type, and some by basic, but we need the most common patterns to be searched first, as it returns search - // results much more quickly if we do this (and the user can click on a result before the rarer patterns complete) - // NB duplicates are removed before processing search array - var startArray = []; - var dirEntries = []; - search.scanCount = 0; - // Launch a full-text search if possible - if (LZ) that.findDirEntriesFromFullTextSearch(search, dirEntries).then(function (fullTextDirEntries) { - // If user initiated a new search, cancel this one - // In particular, do not set the search status back to 'complete' - // as that would cause outdated results to unexpectedly pop up - if (search.status === 'cancelled') return callback([], search); - dirEntries = fullTextDirEntries; - search.status = 'complete'; - callback(dirEntries, search); - }); - // Ensure a search is done on the string exactly as typed - startArray.push(search.prefix); - // Normalize any spacing and make string all lowercase - var prefix = search.prefix.replace(/\s+/g, ' ').toLocaleLowerCase(); - // Add lowercase string with initial uppercase (this is a very common pattern) - startArray.push(prefix.replace(/^./, function (m) { - return m.toLocaleUpperCase(); - })); - // Get the full array of combinations to check number of combinations - var fullCombos = util.removeDuplicateStringsInSmallArray(util.allCaseFirstLetters(prefix, 'full')); - // Put cap on exponential number of combinations (five words = 3^5 = 243 combinations) - search.type = fullCombos.length < 300 ? 'full' : 'basic'; - // We have to remove duplicate string combinations because util.allCaseFirstLetters() can return some combinations - // where uppercase and lowercase combinations are exactly the same, e.g. where prefix begins with punctuation - // or currency signs, for languages without case, or where user-entered case duplicates calculated case - var prefixVariants = util.removeDuplicateStringsInSmallArray( - startArray.concat( - // Get basic combinations first for speed of returning results - util.allCaseFirstLetters(prefix).concat( - search.type === 'full' ? fullCombos : [] - ) - ) - ); - function searchNextVariant() { - // If user has initiated a new search, cancel this one - if (search.status === 'cancelled') return callback([], search); - if (prefixVariants.length === 0 || dirEntries.length >= search.size) { - // We have found all the title-search entries we are going to get, so indicate search type if we're still searching - if (LZ && search.status !== 'complete') search.type = 'fulltext'; - else search.status = 'complete'; - return callback(dirEntries, search); - } - // Dynamically populate list of articles - search.status = 'interim'; - if (!noInterim) callback(dirEntries, search); - search.found = dirEntries.length; - var prefix = prefixVariants[0]; - // console.debug('Searching for: ' + prefixVariants[0]); - prefixVariants = prefixVariants.slice(1); - that.findDirEntriesWithPrefixCaseSensitive(prefix, search, - function (newDirEntries, countReport, interim) { - search.countReport = countReport; + /** + * @callback callbackDirEntryList + * @param {Array.} dirEntryArray Array of DirEntries found + */ + + /** + * Look for DirEntries with title starting with the prefix of the current search object. + * For now, ZIM titles are case sensitive. + * So, as workaround, we try several variants of the prefix to find more results. + * This should be enhanced when the ZIM format will be modified to store normalized titles + * See https://phabricator.wikimedia.org/T108536 + * + * @param {Object} search The current appstate.search object + * @param {callbackDirEntryList} callback The function to call with the result + * @param {Boolean} noInterim A flag to prevent callback until all results are ready (used in testing) + */ + ZIMArchive.prototype.findDirEntriesWithPrefix = function (search, callback, noInterim) { + var that = this; + // Establish array of initial values that must be searched first. All of these patterns are generated by the full + // search type, and some by basic, but we need the most common patterns to be searched first, as it returns search + // results much more quickly if we do this (and the user can click on a result before the rarer patterns complete) + // NB duplicates are removed before processing search array + var startArray = []; + var dirEntries = []; + search.scanCount = 0; + // Launch a full-text search if possible + if (LZ) { + that.findDirEntriesFromFullTextSearch(search, dirEntries).then(function (fullTextDirEntries) { + // If user initiated a new search, cancel this one + // In particular, do not set the search status back to 'complete' + // as that would cause outdated results to unexpectedly pop up if (search.status === 'cancelled') return callback([], search); - if (!noInterim && countReport === true) return callback(dirEntries, search); - if (interim) {// Only push interim results (else results will be pushed again at end of variant loop) - [].push.apply(dirEntries, newDirEntries); - search.found = dirEntries.length; - if (!noInterim && newDirEntries.length) return callback(dirEntries, search); - } else return searchNextVariant(); - } - ); - } - searchNextVariant(); - }; - - /** - * A method to return the namespace in the ZIM file that contains the primary user content. In old-format ZIM files (minor - * version 0) there are a number of content namespaces, but the primary one in which to search for titles is 'A'. In new-format - * ZIMs (minor version 1) there is a single content namespace 'C'. See https://openzim.org/wiki/ZIM_file_format. This method - * throws an error if it cannot determine the namespace or if the ZIM is not ready. - * @returns {String} The content namespace for the ZIM archive - */ - ZIMArchive.prototype.getContentNamespace = function () { - var errorText; - if (this.isReady()) { - var ver = this._file.minorVersion; - // DEV: There are currently only two defined values for minorVersion in the OpenZIM specification - // If this changes, adapt the error checking and return values - if (ver > 1) { - errorText = 'Unknown ZIM minor version!'; - } else { - return ver === 0 ? 'A' : 'C'; + dirEntries = fullTextDirEntries; + search.status = 'complete'; + callback(dirEntries, search); + }); } - } else { - errorText = 'We could not determine the content namespace because the ZIM file is not ready!'; - } - throw new Error(errorText); - }; - - /** - * Look for dirEntries with title starting with the given prefix (case-sensitive) - * - * @param {String} prefix The case-sensitive value against which dirEntry titles (or url) will be compared - * @param {Object} search The appstate.search object (for comparison, so that we can cancel long binary searches) - * @param {callbackDirEntryList} callback The function to call with the array of dirEntries with titles that begin with prefix - */ - ZIMArchive.prototype.findDirEntriesWithPrefixCaseSensitive = function(prefix, search, callback) { - var that = this; - var cns = this.getContentNamespace(); - // Search v1 article listing if available, otherwise fallback to v0 - var articleCount = this._file.articleCount || this._file.entryCount; - util.binarySearch(0, articleCount, function(i) { - return that._file.dirEntryByTitleIndex(i).then(function(dirEntry) { - if (search.status === 'cancelled') return 0; - var ns = dirEntry.namespace; - // DEV: This search is redundant if we managed to populate articlePtrLst and articleCount, but it only takes two instructions and - // provides maximum compatibility with rare ZIMs where attempts to find first and last article (in zimArchive.js) may have failed - if (ns < cns) return 1; - if (ns > cns) return -1; - // We should now be in namespace A (old format ZIM) or C (new format ZIM) - return prefix <= dirEntry.getTitleOrUrl() ? -1 : 1; - }); - }, true).then(function(firstIndex) { - var vDirEntries = []; - var addDirEntries = function(index, lastTitle) { - if (search.status === 'cancelled' || search.found >= search.size || index >= articleCount - || lastTitle && !~lastTitle.indexOf(prefix)) { - // DEV: Diagnostics to be removed before merge - if (vDirEntries.length) { - console.debug('Scanned ' + (index - firstIndex) + ' titles for "' + prefix + - '" (found ' + vDirEntries.length + ' match' + (vDirEntries.length === 1 ? ')' : 'es)')); - } - return { - 'dirEntries': vDirEntries, - 'nextStart': index - }; + // Ensure a search is done on the string exactly as typed + startArray.push(search.prefix); + // Normalize any spacing and make string all lowercase + var prefix = search.prefix.replace(/\s+/g, ' ').toLocaleLowerCase(); + // Add lowercase string with initial uppercase (this is a very common pattern) + startArray.push(prefix.replace(/^./, function (m) { + return m.toLocaleUpperCase(); + })); + // Get the full array of combinations to check number of combinations + var fullCombos = util.removeDuplicateStringsInSmallArray(util.allCaseFirstLetters(prefix, 'full')); + // Put cap on exponential number of combinations (five words = 3^5 = 243 combinations) + search.type = fullCombos.length < 300 ? 'full' : 'basic'; + // We have to remove duplicate string combinations because util.allCaseFirstLetters() can return some combinations + // where uppercase and lowercase combinations are exactly the same, e.g. where prefix begins with punctuation + // or currency signs, for languages without case, or where user-entered case duplicates calculated case + var prefixVariants = util.removeDuplicateStringsInSmallArray( + startArray.concat( + // Get basic combinations first for speed of returning results + util.allCaseFirstLetters(prefix).concat( + search.type === 'full' ? fullCombos : [] + ) + ) + ); + function searchNextVariant () { + // If user has initiated a new search, cancel this one + if (search.status === 'cancelled') return callback([], search); + if (prefixVariants.length === 0 || dirEntries.length >= search.size) { + // We have found all the title-search entries we are going to get, so indicate search type if we're still searching + if (LZ && search.status !== 'complete') search.type = 'fulltext'; + else search.status = 'complete'; + return callback(dirEntries, search); } - return that._file.dirEntryByTitleIndex(index).then(function(dirEntry) { - search.scanCount++; - var title = dirEntry.getTitleOrUrl(); - // Only return dirEntries with titles that actually begin with prefix - if (dirEntry.namespace === cns && title.indexOf(prefix) === 0) { - vDirEntries.push(dirEntry); - // Report interim result - callback([dirEntry], false, true); + // Dynamically populate list of articles + search.status = 'interim'; + if (!noInterim) callback(dirEntries, search); + search.found = dirEntries.length; + var prefix = prefixVariants[0]; + // console.debug('Searching for: ' + prefixVariants[0]); + prefixVariants = prefixVariants.slice(1); + that.findDirEntriesWithPrefixCaseSensitive(prefix, search, + function (newDirEntries, countReport, interim) { + search.countReport = countReport; + if (search.status === 'cancelled') return callback([], search); + if (!noInterim && countReport === true) return callback(dirEntries, search); + if (interim) { // Only push interim results (else results will be pushed again at end of variant loop) + [].push.apply(dirEntries, newDirEntries); + search.found = dirEntries.length; + if (!noInterim && newDirEntries.length) return callback(dirEntries, search); + } else return searchNextVariant(); } - return addDirEntries(index + 1, title); - }); - }; - return addDirEntries(firstIndex); - }).then(callback); - }; + ); + } + searchNextVariant(); + }; - /** - * Find Directory Entries corresponding to the requested search using Full Text search provided by libzim - * - * @param {Object} search The appstate.search object - * @param {Array} dirEntries The array of already found Directory Entries - * @returns {Promise} The augmented array of Directory Entries with titles that correspond to search - */ - ZIMArchive.prototype.findDirEntriesFromFullTextSearch = function (search, dirEntries) { - var cns = this.getContentNamespace(); - var that = this; - // We give ourselves an overhead in caclulating the results needed, because full-text search will return some results already found - // var resultsNeeded = Math.floor(params.maxSearchResultsSize - dirEntries.length / 2); - var resultsNeeded = params.maxSearchResultsSize; - return this.callLibzimWorker({action: "search", text: search.prefix, numResults: resultsNeeded}).then(function (results) { - if (results) { - var dirEntryPaths = []; - var fullTextPaths = []; - // Collect all the found paths for the dirEntries - for (var i = 0; i < dirEntries.length; i++) { - dirEntryPaths.push(dirEntries[i].namespace + '/' + dirEntries[i].url); - } - // Collect all the paths for full text search, pruning as we go - var path; - for (var j = 0; j < results.entries.length; j++) { - search.scanCount++; - path = results.entries[j].path; - // Full-text search result paths are missing the namespace in Type 1 ZIMs, so we add it back - path = cns === 'C' ? cns + '/' + path : path; - if (~dirEntryPaths.indexOf(path)) continue; - fullTextPaths.push(path); - } - var promisesForDirEntries = []; - for (var k = 0; k < fullTextPaths.length; k++) { - promisesForDirEntries.push(that.getDirEntryByPath(fullTextPaths[k])); + /** + * A method to return the namespace in the ZIM file that contains the primary user content. In old-format ZIM files (minor + * version 0) there are a number of content namespaces, but the primary one in which to search for titles is 'A'. In new-format + * ZIMs (minor version 1) there is a single content namespace 'C'. See https://openzim.org/wiki/ZIM_file_format. This method + * throws an error if it cannot determine the namespace or if the ZIM is not ready. + * @returns {String} The content namespace for the ZIM archive + */ + ZIMArchive.prototype.getContentNamespace = function () { + var errorText; + if (this.isReady()) { + var ver = this._file.minorVersion; + // DEV: There are currently only two defined values for minorVersion in the OpenZIM specification + // If this changes, adapt the error checking and return values + if (ver > 1) { + errorText = 'Unknown ZIM minor version!'; + } else { + return ver === 0 ? 'A' : 'C'; } - return Promise.all(promisesForDirEntries).then(function (fullTextDirEntries) { - for (var l = 0; l < fullTextDirEntries.length; l++) { - dirEntries.push(fullTextDirEntries[l]); - } - return(dirEntries); - }); } else { - return(dirEntries); + errorText = 'We could not determine the content namespace because the ZIM file is not ready!'; } - }); - }; + throw new Error(errorText); + }; - /** - * Calls the libzim Web Worker with the given parameters, and returns a Promise with its response - * - * @param {Object} parameters - * @returns {Promise} - */ - ZIMArchive.prototype.callLibzimWorker = function (parameters) { - return new Promise(function (resolve, reject) { - console.debug("Calling libzim WebWorker with parameters", parameters); - var tmpMessageChannel = new MessageChannel(); - // var t0 = performance.now(); - tmpMessageChannel.port1.onmessage = function (event) { - // var t1 = performance.now(); - // var readTime = Math.round(t1 - t0); - // console.debug("Response given by the WebWorker in " + readTime + " ms", event.data); - resolve(event.data); - }; - tmpMessageChannel.port1.onerror = function (event) { - // var t1 = performance.now(); - // var readTime = Math.round(t1 - t0); - // console.error("Error sent by the WebWorker in " + readTime + " ms", event.data); - reject(event.data); - }; - LZ.postMessage(parameters, [tmpMessageChannel.port2]); - }); - }; - - /** - * @callback callbackDirEntry - * @param {DirEntry} dirEntry The DirEntry found - */ + /** + * Look for dirEntries with title starting with the given prefix (case-sensitive) + * + * @param {String} prefix The case-sensitive value against which dirEntry titles (or url) will be compared + * @param {Object} search The appstate.search object (for comparison, so that we can cancel long binary searches) + * @param {callbackDirEntryList} callback The function to call with the array of dirEntries with titles that begin with prefix + */ + ZIMArchive.prototype.findDirEntriesWithPrefixCaseSensitive = function (prefix, search, callback) { + var that = this; + var cns = this.getContentNamespace(); + // Search v1 article listing if available, otherwise fallback to v0 + var articleCount = this._file.articleCount || this._file.entryCount; + util.binarySearch(0, articleCount, function (i) { + return that._file.dirEntryByTitleIndex(i).then(function (dirEntry) { + if (search.status === 'cancelled') return 0; + var ns = dirEntry.namespace; + // DEV: This search is redundant if we managed to populate articlePtrLst and articleCount, but it only takes two instructions and + // provides maximum compatibility with rare ZIMs where attempts to find first and last article (in zimArchive.js) may have failed + if (ns < cns) return 1; + if (ns > cns) return -1; + // We should now be in namespace A (old format ZIM) or C (new format ZIM) + return prefix <= dirEntry.getTitleOrUrl() ? -1 : 1; + }); + }, true).then(function (firstIndex) { + var vDirEntries = []; + var addDirEntries = function (index, lastTitle) { + if (search.status === 'cancelled' || search.found >= search.size || index >= articleCount || + lastTitle && !~lastTitle.indexOf(prefix)) { + // DEV: Diagnostics to be removed before merge + if (vDirEntries.length) { + console.debug('Scanned ' + (index - firstIndex) + ' titles for "' + prefix + + '" (found ' + vDirEntries.length + ' match' + (vDirEntries.length === 1 ? ')' : 'es)')); + } + return { + dirEntries: vDirEntries, + nextStart: index + }; + } + return that._file.dirEntryByTitleIndex(index).then(function (dirEntry) { + search.scanCount++; + var title = dirEntry.getTitleOrUrl(); + // Only return dirEntries with titles that actually begin with prefix + if (dirEntry.namespace === cns && title.indexOf(prefix) === 0) { + vDirEntries.push(dirEntry); + // Report interim result + callback([dirEntry], false, true); + } + return addDirEntries(index + 1, title); + }); + }; + return addDirEntries(firstIndex); + }).then(callback); + }; - /** - * - * @param {DirEntry} dirEntry - * @param {callbackDirEntry} callback - */ - ZIMArchive.prototype.resolveRedirect = function(dirEntry, callback) { - this._file.dirEntryByUrlIndex(dirEntry.redirectTarget).then(callback); - }; - - /** - * @callback callbackStringContent - * @param {String} content String content - */ - - /** - * - * @param {DirEntry} dirEntry - * @param {callbackStringContent} callback - */ - ZIMArchive.prototype.readUtf8File = function(dirEntry, callback) { - dirEntry.readData().then(function(data) { - callback(dirEntry, utf8.parse(data)); - }); - }; + /** + * Find Directory Entries corresponding to the requested search using Full Text search provided by libzim + * + * @param {Object} search The appstate.search object + * @param {Array} dirEntries The array of already found Directory Entries + * @returns {Promise} The augmented array of Directory Entries with titles that correspond to search + */ + ZIMArchive.prototype.findDirEntriesFromFullTextSearch = function (search, dirEntries) { + var cns = this.getContentNamespace(); + var that = this; + // We give ourselves an overhead in caclulating the results needed, because full-text search will return some results already found + // var resultsNeeded = Math.floor(params.maxSearchResultsSize - dirEntries.length / 2); + var resultsNeeded = params.maxSearchResultsSize; + return this.callLibzimWorker({ action: 'search', text: search.prefix, numResults: resultsNeeded }).then(function (results) { + if (results) { + var dirEntryPaths = []; + var fullTextPaths = []; + // Collect all the found paths for the dirEntries + for (var i = 0; i < dirEntries.length; i++) { + dirEntryPaths.push(dirEntries[i].namespace + '/' + dirEntries[i].url); + } + // Collect all the paths for full text search, pruning as we go + var path; + for (var j = 0; j < results.entries.length; j++) { + search.scanCount++; + path = results.entries[j].path; + // Full-text search result paths are missing the namespace in Type 1 ZIMs, so we add it back + path = cns === 'C' ? cns + '/' + path : path; + if (~dirEntryPaths.indexOf(path)) continue; + fullTextPaths.push(path); + } + var promisesForDirEntries = []; + for (var k = 0; k < fullTextPaths.length; k++) { + promisesForDirEntries.push(that.getDirEntryByPath(fullTextPaths[k])); + } + return Promise.all(promisesForDirEntries).then(function (fullTextDirEntries) { + for (var l = 0; l < fullTextDirEntries.length; l++) { + dirEntries.push(fullTextDirEntries[l]); + } + return dirEntries; + }); + } else { + return dirEntries; + } + }); + }; - /** - * @callback callbackBinaryContent - * @param {Uint8Array} content binary content - */ + /** + * Calls the libzim Web Worker with the given parameters, and returns a Promise with its response + * + * @param {Object} parameters + * @returns {Promise} + */ + ZIMArchive.prototype.callLibzimWorker = function (parameters) { + return new Promise(function (resolve, reject) { + console.debug('Calling libzim WebWorker with parameters', parameters); + var tmpMessageChannel = new MessageChannel(); + // var t0 = performance.now(); + tmpMessageChannel.port1.onmessage = function (event) { + // var t1 = performance.now(); + // var readTime = Math.round(t1 - t0); + // console.debug("Response given by the WebWorker in " + readTime + " ms", event.data); + resolve(event.data); + }; + tmpMessageChannel.port1.onerror = function (event) { + // var t1 = performance.now(); + // var readTime = Math.round(t1 - t0); + // console.error("Error sent by the WebWorker in " + readTime + " ms", event.data); + reject(event.data); + }; + LZ.postMessage(parameters, [tmpMessageChannel.port2]); + }); + }; - /** - * Read a binary file. - * @param {DirEntry} dirEntry - * @param {callbackBinaryContent} callback - */ - ZIMArchive.prototype.readBinaryFile = function(dirEntry, callback) { - return dirEntry.readData().then(function(data) { - callback(dirEntry, data); - }); - }; + /** + * @callback callbackDirEntry + * @param {DirEntry} dirEntry The DirEntry found + */ - /** - * Searches the URL pointer list of Directory Entries by pathname - * @param {String} path The pathname of the DirEntry that is required (namespace + filename) - * @return {Promise} A Promise that resolves to a Directory Entry, or null if not found. - */ - ZIMArchive.prototype.getDirEntryByPath = function(path) { - var that = this; - return util.binarySearch(0, this._file.entryCount, function(i) { - return that._file.dirEntryByUrlIndex(i).then(function(dirEntry) { - var url = dirEntry.namespace + "/" + dirEntry.url; - if (path < url) - return -1; - else if (path > url) - return 1; - else - return 0; + /** + * + * @param {DirEntry} dirEntry + * @param {callbackDirEntry} callback + */ + ZIMArchive.prototype.resolveRedirect = function (dirEntry, callback) { + this._file.dirEntryByUrlIndex(dirEntry.redirectTarget).then(callback); + }; + + /** + * @callback callbackStringContent + * @param {String} content String content + */ + + /** + * + * @param {DirEntry} dirEntry + * @param {callbackStringContent} callback + */ + ZIMArchive.prototype.readUtf8File = function (dirEntry, callback) { + dirEntry.readData().then(function (data) { + callback(dirEntry, utf8.parse(data)); }); - }).then(function(index) { - if (index === null) return null; - return that._file.dirEntryByUrlIndex(index); - }).then(function(dirEntry) { - return dirEntry; - }); - }; + }; - /** - * - * @param {callbackDirEntry} callback - */ - ZIMArchive.prototype.getRandomDirEntry = function(callback) { - // Prefer an article-only (v1) title pointer list, if available - var articleCount = this._file.articleCount || this._file.entryCount; - var index = Math.floor(Math.random() * articleCount); - this._file.dirEntryByTitleIndex(index).then(callback); - }; + /** + * @callback callbackBinaryContent + * @param {Uint8Array} content binary content + */ - /** - * Read a Metadata string inside the ZIM file. - * @param {String} key - * @param {callbackMetadata} callback - */ - ZIMArchive.prototype.getMetadata = function (key, callback) { - var that = this; - this.getDirEntryByPath("M/" + key).then(function (dirEntry) { - if (dirEntry === null || dirEntry === undefined) { - console.warn("Title M/" + key + " not found in the archive"); - callback(); - } else { - that.readUtf8File(dirEntry, function (dirEntryRead, data) { - callback(data); + /** + * Read a binary file. + * @param {DirEntry} dirEntry + * @param {callbackBinaryContent} callback + */ + ZIMArchive.prototype.readBinaryFile = function (dirEntry, callback) { + return dirEntry.readData().then(function (data) { + callback(dirEntry, data); + }); + }; + + /** + * Searches the URL pointer list of Directory Entries by pathname + * @param {String} path The pathname of the DirEntry that is required (namespace + filename) + * @return {Promise} A Promise that resolves to a Directory Entry, or null if not found. + */ + ZIMArchive.prototype.getDirEntryByPath = function (path) { + var that = this; + return util.binarySearch(0, this._file.entryCount, function (i) { + return that._file.dirEntryByUrlIndex(i).then(function (dirEntry) { + var url = dirEntry.namespace + '/' + dirEntry.url; + if (path < url) { + return -1; + } else if (path > url) { + return 1; + } else { + return 0; + } }); - } - }).catch(function (e) { - console.warn("Metadata with key " + key + " not found in the archive", e); - callback(); - }); - }; + }).then(function (index) { + if (index === null) return null; + return that._file.dirEntryByUrlIndex(index); + }).then(function (dirEntry) { + return dirEntry; + }); + }; + + /** + * + * @param {callbackDirEntry} callback + */ + ZIMArchive.prototype.getRandomDirEntry = function (callback) { + // Prefer an article-only (v1) title pointer list, if available + var articleCount = this._file.articleCount || this._file.entryCount; + var index = Math.floor(Math.random() * articleCount); + this._file.dirEntryByTitleIndex(index).then(callback); + }; - /** - * Functions and classes exposed by this module - */ - return { - ZIMArchive: ZIMArchive - }; -}); + /** + * Read a Metadata string inside the ZIM file. + * @param {String} key + * @param {callbackMetadata} callback + */ + ZIMArchive.prototype.getMetadata = function (key, callback) { + var that = this; + this.getDirEntryByPath('M/' + key).then(function (dirEntry) { + if (dirEntry === null || dirEntry === undefined) { + console.warn('Title M/' + key + ' not found in the archive'); + callback(); + } else { + that.readUtf8File(dirEntry, function (dirEntryRead, data) { + callback(data); + }); + } + }).catch(function (e) { + console.warn('Metadata with key ' + key + ' not found in the archive', e); + callback(); + }); + }; + + /** + * Functions and classes exposed by this module + */ + return { + ZIMArchive: ZIMArchive + }; + } +); diff --git a/www/js/lib/zimArchiveLoader.js b/www/js/lib/zimArchiveLoader.js index 3cb1a01f9..577076764 100644 --- a/www/js/lib/zimArchiveLoader.js +++ b/www/js/lib/zimArchiveLoader.js @@ -19,73 +19,76 @@ * You should have received a copy of the GNU General Public License * along with Kiwix (file LICENSE-GPLv3.txt). If not, see */ + 'use strict'; + +/* global define */ + define(['zimArchive', 'jquery'], - function(zimArchive, jQuery) { + function (zimArchive, jQuery) { + /** + * Create a ZIMArchive from DeviceStorage location + * @param {DeviceStorage} storage + * @param {String} path + * @param {callbackZIMArchive} callbackReady + * @param {callbackZIMArchive} callbackError + * @returns {ZIMArchive} + */ + function loadArchiveFromDeviceStorage (storage, path, callbackReady, callbackError) { + return new zimArchive.ZIMArchive(storage, path, callbackReady, callbackError); + }; + /** + * Create a ZIMArchive from Files + * @param {Array.} files + * @param {callbackZIMArchive} callbackReady + * @param {callbackZIMArchive} callbackError + * @returns {ZIMArchive} + */ + function loadArchiveFromFiles (files, callbackReady, callbackError) { + if (files.length >= 1) { + return new zimArchive.ZIMArchive(files, null, callbackReady, callbackError); + } + }; + + /** + * @callback callbackPathList + * @param {Array.} directoryList List of directories + */ - /** - * Create a ZIMArchive from DeviceStorage location - * @param {DeviceStorage} storage - * @param {String} path - * @param {callbackZIMArchive} callbackReady - * @param {callbackZIMArchive} callbackError - * @returns {ZIMArchive} - */ - function loadArchiveFromDeviceStorage(storage, path, callbackReady, callbackError) { - return new zimArchive.ZIMArchive(storage, path, callbackReady, callbackError); - }; - /** - * Create a ZIMArchive from Files - * @param {Array.} files - * @param {callbackZIMArchive} callbackReady - * @param {callbackZIMArchive} callbackError - * @returns {ZIMArchive} - */ - function loadArchiveFromFiles(files, callbackReady, callbackError) { - if (files.length >= 1) { - return new zimArchive.ZIMArchive(files, null, callbackReady, callbackError); - } - }; - - /** - * @callback callbackPathList - * @param {Array.} directoryList List of directories - */ - - - /** - * Scans the DeviceStorage for archives - * - * @param {Array.} storages List of DeviceStorage instances - * @param {callbackPathList} callbackFunction Function to call with the list of directories where archives are found - * @param {callbackPathList} callbackError Function to call in case of an error - */ - function scanForArchives(storages, callbackFunction, callbackError) { - var directories = []; - var promises = jQuery.map(storages, function(storage) { - return storage.scanForArchives() - .then(function(dirs) { - jQuery.merge(directories, dirs); - return true; - }); - }); - jQuery.when.apply(null, promises).then(function() { - callbackFunction(directories); - }).catch(function (error) { - callbackError("Error scanning your device storage : " + error - + ". If you're using the Firefox OS Simulator, please put the archives in " - + "a 'fake-sdcard' directory inside your Firefox profile " - + "(ex : ~/.mozilla/firefox/xxxx.default/extensions/fxos_2_x_simulator@mozilla.org/" - + "profile/fake-sdcard/wikipedia_en_ray_charles_2015-06.zim)", "Error reading Device Storage"); - }); - }; + /** + * Scans the DeviceStorage for archives + * + * @param {Array.} storages List of DeviceStorage instances + * @param {callbackPathList} callbackFunction Function to call with the list of directories where archives are found + * @param {callbackPathList} callbackError Function to call in case of an error + */ + function scanForArchives (storages, callbackFunction, callbackError) { + var directories = []; + var promises = jQuery.map(storages, function (storage) { + return storage.scanForArchives() + .then(function (dirs) { + jQuery.merge(directories, dirs); + return true; + }); + }); + jQuery.when.apply(null, promises).then(function () { + callbackFunction(directories); + }).catch(function (error) { + callbackError('Error scanning your device storage : ' + error + + ". If you're using the Firefox OS Simulator, please put the archives in " + + "a 'fake-sdcard' directory inside your Firefox profile " + + '(ex : ~/.mozilla/firefox/xxxx.default/extensions/fxos_2_x_simulator@mozilla.org/' + + 'profile/fake-sdcard/wikipedia_en_ray_charles_2015-06.zim)', 'Error reading Device Storage'); + }); + }; - /** - * Functions and classes exposed by this module - */ - return { - loadArchiveFromDeviceStorage: loadArchiveFromDeviceStorage, - loadArchiveFromFiles: loadArchiveFromFiles, - scanForArchives: scanForArchives - }; -}); + /** + * Functions and classes exposed by this module + */ + return { + loadArchiveFromDeviceStorage: loadArchiveFromDeviceStorage, + loadArchiveFromFiles: loadArchiveFromFiles, + scanForArchives: scanForArchives + }; + } +); diff --git a/www/js/lib/zimDirEntry.js b/www/js/lib/zimDirEntry.js index aff3233af..ee1aa2705 100644 --- a/www/js/lib/zimDirEntry.js +++ b/www/js/lib/zimDirEntry.js @@ -19,33 +19,36 @@ * You should have received a copy of the GNU General Public License * along with Kiwix (file LICENSE-GPLv3.txt). If not, see */ + 'use strict'; -define([], function() { - + +/* global define */ + +define([], function () { /** * A Directory Entry in a ZIM File - * + * * See https://wiki.openzim.org/wiki/ZIM_file_format#Directory_Entries - * + * * @typedef DirEntry * @property {File} _zimfile The ZIM file * @property {Boolean} redirect * @property {Integer} offset * @property {Integer} mimetypeInteger MIME type number as defined in the MIME type list - * @property {String} namespace defines to which namespace this directory entry belongs + * @property {String} namespace defines to which namespace this directory entry belongs * @property {Integer} redirectTarget - * @property {Integer} cluster cluster number in which the data of this directory entry is stored - * @property {Integer} blob blob number inside the compressed cluster where the contents are stored + * @property {Integer} cluster cluster number in which the data of this directory entry is stored + * @property {Integer} blob blob number inside the compressed cluster where the contents are stored * @property {String} url string with the URL as refered in the URL pointer list * @property {String} title string with a title as refered in the Title pointer list (or empty) - * + * */ - + /** * @param {File} zimfile * @param {unresolved} dirEntryData - */ - function DirEntry(zimfile, dirEntryData) { + */ + function DirEntry (zimfile, dirEntryData) { this._zimfile = zimfile; this.redirect = dirEntryData.redirect; this.offset = dirEntryData.offset; @@ -61,40 +64,40 @@ define([], function() { /** * Serialize some attributes of a DirEntry, to be able to store them in a HTML tag attribute, * and retrieve them later. - * + * * @returns {String} */ - DirEntry.prototype.toStringId = function() { - //@todo also store isRedirect and redirectTarget + DirEntry.prototype.toStringId = function () { + // @todo also store isRedirect and redirectTarget return this.offset + '|' + this.mimetypeInteger + '|' + this.namespace + '|' + this.cluster + '|' + this.blob + '|' + this.url + '|' + this.title + '|' + this.redirect + '|' + this.redirectTarget; }; - + /** - * + * * @returns {Boolean} */ - DirEntry.prototype.isRedirect = function() { + DirEntry.prototype.isRedirect = function () { return this.redirect; }; - + /** - * + * * @returns {Promise} */ - DirEntry.prototype.readData = function() { + DirEntry.prototype.readData = function () { return this._zimfile.blob(this.cluster, this.blob); }; /** - * + * * @param {File} zimfile * @param {String} stringId * @returns {DirEntry} */ - DirEntry.fromStringId = function(zimfile, stringId) { + DirEntry.fromStringId = function (zimfile, stringId) { var data = {}; - var idParts = stringId.split("|"); + var idParts = stringId.split('|'); data.offset = parseInt(idParts[0], 10); data.mimetypeInteger = parseInt(idParts[1], 10); data.namespace = idParts[2]; @@ -102,7 +105,7 @@ define([], function() { data.blob = parseInt(idParts[4], 10); data.url = idParts[5]; data.title = idParts[6]; - data.redirect = ( idParts[7] === "true" ); + data.redirect = (idParts[7] === 'true'); data.redirectTarget = idParts[8]; return new DirEntry(zimfile, data); }; @@ -110,19 +113,19 @@ define([], function() { /** * Defines a function that returns the URL if the title is empty, as per the specification * See https://wiki.openzim.org/wiki/ZIM_file_format#Directory_Entries - * - * @returns {String} The dirEntry's title or, if empty, the dirEntry's (unescaped) URL + * + * @returns {String} The dirEntry's title or, if empty, the dirEntry's (unescaped) URL */ - DirEntry.prototype.getTitleOrUrl = function() { + DirEntry.prototype.getTitleOrUrl = function () { return this.title ? this.title : this.url; }; - + /** * Looks up the dirEntry's mimetype number in the ZIM file's MIME type list, and returns the corresponding MIME type - * + * * @return {String} The MIME type corresponding to mimetypeInteger in the ZIM file's MIME type list */ - DirEntry.prototype.getMimetype = function() { + DirEntry.prototype.getMimetype = function () { return this._zimfile.mimeTypes.get(this.mimetypeInteger); }; diff --git a/www/js/lib/zimfile.js b/www/js/lib/zimfile.js index 582088bb8..11f5f535d 100644 --- a/www/js/lib/zimfile.js +++ b/www/js/lib/zimfile.js @@ -21,6 +21,8 @@ */ 'use strict'; +/* global define, params */ + /** * This code makes an assumption that no Directory Entry will be larger that MAX_SUPPORTED_DIRENTRY_SIZE bytes. * If a larger dirEntry is encountered, a warning will display in console. Increase this value if necessary. @@ -37,8 +39,8 @@ const MAX_SUPPORTED_DIRENTRY_SIZE = 5120; */ if (!String.prototype.startsWith) { Object.defineProperty(String.prototype, 'startsWith', { - value: function(search, rawPos) { - var pos = rawPos > 0 ? rawPos|0 : 0; + value: function (search, rawPos) { + var pos = rawPos > 0 ? rawPos | 0 : 0; return this.substring(pos, pos + search.length) === search; } }); @@ -58,12 +60,11 @@ params.decompressorAPI = { errorStatus: null }; -define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'filecache'], function(xz, zstd, util, utf8, zimDirEntry, FileCache) { - +define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'filecache'], function (xz, zstd, util, utf8, zimDirEntry, FileCache) { /** * A variable to keep track of the currently loaded ZIM archive, e.g., for labelling cache entries * The ID is temporary and is reset to 0 at each session start; it is incremented by 1 each time a new ZIM is loaded - * @type {Integer} + * @type {Integer} */ var tempFileId = 0; @@ -81,13 +82,13 @@ define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'file } return r; }; - + /** * A ZIM File - * + * * See https://wiki.openzim.org/wiki/ZIM_file_format#Header * Some properties below are extended and are not part of the official OpenZIM specification - * + * * @typedef {Object} ZIMFile * @property {Array} _files Array of ZIM files * @property {String} name Abstract archive name for file set @@ -107,12 +108,12 @@ define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'file * @property {String} zimType Extended property: currently either 'open' for OpenZIM file type, or 'zimit' for the warc2zim file type used by Zimit (set in zimArchive.js) * @property {Map} mimeTypes Extended property: the ZIM file's MIME type table rendered as a Map (calculated entry) */ - + /** * Abstract an array of one or more (split) ZIM archives * @param {Array} abstractFileArray An array of ZIM file parts */ - function ZIMFile(abstractFileArray) { + function ZIMFile (abstractFileArray) { this._files = abstractFileArray; } @@ -120,7 +121,7 @@ define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'file * Read and decode an integer value from the ZIM archive * @param {Integer} offset The offset at which the integer is found * @param {Integer} size The size of data to read - * @returns {Promise} A Promise for the returned value + * @returns {Promise} A Promise for the returned value */ ZIMFile.prototype._readInteger = function (offset, size) { return this._readSlice(offset, size).then(function (data) { @@ -134,15 +135,15 @@ define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'file * @param {Integer} size The number of bytes to read * @returns {Promise} A Promise for a Uint8Array containing the requested data */ - ZIMFile.prototype._readSlice = function(offset, size) { + ZIMFile.prototype._readSlice = function (offset, size) { return FileCache.read(this, offset, offset + size); }; /** * Read a slice from a set of one or more ZIM files constituting a single archive, and concatenate the data parts - * @param {Integer} begin The absolute byte offset from which to start reading + * @param {Integer} begin The absolute byte offset from which to start reading * @param {Integer} end The absolute byte offset where reading should stop (the end byte is not read) - * @returns {Promise} A Promise for a Uint8Array containing the concatenated data + * @returns {Promise} A Promise for a Uint8Array containing the concatenated data */ ZIMFile.prototype._readSplitSlice = function (begin, end) { var file = this; @@ -308,14 +309,14 @@ define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'file * @typedef {Object} DirListing A list of pointers to directory entries (via the URL pointerlist) * @property {String} path The path (url) to the directory entry for the Listing * @property {String} ptrName The name of the pointer to the Listing's data that will be added to the ZIMFile obect - * @property {String} countName The name of the key that will contain the number of entries in the Listing, to be added to the ZIMFile object + * @property {String} countName The name of the key that will contain the number of entries in the Listing, to be added to the ZIMFile object */ /** * Read the metadata (archive offset pointer, and number of entiries) of one or more ZIM directory Listings. * This supports reading a subset of user content that might be ordered differently from the main URL pointerlist. * In particular, it supports the v1 article pointerlist, which contains articles sorted by title, superseding the article - * namespace ('A') in legazy ZIM archives. + * namespace ('A') in legazy ZIM archives. * @param {Array} listings An array of DirListing objects (see zimArchive.js for examples) * @returns {Promise} A promise that populates calculated entries in the ZIM file header */ @@ -327,8 +328,8 @@ define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'file // console.debug('ZIM DirListing version: 0 (legacy)', this); // Initiate a binary search for the first or last article var getArticleIndexByOrdinal = function (ordinal) { - return util.binarySearch(0, that.entryCount, function(i) { - return that.dirEntryByTitleIndex(i).then(function(dirEntry) { + return util.binarySearch(0, that.entryCount, function (i) { + return that.dirEntryByTitleIndex(i).then(function (dirEntry) { var ns = dirEntry.namespace; var url = ns + '/' + dirEntry.getTitleOrUrl(); var prefix = ordinal === 'first' ? 'A' : 'B'; @@ -336,12 +337,12 @@ define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'file else if (prefix > ns) return 1; return prefix < url ? -1 : 1; }); - }, true).then(function(index) { + }, true).then(function (index) { return index; }); }; - getArticleIndexByOrdinal('first').then(function(idxFirstArticle) { - return getArticleIndexByOrdinal('last').then(function(idxLastArticle) { + getArticleIndexByOrdinal('first').then(function (idxFirstArticle) { + return getArticleIndexByOrdinal('last').then(function (idxLastArticle) { // Technically idxLastArticle points to the entry after the last article in the 'A' namespace, // We subtract the first from the last to get the number of entries in the 'A' namespace that.articlePtrPos = that.titlePtrPos + idxFirstArticle * 4; @@ -366,20 +367,21 @@ define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'file return listingAccessor(listings.pop()); } // Initiate a binary search for the listing URL - return util.binarySearch(0, that.entryCount, function(i) { - return that.dirEntryByUrlIndex(i).then(function(dirEntry) { - var url = dirEntry.namespace + "/" + dirEntry.url; - if (listing.path < url) + return util.binarySearch(0, that.entryCount, function (i) { + return that.dirEntryByUrlIndex(i).then(function (dirEntry) { + var url = dirEntry.namespace + '/' + dirEntry.url; + if (listing.path < url) { return -1; - else if (listing.path > url) + } else if (listing.path > url) { return 1; - else + } else { return 0; + } }); - }).then(function(index) { + }).then(function (index) { if (index === null) return null; return that.dirEntryByUrlIndex(index); - }).then(function(dirEntry) { + }).then(function (dirEntry) { if (!dirEntry) return null; // Detect a full text index if (/fulltext\//.test(dirEntry.url)) { @@ -387,7 +389,7 @@ define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'file } // Request the metadata for the blob represented by the dirEntry return that.blob(dirEntry.cluster, dirEntry.blob, true); - }).then(function(metadata) { + }).then(function (metadata) { // Note that we do not accept a listing if its size is 0, i.e. if it contains no data // (although this should not occur, we have been asked to handle it - see kiwix-js #708) if (metadata && metadata.size) { @@ -397,25 +399,25 @@ define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'file } // Get the next Listing return listingAccessor(listings.pop()); - }).catch(function(err) { + }).catch(function (err) { console.error('There was an error accessing a Directory Listing', err); }); }; return listingAccessor(listings.pop()); - }; + }; /** * Reads the whole MIME type list and returns it as a populated Map * The mimeTypeMap is extracted once after the user has picked the ZIM file * and is stored as ZIMFile.mimeTypes - * @param {File} file The ZIM file (or first file in array of files) from which the MIME type list + * @param {File} file The ZIM file (or first file in array of files) from which the MIME type list * is to be extracted * @param {Integer} mimeListPos The offset in at which the MIME type list is found * @param {Integer} urlPtrPos The offset of URL Pointer List in the archive * @returns {Promise} A promise for the MIME Type list as a Map */ - function readMimetypeMap(file, mimeListPos, urlPtrPos) { - var typeMap = new Map; + function readMimetypeMap (file, mimeListPos, urlPtrPos) { + var typeMap = new Map(); var size = urlPtrPos - mimeListPos; // ZIM archives produced since May 2020 relocate the URL Pointer List to the end of the archive // so we limit the slice size to max 1024 bytes in order to prevent reading the entire archive into an array buffer @@ -429,7 +431,7 @@ define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'file while (pos < size) { pos++; mimeString = utf8.parse(data.subarray(pos), true); - // If the parsed data is an empty string, we have reached the end of the MIME type list, so break + // If the parsed data is an empty string, we have reached the end of the MIME type list, so break if (!mimeString) break; // Store the parsed string in the Map typeMap.set(i, mimeString); @@ -442,7 +444,7 @@ define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'file return typeMap; }).catch(function (err) { console.error('Unable to read MIME type list', err); - return new Map; + return new Map(); }); } @@ -477,13 +479,13 @@ define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'file zf.majorVersion = readInt(header, 4, 2); // Not currently used by this implementation zf.minorVersion = readInt(header, 6, 2); // Used to determine the User Content namespace zf.entryCount = readInt(header, 24, 4); - zf.articleCount = null; // Calculated async by setListings() called from zimArchive.js + zf.articleCount = null; // Calculated async by setListings() called from zimArchive.js zf.clusterCount = readInt(header, 28, 4); zf.urlPtrPos = urlPtrPos; zf.titlePtrPos = readInt(header, 40, 8); zf.articlePtrPos = null; // Calculated async by setListings() zf.fullTextIndex = null; // Calculated async by setListings() - zf.fullTextIndexSize = null; // Calbulated async by setListings() + zf.fullTextIndexSize = null; // Calbulated async by setListings() zf.clusterPtrPos = readInt(header, 48, 8); zf.mimeListPos = mimeListPos; zf.mainPage = readInt(header, 64, 4); diff --git a/www/js/lib/zstddec_wrapper.js b/www/js/lib/zstddec_wrapper.js index 3c057c073..905774020 100644 --- a/www/js/lib/zstddec_wrapper.js +++ b/www/js/lib/zstddec_wrapper.js @@ -1,7 +1,7 @@ /** * zstddec_wrapper.js: Javascript wrapper around compiled zstd decompressor. * - * Copyright 2020 Jaifroid, Mossroy and contributors + * Copyright 2023 Jaifroid, Mossroy and contributors * License GPL v3: * * This file is part of Kiwix. @@ -19,8 +19,12 @@ * You should have received a copy of the GNU General Public License * along with Kiwix (file LICENSE-GPLv3.txt). If not, see */ + 'use strict'; +/* global define, params, ZD */ +/* eslint-disable no-multi-spaces */ + // DEV: Put your RequireJS definition in the rqDefZD array below, and any function exports in the function parenthesis of the define statement // We need to do it this way in order to load the wasm or asm versions of zstddec conditionally. Older browsers can only use the asm version // because they cannot interpret WebAssembly. @@ -40,7 +44,7 @@ if ('WebAssembly' in self) { rqDefZD.push('zstddec-asm'); } -define(rqDefZD, function(uiUtil) { +define(rqDefZD, function (uiUtil) { // DEV: zstddec.js has been compiled with `-s EXPORT_NAME="ZD" -s MODULARIZE=1` to avoid a clash with xzdec.js // Note that we include zstddec-wasm or zstddec-asm above in requireJS definition, but we cannot change the name in the function list // For explanation of loading method below to avoid conflicts, see https://github.com/emscripten-core/emscripten/blob/master/src/settings.js @@ -68,7 +72,7 @@ define(rqDefZD, function(uiUtil) { // Get a permanent decoder handle (pointer to control structure) // NB there is no need to change this handle even between ZIM loads: zstddeclib encourages re-using assigned structures zd._decHandle = zd._ZSTD_createDStream(); - // In-built function below provides a max recommended chunk size + // In-built function below provides a max recommended chunk size zd._chunkSize = zd._ZSTD_DStreamInSize(); // Change _chunkSize if you need a more conservative memory environment, but you may need to experiment with INITIAL_MEMORY // in zstddec.js (see below) for this to make any difference @@ -142,7 +146,7 @@ define(rqDefZD, function(uiUtil) { /** * @typedef Decompressor * @property {FileReader} _reader The filereader to use (uses plain blob reader defined in zimfile.js) - * @property {Integer} _inStreamPos The current known position in the steam of compressed bytes + * @property {Integer} _inStreamPos The current known position in the steam of compressed bytes * @property {Integer} _inStreamChunkedPos The position once the currently loaded chunk will have been consumed * @property {Integer} _outStreamPos The position in the decoded byte stream (offset from start of cluster) * @property {Array} _outDataBuf The buffer that stores decoded bytes (it is set to the requested blob's length, and when full, the data are returned) @@ -153,7 +157,7 @@ define(rqDefZD, function(uiUtil) { * @constructor * @param {FileReader} reader The reader used to extract file slices (defined in zimfile.js) */ - function Decompressor(reader) { + function Decompressor (reader) { params.decompressorAPI.decompressorLastUsed = 'ZSTD'; this._reader = reader; } @@ -174,7 +178,7 @@ define(rqDefZD, function(uiUtil) { this._outDataBufPos = 0; var ret = zd._ZSTD_initDStream(zd._decHandle); if (zd._ZSTD_isError(ret)) { - return Promise.reject('Failed to initialize ZSTD decompression'); + return Promise.reject(new Error('Failed to initialize ZSTD decompression')); } return this._readLoop(offset, length).then(function (data) { @@ -225,7 +229,7 @@ define(rqDefZD, function(uiUtil) { var finished = false; var ret = zd._ZSTD_decompressStream(zd._decHandle, zd._outBuffer.ptr, zd._inBuffer.ptr); if (zd._ZSTD_isError(ret)) { - var errorMessage = "Failed to decompress data stream!\n" + zd.getErrorString(ret); + var errorMessage = 'Failed to decompress data stream!\n' + zd.getErrorString(ret); return Promise.reject(errorMessage); } // Get updated outbuffer values @@ -236,8 +240,9 @@ define(rqDefZD, function(uiUtil) { if (outPos > 0 && that._outStreamPos + outPos >= offset) { var copyStart = offset - that._outStreamPos; if (copyStart < 0) copyStart = 0; - for (var i = copyStart; i < outPos && that._outDataBufPos < that._outDataBuf.length; i++) + for (var i = copyStart; i < outPos && that._outDataBufPos < that._outDataBuf.length; i++) { that._outDataBuf[that._outDataBufPos++] = zd.HEAP8[zd._outBuffer.dst + i]; + } } if (that._outDataBufPos === that._outDataBuf.length) finished = true; // Return without further processing if decompressor has finished @@ -293,7 +298,7 @@ define(rqDefZD, function(uiUtil) { * @param {Integer} sizeOfData The number of bytes to be allocated * @returns {Integer} Pointer to the assigned data block */ - function mallocOrDie(sizeOfData) { + function mallocOrDie (sizeOfData) { const dataPointer = zd._malloc(sizeOfData); if (dataPointer === 0) { // error allocating memory var errorMessage = 'Failed allocation of ' + sizeOfData + ' bytes.'; @@ -306,4 +311,4 @@ define(rqDefZD, function(uiUtil) { return { Decompressor: Decompressor }; -}); \ No newline at end of file +});