Webpack 4 -> 5 changes

package.json

@@ -38,14 +38,14 @@
     "@babel/preset-react": "7.18.6",
     "@babel/runtime-corejs3": "7.20.0",
     "@storybook/react": "6.3.7",
-    "antd": "^4.24.8",
-    "babel-loader": "8.1.0",
-    "babel-plugin-styled-components": "1.10.7",
+    "antd": "4.24.8",
+    "babel-loader": "8.2.2",
+    "babel-plugin-styled-components": "1.13.2",
     "babel-plugin-transform-remove-strict-mode": "0.0.2",
     "cesium": "1.91.0",
-    "copy-webpack-plugin": "5.0.4",
+    "copy-webpack-plugin": "9.0.1",
     "core-js": "3.26.0",
-    "css-loader": "3.2.0",
+    "css-loader": "6.2.0",
     "d3": "7.8.4",
     "dompurify": "2.3.6",
     "draft-js": "0.11.7",
@@ -59,40 +59,41 @@
     "eslint-plugin-node": "11.1.0",
     "eslint-plugin-promise": "6.1.1",
     "eslint-plugin-react": "7.32.2",
-    "expose-loader": "0.7.5",
-    "file-loader": "4.2.0",
-    "fs-extra": "8.1.0",
+    "expose-loader": "3.0.0",
+    "file-loader": "6.2.0",
+    "fs-extra": "10.0.0",
     "geostats": "2.0.0",
     "gm": "1.23.1",
-    "imports-loader": "0.8.0",
+    "imports-loader": "3.0.0",
     "intl-messageformat": "9.11.4",
     "jest": "27.0.6",
     "jquery": "3.6.0",
     "jsts": "2.2.0",
-    "less": "3.9.0",
-    "less-loader": "5.0.0",
-    "loader-utils": "1.2.3",
+    "less": "4.1.1",
+    "less-loader": "10.0.1",
+    "loader-utils": "2.0.0",
     "lodash": "4.17.21",
-    "merge": "1.2.1",
-    "mini-css-extract-plugin": "0.8.0",
+    "merge": "2.1.1",
+    "mini-css-extract-plugin": "2.2.0",
     "mobile-detect": "1.4.5",
     "node-sass": "6.0.1",
     "ol": "7.2.2",
     "olcs": "2.13.1",
-    "optimize-css-assets-webpack-plugin": "5.0.3",
+    "ol-mapbox-style": "6.4.1",
+    "optimize-css-assets-webpack-plugin": "6.0.1",
     "query-string": "6.8.3",
     "react": "16.14.0",
     "react-beautiful-dnd": "13.1.0",
     "react-dom": "16.14.0",
-    "sass-loader": "10.2.0",
+    "sass-loader": "12.1.0",
     "script-loader": "0.7.2",
-    "style-loader": "1.0.0",
+    "style-loader": "3.2.1",
     "styled-components": "5.3.3",
     "sumoselect": "3.0.5",
     "uglifyjs-webpack-plugin": "2.2.0",
-    "webpack": "4.43.0",
-    "webpack-cli": "3.3.11",
-    "webpack-dev-server": "3.8.0"
+    "webpack": "5.51.1",
+    "webpack-cli": "4.8.0",
+    "webpack-dev-server": "4.0.0"
   },
   "devDependencies": {
     "canvas": "2.8.0"

webpack.config.js

@@ -51,20 +51,21 @@ module.exports = (env, argv) => {
 
     // Common config for both prod & dev
     const config = {
-        node: {
-            fs: 'empty'
-        },
         amd: {
             // Enable webpack-friendly use of require in Cesium
             toUrlUndefined: true
         },
+        cache: {
+            type: 'filesystem'
+        },
         mode: isProd ? 'production' : 'development',
         entry: entries,
         devtool: isProd ? 'source-map' : 'cheap-module-eval-source-map',
         output: {
             path: path.resolve(`dist/${version}/`),
             publicPath: `${publicPathPrefix}Oskari/dist/${version}/`,
             filename: '[name]/oskari.min.js',
+            assetModuleFilename: 'assets/[hash][ext][query]',
 
             // Needed to compile multiline strings in Cesium
             sourcePrefix: ''

webpack/config.js

@@ -42,7 +42,9 @@ const getStyleFileRules = (isProd, antThemeFile) => {
                 { loader: 'css-loader' },
                 {
                     loader: 'less-loader',
-                    options: lessLoaderOptions
+                    options: {
+                        lessOptions: lessLoaderOptions
+                    }
                 }
             ]
         }
@@ -123,21 +125,21 @@ const getModuleRules = (isProd = false, antThemeFile) => {
     const rules = [
         {
             test: require.resolve('sumoselect'),
-            use: 'imports-loader?define=>undefined,exports=>undefined'
-        },
-        BABEL_LOADER_RULE,
-        ...styleFileRules,
-        {
-            test: /\.(ttf|png|jpg|gif|svg)$/,
             use: [
                 {
-                    loader: 'file-loader',
+                    loader: 'imports-loader',
                     options: {
-                        outputPath: 'assets/'
+                        imports: ["default jquery $"]
                     }
                 }
             ]
         },
+        BABEL_LOADER_RULE,
+        ...styleFileRules,
+        {
+            test: /\.(ttf|png|jpg|gif|svg)$/,
+            type: 'asset/resource'
+        },
         {
             type: 'javascript/auto',
             test: /minifierAppSetup.json$/,
@@ -158,6 +160,9 @@ const RESOLVE = {
     symlinks: false,
     alias: {
         'oskari-ui': path.resolve(__dirname, '../src/react')
+    },
+    fallback: {
+        fs: false
     }
 };
 const RESOLVE_LOADER = {

webpack/generateEntries.js

@@ -1,18 +1,16 @@
 const path = require('path');
 const { IgnorePlugin } = require('webpack');
 const CopyWebpackPlugin = require('copy-webpack-plugin');
-const LocalizationPlugin = require('./localizationPlugin.js');
+const LocalizationPlugin = require('./localizationPluginFor5.js');
 const { existsSync } = require('fs');
 
 module.exports = function generateEntries (appsetupPaths, isProd, context) {
     const entries = {};
     const plugins = [
-        new IgnorePlugin(/^\.\/locale$/, /moment$/),
-        new CopyWebpackPlugin(
-            [
-                { from: 'resources', to: 'resources', context }
-            ]
-        )
+        new IgnorePlugin(/^\.\/locale$/),
+        new CopyWebpackPlugin({
+            patterns: [{ from: 'resources', to: 'resources', context }]
+        })
     ];
 
     appsetupPaths.forEach(appDir => {
@@ -42,7 +40,7 @@ module.exports = function generateEntries (appsetupPaths, isProd, context) {
             path.resolve(context, './webpack/oskari-core.js'),
             targetPath
         ];
-        plugins.push(new CopyWebpackPlugin(copyDef));
+        plugins.push(new CopyWebpackPlugin({ patterns: copyDef }));
         plugins.push(new LocalizationPlugin(appName));
     });
 

webpack/localizationPluginFor5.js

@@ -0,0 +1,183 @@
+const fs = require('fs');
+const path = require('path');
+const merge = require('merge');;
+const { sources, Compilation } = require('webpack');
+
+const fileRex = /^(.{2,3})\.js$/;
+const pluginName = 'LocalizationPlugin'
+
+function isLocaleFile (filePath) {
+    if (path.basename(path.dirname(filePath)) !== 'locale') {
+        return false;
+    }
+    const parts = filePath.split(path.sep);
+    if (parts[parts.length -3] !== 'resources') {
+        // this could be things like:
+        // - oskari-frontend\node_modules\moment\locale\af.js
+        // - oskari-frontend\node_modules\antd\es\locale\en_US.js
+        // for oskari locs we only want to process files in paths like:
+        // - oskari-frontend\bundles\*\resources\locale\en.js
+        return false;
+    }
+    return fileRex.test(path.basename(filePath));
+}
+
+const langFromPath = (filePath) => {
+    const match = path.basename(filePath).match(fileRex);
+    return match ? match[1] : null;
+};
+
+const getLanguagesToWrite = (files) => {
+    const changedLanguages = new Set();
+    files.forEach(path => {
+        const lang = langFromPath(path);
+        if (lang) {
+            changedLanguages.add(lang);
+        }
+    });
+    return changedLanguages;
+};
+
+const readLocalizationContent = (localeFiles) => {
+    const changedLanguages = getLanguagesToWrite(localeFiles);
+
+    const langToLoc = new Map();
+    const langToOverride = new Map();
+    const allKeys = new Set();
+    /* eslint-disable-next-line */
+    const Oskari = {
+        registerLocalization: (loc, isOverride) => {
+            const lang = loc.lang;
+            if (!lang) {
+                throw new Error('Localization file has no "lang"!');
+            }
+            const collection = isOverride ? langToOverride : langToLoc;
+            let agg = new Map();
+            if (collection.has(lang)) {
+                agg = collection.get(lang);
+            } else {
+                collection.set(lang, agg);
+            }
+            agg.set(loc.key, loc);
+            allKeys.add(loc.key);
+        }
+    };
+    localeFiles
+        .filter(path => {
+            const lang = langFromPath(path);
+            return lang === 'en' || changedLanguages.has(lang); // Always process English as it might be needed as fallback
+        })
+        .forEach(path => {
+            const source = fs.readFileSync(path, 'utf8');
+            /* eslint-disable-next-line */
+            eval(source);
+        });
+
+    const englishLoc = langToLoc.get('en') || new Map();
+    const result = {};
+    for (const entry of langToLoc.entries()) {
+        const lang = entry[0];
+        const langLoc = entry[1];
+        const langOverride = langToOverride.get(lang) || new Map();
+
+        const keyContents = Array.from(allKeys)
+            .filter(key => englishLoc.has(key) || langLoc.has(key) || langOverride.has(key))
+            .map(key => {
+                const englishForKey = lang === 'en' ? {} : englishLoc.get(key) || {}; // don't merge English with itself
+                const locForKey = langLoc.get(key) || {};
+                const overrideForKey = langOverride.get(key) || {};
+
+                const mergedEnglish = merge.recursive(true, englishForKey, locForKey);
+                const mergedOverride = merge.recursive(true, mergedEnglish, overrideForKey);
+                mergedOverride.lang = lang; // value for "lang" key might be from fallback. Ensuring it's correct
+                return mergedOverride;
+            });
+
+        const fileContent = keyContents.map(content => `Oskari.registerLocalization(${JSON.stringify(content)});`).join('\n');
+        result[lang] = fileContent;
+    }
+    return result;
+}
+
+/**
+ * 1) Processes oskari-frontend/bundles/[bundle id]/resources/locale/[lang].js,
+ * 2) gathers the localizations to language specific files
+ * 3) and generates new "assets" for the build to write to dist/[version]/appName/oskari_lang_[lang].js
+ */
+class LocalizationPlugin {
+    constructor (appName) {
+        this.appPath = appName ? appName + '/' : '';
+    }
+    apply (compiler) {
+/*
+This implementation gives deprecation warning:
+        [DEP_WEBPACK_COMPILATION_ASSETS] DeprecationWarning: Compilation.assets will be frozen in future, all modifications are deprecated.
+        BREAKING CHANGE: No more changes should happen to Compilation.assets after sealing the Compilation.
+                Do changes to assets earlier, e. g. in Compilation.hooks.processAssets.
+                Make sure to select an appropriate stage from Compilation.PROCESS_ASSETS_STAGE_*.
+*/
+        compiler.hooks.emit.tap(pluginName, (compilation) => {
+            const localeFiles = Array.from(compilation.fileDependencies).filter(isLocaleFile);
+
+            const oskariLangContents = readLocalizationContent(localeFiles)
+            Object.keys(oskariLangContents).forEach(lang => {
+                const fileContent = oskariLangContents[lang];
+                compilation.emitAsset(`${this.appPath}oskari_lang_${lang}.js`, new sources.RawSource(fileContent));
+            });
+        });
+
+// Here's some links that might help updating the impl:
+//  problem so far is that the process assets only get "asset" files like svg/png or files from dependencies (moment/locale etc)
+//  The solution requires processing of the actual source files in the app and oskari-frontend/oskari-frontend-contrib etc
+// https://stackoverflow.com/questions/65535038/webpack-processassets-hook-and-asset-source
+// https://github.com/webpack/webpack/issues/11425
+// https://survivejs.com/webpack/extending/plugins/
+// https://webpack.js.org/api/compilation-hooks/#processassets
+/*
+        compiler.hooks.thisCompilation.tap(pluginName, (compilation) => {
+              compilation.hooks.processAssets.tap({
+                name: pluginName,
+                stage: Compilation.PROCESS_ASSETS_STAGE_ADDITIONAL,
+              }, () => {
+                    const localeFiles = getScriptFilesForChunks(compilation);
+                    const oskariLangContents = readLocalizationContent(localeFiles);
+                    Object.keys(oskariLangContents).forEach(lang => {
+                        const fileContent = oskariLangContents[lang];
+                        compilation.emitAsset(`${this.appPath}oskari_lang_${lang}.js`, new sources.RawSource(fileContent));
+                    });
+                },
+              );
+        });
+        */
+    }
+}
+// try to get file links when processing assets (so we get rid of deprecation warning)
+const getScriptFilesForChunks = (compilation) => {
+    const { chunks } = compilation.getStats().toJson({ chunks: true });
+    const { publicPath } = compilation.options.output;
+    const scriptFiles = new Set();
+
+    chunks.forEach(chunk => {
+        // seems to only give locales from moment and antd, not ones from "our" bundles
+        const fileNames = chunk.modules.map(mod => mod.issuerName)
+            .filter(name => !!name && name.indexOf('oskari-frontend') > -1 && name.indexOf('locale') > -1);
+        console.log(fileNames);
+        const before = scriptFiles.length;
+        fileNames
+            .filter(name => !!name && isLocaleFile(name))
+            .forEach(name => scriptFiles.add(name));
+        if (scriptFiles.length > before) {
+            const { modules, names, runtime, ...rest} = chunk;
+            console.log('chunk: ', names, runtime);
+        }
+    });
+
+    if (scriptFiles.size === 0) {
+        compilation.warnings.push(`There were no assets matching ` +
+            `importScriptsViaChunks: [meh].`);
+    }
+
+    return Array.from(scriptFiles);
+};
+
+module.exports = LocalizationPlugin;

webpack/oskariLoader.js

@@ -41,7 +41,7 @@ module.exports = function (source) {
 
     const output = source + '\n' + dependencies.map(d => {
         if (d.expose) {
-            return `import 'expose-loader?${d.expose}!${d.src}'`;
+            return `import 'expose-loader?exposes=${d.expose}!${d.src}'`;
         }
         return `import '${d.src}'`;
     }).join('\n');