Recompile Emscripten binaries, implement WASM with fallback, add API …

…info, add compile scripts (#740)

Fixes #739, #652 and #511. Instructions and helper scripts are included for easy future recompiling of the binaries with docker.

.gitattributes

@@ -40,3 +40,4 @@
 *.woff	binary
 *.woff2	binary
 *.deb   binary
+*.wasm  binary

emscripten/xzdec/compile.sh

@@ -1 +1,5 @@
-emcc --memory-init-file 0 -O3 -s WASM=0 -s MALLOC="emmalloc" -s TOTAL_MEMORY=83886080 -s NO_FILESYSTEM=1 -s AGGRESSIVE_VARIABLE_ELIMINATION=1 -s DOUBLE_MODE=0 -s PRECISE_I64_MATH=0 -s NO_DYNAMIC_EXECUTION=1 -s LEGACY_VM_SUPPORT=1 -s EXPORTED_FUNCTIONS="['_init', '_init_decompression', '_input_empty', '_get_in_buffer', '_set_new_input', '_decompress', '_get_out_pos', '_get_out_buffer', '_out_buffer_cleared', '_release']" -DXZ_USE_CRC64=1 -DXZ_INTERNAL_CRC64=1 *.c -o xzdec.js
+echo "Compiling ASM version xzdec-asm.js"
+emcc --memory-init-file 0 -O3 --closure 1 -s ENVIRONMENT="web" -s WASM=0 -s MALLOC="emmalloc" -s TOTAL_MEMORY=83886080 -s FILESYSTEM=0 -s DOUBLE_MODE=0 -s DYNAMIC_EXECUTION=0 -s MIN_IE_VERSION=11 -s EXPORT_NAME="XZ" -s MODULARIZE=1 -s AGGRESSIVE_VARIABLE_ELIMINATION=1 -s EXPORTED_FUNCTIONS="['_init', '_init_decompression', '_input_empty', '_get_in_buffer', '_set_new_input', '_decompress', '_get_out_pos', '_get_out_buffer', '_out_buffer_cleared', '_release']" -DXZ_USE_CRC64=1 -DXZ_INTERNAL_CRC64=1 ./*.c -o xzdec-asm.js
+echo "Compiling WASM version xzdec-wasm.js"
+emcc --memory-init-file 0 -O3 --closure 1 -s ENVIRONMENT="web" -s WASM=1 -s MALLOC="emmalloc" -s TOTAL_MEMORY=83886080 -s FILESYSTEM=0 -s DOUBLE_MODE=0 -s DYNAMIC_EXECUTION=0 -s EXPORT_NAME="XZ" -s MODULARIZE=1 -s AGGRESSIVE_VARIABLE_ELIMINATION=1 -s EXPORTED_FUNCTIONS="['_init', '_init_decompression', '_input_empty', '_get_in_buffer', '_set_new_input', '_decompress', '_get_out_pos', '_get_out_buffer', '_out_buffer_cleared', '_release']" -DXZ_USE_CRC64=1 -DXZ_INTERNAL_CRC64=1 ./*.c -o xzdec-wasm.js
+echo "Finished."

emscripten/zstandard/compile.sh

@@ -1 +1,5 @@
-emcc --memory-init-file 0 -O3 --closure 1 -s ENVIRONMENT="web" -s WASM=0 -s MALLOC="emmalloc" -s TOTAL_MEMORY=140247040 -s FILESYSTEM=0 -s DOUBLE_MODE=0 -s DYNAMIC_EXECUTION=0 -s MIN_IE_VERSION=11 -s EXPORT_NAME="ZD" -s MODULARIZE=1 -s EXPORTED_FUNCTIONS="['_malloc', '_free', '_ZSTD_createDStream', '_ZSTD_initDStream', '_ZSTD_decompressStream', '_ZSTD_isError', '_ZSTD_getErrorName', '_ZSTD_freeDStream', '_ZSTD_DStreamInSize', '_ZSTD_DStreamOutSize']" -s EXPORTED_RUNTIME_METHODS="['cwrap']" *.c -o zstddec.js
+echo "Compiling ASM version zstddec-asm.js"
+emcc --memory-init-file 0 -O3 --closure 1 -s ENVIRONMENT="web" -s WASM=0 -s MALLOC="emmalloc" -s TOTAL_MEMORY=150994944 -s FILESYSTEM=0 -s DOUBLE_MODE=0 -s DYNAMIC_EXECUTION=0 -s MIN_IE_VERSION=11 -s EXPORT_NAME="ZD" -s MODULARIZE=1 -s EXPORTED_FUNCTIONS="['_malloc', '_free', '_ZSTD_createDStream', '_ZSTD_initDStream', '_ZSTD_decompressStream', '_ZSTD_isError', '_ZSTD_getErrorName', '_ZSTD_freeDStream', '_ZSTD_DStreamInSize', '_ZSTD_DStreamOutSize']" -s EXPORTED_RUNTIME_METHODS="['cwrap']" ./*.c -o zstddec-asm.js
+echo "Compiling WASM version zstddec-wasm.js"
+emcc --memory-init-file 0 -O3 --closure 1 -s ENVIRONMENT="web" -s WASM=1 -s MALLOC="emmalloc" -s TOTAL_MEMORY=150994944 -s FILESYSTEM=0 -s DOUBLE_MODE=0 -s DYNAMIC_EXECUTION=0 -s EXPORT_NAME="ZD" -s MODULARIZE=1 -s EXPORTED_FUNCTIONS="['_malloc', '_free', '_ZSTD_createDStream', '_ZSTD_initDStream', '_ZSTD_decompressStream', '_ZSTD_isError', '_ZSTD_getErrorName', '_ZSTD_freeDStream', '_ZSTD_DStreamInSize', '_ZSTD_DStreamOutSize']" -s EXPORTED_RUNTIME_METHODS="['cwrap']" ./*.c -o zstddec-wasm.js
+echo "Finished."

emscripten/zstandard/readme.md

@@ -5,4 +5,6 @@ It has been combined by following the instructions at https://github.com/faceboo
 
 The source code can be compiled with Emscripten by setting up the Emscripten SDK and running the Linux script `./compile.sh` in this directory. This will produce `zstddec.js`. Further optimization and minification of this file may be possible by adjusting the commandline options in `compile.sh`.
 
+You may be able to compile easily using docker. Look in the `/scripts` directory for helper scripts for your platform.
+
 For a fuller Emscripten distribution of the full zstandard library for `node.js`, see https://github.com/yoshihitoh/zstd-codec/tree/develop/js/example.

scripts/Compile-Xzdec.ps1

@@ -0,0 +1,14 @@
+# This PowerShell script (principally for Windows 10/11) compiles the legacy xzdec.js using docker according to the values set in 
+# /emscripten/xzdec/compile.sh . Please be sure to review the commandline in compile.sh before running this script, to
+# be sure it's doing what you want. If you want to compile with a later version of emnsdk, specify it below after 'emsdk:'.
+#
+# Prerequisites:
+#
+#     * Ensure WSL 2 is enabled in Windows (WSL = Windows Subsystem for Linux) 
+#     * Install docker desktop on Windows: choose the WSL 2 docker backend (not the legacy Hyper-V one)
+#     * Open docker and ensure the docker daemon is running correctly (icon should be green)
+#     * Ensure docker desktop is signed in to your dockerhub account (at least the first time you run this script)
+#     * Run this script by opening a PowerShell terminal, cd to the kiwix-js repo and run './scripts/Compile-Zstddec.ps1'
+
+$repo_dir = ($PSScriptRoot -replace '[\\/]scripts[\\/]*$', '')
+docker container run -v $repo_dir\:/project -w /project emscripten/emsdk:2.0.25 /bin/sh -c 'cd emscripten/xzdec/; ./compile.sh'

scripts/Compile-Zstddec.ps1

@@ -0,0 +1,14 @@
+# This PowerShell script (principally for Windows 10/11) compiles zstddec.js using docker according to the values set in 
+# /emscripten/ztandard/compile.sh . Please be sure to review the commandline in compile.sh before running this script, to
+# be sure it's doing what you want. If you want to compile with a later version of emnsdk, specify it below after 'emsdk:'.
+#
+# Prerequisites:
+#
+#     * Ensure WSL 2 is enabled in Windows (WSL = Windows Subsystem for Linux) 
+#     * Install docker desktop on Windows: choose the WSL 2 docker backend (not the legacy Hyper-V one)
+#     * Open docker and ensure the docker daemon is running correctly (icon should be green)
+#     * Ensure docker desktop is signed in to your dockerhub account (at least the first time you run this script)
+#     * Run this script by opening a PowerShell terminal, cd to the kiwix-js repo and run './scripts/Compile-Zstddec.ps1'
+
+$repo_dir = ($PSScriptRoot -replace '[\\/]scripts[\\/]*$', '')
+docker container run -v $repo_dir\:/project -w /project emscripten/emsdk:2.0.25 /bin/sh -c 'cd emscripten/zstandard/; ./compile.sh'

scripts/compile_xzdec_with_docker_emscripten.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# This bash script compiles xzdec.js using emscripten docker image, according to the values set in 
+# /emscripten/xzdec/compile.sh . Please be sure to review the commandline in compile.sh before running this script, to
+# be sure it's doing what you want.
+#
+# Prerequisites:
+#
+#     * Ensure Docker is installed and its daemon is running
+#     * If the current user is not in the docker group, you might need to run this script with sudo. See https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user
+
+# Find the repo dir (it's the parent of the dir that contains this script)
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+REPO_DIR="$(dirname "$SCRIPT_DIR")"
+
+# Run the emscripten compilation through docker
+docker run -v "$REPO_DIR":/project -w /project -u $(id -u):$(id -g) emscripten/emsdk:2.0.25 /bin/sh -c 'cd emscripten/xzdec/; sh ./compile.sh'