Port marian to WASM for inference

.github/workflows/macos-wasm.yml

@@ -0,0 +1,56 @@
+name: MacOS WASM
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build-wasm:
+    name: WASM
+    runs-on: macos-10.15
+
+    steps:
+    - name: Setup Emscripten toolchain
+      uses: mymindstorm/setup-emsdk@v8
+
+    - name: Verify Emscripten setup
+      run: emcc -v
+
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Configure builds
+      run: |
+        mkdir -p build-wasm
+        cd build-wasm
+        emcmake cmake \
+          -DCOMPILE_CUDA=off \
+          -DUSE_DOXYGEN=off \
+          -DCOMPILE_EXAMPLES=off \
+          -DCOMPILE_SERVER=off \
+          -DCOMPILE_TESTS=off \
+          -DUSE_FBGEMM=off \
+          -DUSE_SENTENCEPIECE=on \
+          -DUSE_STATIC_LIBS=on \
+          -DUSE_MKL=off \
+          -DCOMPILE_WASM=on ../
+
+    - name: Compile
+      working-directory: build-wasm
+      run: emmake make -j2
+
+    - name: Check artifacts
+      working-directory: build-wasm
+      run: |
+        ls -all .
+        if ls marian-decoder.wasm &>/dev/null && ls marian-decoder.js &>/dev/null
+        then
+          echo "Artifacts Successfully Generated"
+        else
+          echo "Failure: Artifacts Not Present"
+          exit 1
+        fi

.gitmodules

@@ -6,7 +6,7 @@
 	url = https://github.com/marian-nmt/marian-regression-tests
 [submodule "src/3rd_party/sentencepiece"]
 	path = src/3rd_party/sentencepiece
-	url = https://github.com/marian-nmt/sentencepiece
+	url = https://github.com/browsermt/sentencepiece
 [submodule "src/3rd_party/nccl"]
 	path = src/3rd_party/nccl
 	url = https://github.com/marian-nmt/nccl
@@ -16,7 +16,10 @@
 	branch = master
 [submodule "src/3rd_party/intgemm"]
 	path = src/3rd_party/intgemm
-	url = https://github.com/kpu/intgemm/
+	url = https://github.com/kpu/intgemm
 [submodule "src/3rd_party/simple-websocket-server"]
 	path = src/3rd_party/simple-websocket-server
 	url = https://github.com/marian-nmt/Simple-WebSocket-Server
+[submodule "src/3rd_party/onnxjs"]
+	path = src/3rd_party/onnxjs
+	url = https://github.com/abhi-agg/onnxjs.git

CHANGELOG.md

@@ -32,6 +32,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Quantized training (fixed point or log-based quantization) with --quantize-bits N command
 - Support for loading lexical shortlist from a binary blob
 - Integrate a shortlist converter (which can convert a text lexical shortlist to a binary shortlist) into marian-conv with --shortlist option
+- Added ONNXJS submodule to use its wasm-compatible sgemm routine for wasm builds
+- Enable compiling marian on wasm platform
 
 ### Fixed
 - Segfault of spm_train when compiled with -DUSE_STATIC_LIBS=ON seems to have gone away with update to newer SentencePiece version.
@@ -55,8 +57,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Added default "none" for option shuffle in BatchGenerator, so that it works in executables where shuffle is not an option.
 - Added a few missing header files in shortlist.h and beam_search.h.
 - Improved handling for receiving SIGTERM during training. By default, SIGTERM triggers 'save (now) and exit'. Prior to this fix, batch pre-fetching did not check for this sigal, potentially delaying exit considerably. It now pays attention to that. Also, the default behaviour of save-and-exit can now be disabled on the command line with --sigterm exit-immediately.
+- Fix the runtime failures for FASTOPT on 32-bit builds (wasm just happens to be 32-bit) because it uses hashing with an inconsistent mix of uint64_t and size_t.
+- Fix loading the binary model on 32-bit builds and for wasm platform
 
 ### Changed
+- Updated intgemm repository to version 1a176394bb0c2d243c42fe574e063924a92aa120 from https://github.com/kpu/intgemm.
+- Changed SentencePiece repository from https://github.com/google/sentencepiece to the version 3ffdc0065a03cadd9d0e5e123aaf9b6ea7ffb05d of https://github.com/browsermt/sentencepiece.
 - Updated SentencePiece repository to version 8336bbd0c1cfba02a879afe625bf1ddaf7cd93c5 from https://github.com/google/sentencepiece. 
 - Enabled compilation of SentencePiece by default since no dependency on protobuf anymore. 
 - Changed default value of --sentencepiece-max-lines from 10000000 to 2000000 since apparently the new version doesn't sample automatically anymore (Not quite clear how that affects quality of the vocabulary).
@@ -68,6 +74,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Changed compile flags -Ofast to -O3 and remove --ffinite-math
 - Moved old graph groups to depracated folder
 - Make cublas and cusparse handle inits lazy to save memory when unused
+- Replaced exception-based implementation for type determination in FastOpt::makeScalar
 
 ## [1.9.0] - 2020-03-10
 

CMakeLists.txt

@@ -8,7 +8,8 @@ endif ()
 project(marian CXX C)
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(BUILD_ARCH native CACHE STRING "Compile for this CPU architecture.")
+
+include(CMakeDependentOption)
 
 # Custom CMake options
 option(COMPILE_CPU "Compile CPU version" ON)
@@ -26,6 +27,41 @@ option(USE_NCCL "Use NCCL library" ON)
 option(USE_SENTENCEPIECE "Download and compile SentencePiece" ON)
 option(USE_STATIC_LIBS "Link statically against non-system libs" OFF)
 option(GENERATE_MARIAN_INSTALL_TARGETS "Generate Marian install targets (requires CMake 3.12+)" OFF)
+option(M32_BINARIES "Generate 32bit binaries even when building outside of WASM. Useful for testing some WASM specific functionality without the need for the compiling to WASM." OFF)
+option(COMPILE_WASM "Compile customized marian for WASM target" OFF)
+
+# cmake options that are dependent on COMPILE_WASM cmake option
+CMAKE_DEPENDENT_OPTION(USE_THREADS "Compile with multi-threading support" OFF
+                       "COMPILE_WASM" ON)
+CMAKE_DEPENDENT_OPTION(USE_WASM_COMPATIBLE_BLAS "Compile with wasm compatible blas" ON
+                       "COMPILE_WASM" OFF)
+CMAKE_DEPENDENT_OPTION(COMPILE_WITHOUT_EXCEPTIONS "Compile without exceptions" ON
+                       "COMPILE_WASM" OFF)
+
+if (COMPILE_WASM)
+  set(WORMHOLE ON CACHE BOOL "Use WASM wormhole in intgemm https://bugzilla.mozilla.org/show_bug.cgi?id=1672160")
+endif()
+
+if(M32_BINARIES OR COMPILE_WASM)
+    set("BUILD_WIDTH" "-m32")
+else(M32_BINARIES OR COMPILE_WASM)
+    set("BUILD_WIDTH" "-m64")
+endif()
+
+if(NOT COMPILE_WASM)
+  # Setting BUILD_ARCH to native invokes CPU intrinsic detection logic below.
+  # Prevent invoking that logic for WASM builds.
+  set(BUILD_ARCH native CACHE STRING "Compile for this CPU architecture.")
+endif()
+
+if(USE_THREADS)
+  # Need to set compile definition as well
+  add_compile_definitions(USE_PTHREADS)
+endif()
+
+if(COMPILE_WITHOUT_EXCEPTIONS)
+  add_compile_definitions(WITHOUT_EXCEPTIONS)
+endif()
 
 # fbgemm and sentencepiece are both defined with "non-local" installation targets (the source projects don't define them,
 # so we define them in src\3rd_party\CMakeLists.txt), but that isn't supported until CMake 3.12. Prior to CMake 3.12,
@@ -165,6 +201,10 @@ else(MSVC)
       set(INTRINSICS "${INTRINSICS} -mavx512f")
       list(APPEND INTRINSICS_NVCC -Xcompiler\ -mavx512f)
     endif(AVX512_FOUND)
+  elseif(COMPILE_WASM)
+    # Can't set to -msse4.1 because onnxjs doesn't compile with this flag. It can be upgraded to
+    # -msse4.1 once marian can solely be compiled with intgemm ("onnxjs" will be removed in that case)
+    set(INTRINSICS "-mssse3 -msimd128")
   else()
     set(INTRINSICS "-msse4.1")
   endif()
@@ -197,24 +237,44 @@ else(MSVC)
     set(CMAKE_RDYNAMIC_FLAG "-rdynamic")
   endif(CMAKE_COMPILER_IS_GNUCC)
 
-  set(CMAKE_CXX_FLAGS                 "-std=c++11 -pthread ${CMAKE_GCC_FLAGS} -fPIC ${DISABLE_GLOBALLY} -march=${BUILD_ARCH} ${INTRINSICS}")
-  set(CMAKE_CXX_FLAGS_RELEASE         "-O3 -m64 -funroll-loops -g ${CMAKE_RDYNAMIC_FLAG}")
-  set(CMAKE_CXX_FLAGS_DEBUG           "-O0 -g ${CMAKE_RDYNAMIC_FLAG}")
-  set(CMAKE_CXX_FLAGS_SLIM            "-O3 -m64 -funroll-loops -DNDEBUG")
-  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO  "${CMAKE_CXX_FLAGS_RELEASE}")
-  set(CMAKE_CXX_FLAGS_PROFILE         "${CMAKE_CXX_FLAGS_RELEASE} -pg")
-  set(CMAKE_CXX_FLAGS_PROFGEN         "${CMAKE_CXX_FLAGS_RELEASE} -fprofile-generate -fprofile-correction")
-  set(CMAKE_CXX_FLAGS_PROFUSE         "${CMAKE_CXX_FLAGS_RELEASE} -fprofile-use -fprofile-correction")
-
-  # these need to be set separately
-  set(CMAKE_C_FLAGS                 "-pthread ${CMAKE_GCC_FLAGS} -fPIC ${DISABLE_GLOBALLY} -march=${BUILD_ARCH} ${INTRINSICS}")
-  set(CMAKE_C_FLAGS_RELEASE         "-O3 -m64 -funroll-loops -g ${CMAKE_RDYNAMIC_FLAG}")
-  set(CMAKE_C_FLAGS_DEBUG           "-O0 -g ${CMAKE_RDYNAMIC_FLAG}")
-  set(CMAKE_C_FLAGS_SLIM            "-O3 -m64 -funroll-loops -DNDEBUG")
-  set(CMAKE_C_FLAGS_RELWITHDEBINFO  "${CMAKE_C_FLAGS_RELEASE}")
-  set(CMAKE_C_FLAGS_PROFILE         "${CMAKE_C_FLAGS_RELEASE} -pg")
-  set(CMAKE_C_FLAGS_PROFGEN         "${CMAKE_C_FLAGS_RELEASE} -fprofile-generate -fprofile-correction")
-  set(CMAKE_C_FLAGS_PROFUSE         "${CMAKE_C_FLAGS_RELEASE} -fprofile-use -fprofile-correction")
+  if(COMPILE_WASM)
+    # Setting USE_SSE2 definition to enable SSE2 specific code in "3rd_party/sse_mathfun.h" for wasm builds
+    add_compile_definitions(USE_SSE2)
+    # Add compile definition for wasm builds
+    add_compile_definitions(WASM)
+    set(PTHREAD_FLAG "-pthread")
+    set(DISABLE_PTHREAD_MEMGROWTH_WARNING -Wno-error=pthreads-mem-growth)
+    set(CMAKE_CXX_FLAGS                 "-std=c++11 ${PTHREAD_FLAG} ${CMAKE_GCC_FLAGS} -fPIC ${DISABLE_GLOBALLY} ${INTRINSICS}")
+    set(CMAKE_CXX_FLAGS_RELEASE         "-O3 -s WASM=1 -s ASSERTIONS=0 -s DISABLE_EXCEPTION_CATCHING=1 -s LLD_REPORT_UNDEFINED -s FORCE_FILESYSTEM=1 -s ALLOW_MEMORY_GROWTH=1 -g2 ${DISABLE_PTHREAD_MEMGROWTH_WARNING} -funroll-loops")
+    # Disabling Pthreads + memory growth warning to be an error
+    # Pthreads + memory growth causes JS accessing the wasm memory to be slow
+    # https://github.com/WebAssembly/design/issues/1271
+    list(APPEND ALL_WARNINGS ${DISABLE_PTHREAD_MEMGROWTH_WARNING})
+
+    # use our customizations to the generated emscripted html and js resources
+    set(MARIAN_DECODER_EMSCRIPTEN_LINK_FLAGS "--pre-js ${CMAKE_SOURCE_DIR}/wasm/pre-module.js \
+                                              --post-js ${CMAKE_SOURCE_DIR}/wasm/post-module.js \
+                                              --shell-file ${CMAKE_SOURCE_DIR}/wasm/custom_shell.html")
+  else(COMPILE_WASM)
+    set(CMAKE_CXX_FLAGS                 "-std=c++11 -pthread ${CMAKE_GCC_FLAGS} -fPIC ${DISABLE_GLOBALLY} -march=${BUILD_ARCH} ${INTRINSICS} ${BUILD_WIDTH}")
+    set(CMAKE_CXX_FLAGS_RELEASE         "-O3 -funroll-loops -g ${CMAKE_RDYNAMIC_FLAG}")
+    set(CMAKE_CXX_FLAGS_DEBUG           "-O0 -g ${CMAKE_RDYNAMIC_FLAG}")
+    set(CMAKE_CXX_FLAGS_SLIM            "-O3 ${BUILD_WIDTH} -funroll-loops -DNDEBUG")
+    set(CMAKE_CXX_FLAGS_RELWITHDEBINFO  "${CMAKE_CXX_FLAGS_RELEASE}")
+    set(CMAKE_CXX_FLAGS_PROFILE         "${CMAKE_CXX_FLAGS_RELEASE} -pg")
+    set(CMAKE_CXX_FLAGS_PROFGEN         "${CMAKE_CXX_FLAGS_RELEASE} -fprofile-generate -fprofile-correction")
+    set(CMAKE_CXX_FLAGS_PROFUSE         "${CMAKE_CXX_FLAGS_RELEASE} -fprofile-use -fprofile-correction")
+
+    # these need to be set separately
+    set(CMAKE_C_FLAGS                 "-pthread ${CMAKE_GCC_FLAGS} -fPIC ${DISABLE_GLOBALLY} -march=${BUILD_ARCH} ${INTRINSICS}")
+    set(CMAKE_C_FLAGS_RELEASE         "-O3 ${BUILD_WIDTH} -funroll-loops -g ${CMAKE_RDYNAMIC_FLAG}")
+    set(CMAKE_C_FLAGS_DEBUG           "-O0 -g ${CMAKE_RDYNAMIC_FLAG}")
+    set(CMAKE_C_FLAGS_SLIM            "-O3 ${BUILD_WIDTH} -funroll-loops -DNDEBUG")
+    set(CMAKE_C_FLAGS_RELWITHDEBINFO  "${CMAKE_C_FLAGS_RELEASE}")
+    set(CMAKE_C_FLAGS_PROFILE         "${CMAKE_C_FLAGS_RELEASE} -pg")
+    set(CMAKE_C_FLAGS_PROFGEN         "${CMAKE_C_FLAGS_RELEASE} -fprofile-generate -fprofile-correction")
+    set(CMAKE_C_FLAGS_PROFUSE         "${CMAKE_C_FLAGS_RELEASE} -fprofile-use -fprofile-correction")
+  endif(COMPILE_WASM)
 endif(MSVC)
 
 # with gcc 7.0 and above we need to mark fallthrough in switch case statements
@@ -435,29 +495,37 @@ if(USE_MPI)
 endif(USE_MPI)
 
 ###############################################################################
-# Find MKL
+# Find BLAS library for CPU compilation
 if(COMPILE_CPU)
-  set(EXT_LIBS ${EXT_LIBS} intgemm)
-  if(USE_MKL)
-    find_package(MKL)
-  endif(USE_MKL)
-  if(MKL_FOUND)
-    include_directories(${MKL_INCLUDE_DIR})
-    set(EXT_LIBS ${EXT_LIBS} ${MKL_LIBRARIES})
+  if(USE_WASM_COMPATIBLE_BLAS)
+    ## Use a wasm compatible BLAS
+    set(EXT_LIBS ${EXT_LIBS} intgemm onnx-sgemm)
     set(BLAS_FOUND TRUE)
-    add_definitions(-DCOMPILE_CPU=1 -DBLAS_FOUND=1 -DMKL_FOUND=1)
-  else(MKL_FOUND)
-    set(BLAS_VENDOR "OpenBLAS")
-    find_package(BLAS)
-    if(BLAS_FOUND)
-      include(FindCBLAS)
-      if(CBLAS_FOUND)
-        include_directories(${BLAS_INCLUDE_DIR} ${CBLAS_INCLUDE_DIR})
-        set(EXT_LIBS ${EXT_LIBS} ${BLAS_LIBRARIES} ${CBLAS_LIBRARIES})
-        add_definitions(-DCOMPILE_CPU=1 -DBLAS_FOUND=1)
-      endif(CBLAS_FOUND)
-    endif(BLAS_FOUND)
-  endif(MKL_FOUND)
+    set(BLAS_VENDOR "ONNX-SGEMM")
+    add_compile_definitions(COMPILE_CPU  BLAS_FOUND  WASM_COMPATIBLE_BLAS)
+  else(USE_WASM_COMPATIBLE_BLAS)
+    set(EXT_LIBS ${EXT_LIBS} intgemm)
+    if(USE_MKL)
+      find_package(MKL)
+    endif(USE_MKL)
+    if(MKL_FOUND)
+      include_directories(${MKL_INCLUDE_DIR})
+      set(EXT_LIBS ${EXT_LIBS} ${MKL_LIBRARIES})
+      set(BLAS_FOUND TRUE)
+      add_definitions(-DCOMPILE_CPU=1 -DBLAS_FOUND=1 -DMKL_FOUND=1)
+    else(MKL_FOUND)
+      set(BLAS_VENDOR "OpenBLAS")
+      find_package(BLAS)
+      if(BLAS_FOUND)
+        include(FindCBLAS)
+        if(CBLAS_FOUND)
+          include_directories(${BLAS_INCLUDE_DIR} ${CBLAS_INCLUDE_DIR})
+          set(EXT_LIBS ${EXT_LIBS} ${BLAS_LIBRARIES} ${CBLAS_LIBRARIES})
+          add_definitions(-DCOMPILE_CPU=1 -DBLAS_FOUND=1)
+        endif(CBLAS_FOUND)
+      endif(BLAS_FOUND)
+    endif(MKL_FOUND)
+  endif(USE_WASM_COMPATIBLE_BLAS)
 endif(COMPILE_CPU)
 
 ###############################################################################

src/3rd_party/CMakeLists.txt

@@ -2,15 +2,21 @@
 include_directories(.)
 
 add_subdirectory(./yaml-cpp)
-add_subdirectory(./SQLiteCpp)
+if(NOT COMPILE_WASM)
+  add_subdirectory(./SQLiteCpp)
+  add_subdirectory(./zlib)
+  add_subdirectory(./faiss)
+  include_directories(./faiss)
+endif()
 add_subdirectory(./pathie-cpp)
-add_subdirectory(./zlib)
-add_subdirectory(./faiss)
-include_directories(./faiss)
 
 set(INTGEMM_DONT_BUILD_TESTS ON CACHE BOOL "Disable intgemm tests")
 add_subdirectory(./intgemm)
 
+if(USE_WASM_COMPATIBLE_BLAS)
+  add_subdirectory(./onnxjs)
+endif(USE_WASM_COMPATIBLE_BLAS)
+
 if(USE_FBGEMM)
   # @TODO: find out if this is somehow harmful. This is supppressing CMake warnings for CMAKE_SUPPRESS_DEVELOPER_WARNINGS
   # meant to silence CMakeFiles of 3rd_party tools.
@@ -78,7 +84,6 @@ if(USE_SENTENCEPIECE)
 
   add_subdirectory(./sentencepiece)
   include_directories(./sentencepiece)
-
   set_target_properties(spm_encode spm_decode spm_train spm_normalize spm_export_vocab
     PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
 
@@ -123,13 +128,15 @@ include_directories(./CLI)
 include_directories(./pathie-cpp/include)
 
 if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  if(NOT COMPILE_WASM)
   #set_target_properties(SQLiteCpp PROPERTIES COMPILE_FLAGS
   set_property(TARGET SQLiteCpp APPEND_STRING PROPERTY COMPILE_FLAGS
     " -Wno-parentheses-equality -Wno-unused-value")
   if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9.0)
     set_property(TARGET SQLiteCpp APPEND_STRING PROPERTY COMPILE_FLAGS
       " -Wno-implicit-int-float-conversion")
   endif()
+  endif()
   set_property(TARGET libyaml-cpp APPEND_STRING PROPERTY COMPILE_FLAGS
     " -fPIC -Wno-unused-value")
   set_property(TARGET pathie-cpp APPEND_STRING PROPERTY COMPILE_FLAGS

src/3rd_party/cnpy/cnpy.h

@@ -5,7 +5,9 @@
 #ifndef LIBCNPY_H_
 #define LIBCNPY_H_
 
+#if !defined(WASM)
 #include "3rd_party/zlib/zlib.h"
+#endif
 
 #include<string>
 #include<stdexcept>
@@ -18,7 +20,7 @@
 #include<map>
 #include <memory>
 
-#ifdef __APPLE__
+#if defined(__APPLE__) || defined(__unix__)
 #include <unistd.h>
 #endif
 
@@ -133,6 +135,9 @@ namespace cnpy {
 
     template<typename T> void npz_save(std::string zipname, std::string fname, const T* data, const unsigned int* shape, const unsigned int ndims, std::string mode = "w")
     {
+#if defined(WASM)
+        throw std::runtime_error("npz_save() not supported in WASM builds");
+#else
         //first, append a .npy to the fname
         fname += ".npy";
 
@@ -221,6 +226,7 @@ namespace cnpy {
         fwrite(&footer[0],sizeof(char),footer.size(),fp);
         //BUGBUG: no check for write error
         fclose(fp);
+#endif
     }
 
     //one item pass to npz_save() below
@@ -265,6 +271,9 @@ namespace cnpy {
     static inline
     void npz_save(std::string zipname, const std::vector<NpzItem>& items)
     {
+#if defined(WASM)
+        throw std::runtime_error("npz_save() not supported in WASM builds");
+#else
         auto tmpname = zipname + "$$"; // TODO: add thread id or something
         unlink(tmpname.c_str()); // when saving to HDFS, we cannot overwrite an existing file
         FILE* fp = fopen(tmpname.c_str(),"wb");
@@ -366,6 +375,7 @@ namespace cnpy {
             unlink(tmpname.c_str());
             throw std::runtime_error("npz_save: error saving to file: " + zipname);
         }
+#endif
     }
 
     static inline