Merge branch 'kaldi-asr:master' into master

.github/stale.yml

@@ -0,0 +1,29 @@
+# Number of days of inactivity before an issue becomes stale.
+daysUntilStale: 60
+# Number of days of inactivity before a stale issue is closed.
+# TODO(kkm): Re-enable auto-closing when done with the current heap of old PRs.
+daysUntilClose: false
+# Issues with these labels will never be considered stale.
+exemptLabels:
+  - discussion
+  - enhancement
+  - help-wanted
+  - in progress
+  - low-priority
+  - newbie
+  - stale-exclude
+  - stopped development
+# Label to use when marking an issue as stale.
+staleLabel: stale
+# Comment to post when marking an issue as stale.
+markComment: >
+  This issue has been automatically marked as stale by a bot solely because it
+  has not had recent activity. Please add any comment (simply 'ping' is enough)
+  to prevent the issue from being closed for 60 more days if you believe it
+  should be kept open.
+# Comment to post when closing a stale issue.
+closeComment: >
+  This issue has been automatically closed by a bot strictly because of
+  inactivity. This does not mean that we think that this issue is not
+  important! If you believe it has been closed hastily, add a comment
+  to the issue and mention @kkm000, and I'll gladly reopen it.

.gitignore

@@ -7,58 +7,65 @@
 !/src/*/Makefile
 !/src/*/README
 
-# Compiled Object files and python ciles
+# Object files.
 *.slo
 *.lo
 *.o
 *.obj
 *.pyc
 
-# profiling files and core files
+# Profiling files and core dumps.
 *.nvprof
 core
 
-# Compiled Dynamic libraries
+# Compiled dynamic libraries.
 *.so
 *.dylib
 *.dll
 
-# Compiled Static libraries
+# Compiled static libraries.
 *.lai
 *.la
 *.a
 *.lib
 
-# Test log files
+# Test log files.
 *.testlog
 
-# emacs saves
+# Emacs autosave and backup files.
 [#]*[#]
 .[#]*
 *~
 
-# [ecg]tag files
+# [ecg]tag files.
 TAGS
 tags
 GTAGS
 GRTAGS
 GPATH
 GSYMS
 
-# python compiled sources
+# Python compiled bytecode files.
 *.pyc
 
-# Make dependencies
+# Python virtual environment
+venv/
+
+# Make dependencies.
 .depend.mk
 
-# some weird thing that MacOs creates.
+# Some weird thing that macOS creates.
 *.dSYM
+.DS_Store
 
 # Windows executable, symbol and some weird files.
 *.exe
 *.pdb
 *.exp
 *.manifest
+/kaldiwin_vs*
+.vscode
+.idea
 
 # /src/
 /src/.short_version
@@ -80,81 +87,6 @@ GSYMS
 /egs/*/*/wav
 /egs/*/*/enhan
 
-# /tools/
-/tools/pocolm/
-/tools/ATLAS/
-/tools/atlas3.8.3.tar.gz
-/tools/irstlm/
-/tools/mitlm/
-/tools/openfst
-/tools/openfst-1.3.2.tar.gz
-/tools/openfst-1.3.2/
-/tools/openfst-1.3.4.tar.gz
-/tools/openfst-1.3.4/
-/tools/openfst-1.4.1.tar.gz
-/tools/openfst-1.4.1/
-/tools/openfst-1.5.0.tar.gz
-/tools/openfst-1.5.0/
-/tools/openfst-1.5.4.tar.gz
-/tools/openfst-1.5.4/
-/tools/openfst-1.6.0.tar.gz
-/tools/openfst-1.6.0/
-/tools/openfst-1.6.1.tar.gz
-/tools/openfst-1.6.1/
-/tools/openfst-1.6.2.tar.gz
-/tools/openfst-1.6.2/
-/tools/openfst-1.6.5.tar.gz
-/tools/openfst-1.6.5/
-/tools/openfst-1.6.7.tar.gz
-/tools/openfst-1.6.7/
-/tools/BeamformIt/
-/tools/libsndfile-1.0.25.tar.gz
-/tools/libsndfile-1.0.25/
-/tools/pa_stable_v19_20111121.tgz
-/tools/portaudio/
-/tools/sctk-2.4.0-20091110-0958.tar.bz2
-/tools/sctk-2.4.0/
-/tools/sctk-2.4.10-20151007-1312Z.tar.bz2
-/tools/sctk-2.4.10/
-/tools/sph2pipe_v2.5.tar.gz
-/tools/sph2pipe_v2.5/
-/tools/kaldi_lm.tar.gz
-/tools/sctk-2.4.8-20130429-2145.tar.bz2
-/tools/sctk-2.4.9-20141015-1634Z.tar.bz2
-/tools/sctk-2.4.9/
-/tools/OpenBLAS
-/tools/sctk
-/tools/sctk-2.4.8
-/tools/srilm
-/tools/CLAPACK_include
-/tools/kaldi_lm
-/tools/env.sh
-/tools/g2p-r1668.tar.gz
-/tools/mpg123
-/tools/mpg123-1.21.0.tar.bz2
-/tools/mpg123-1.21.0
-/tools/pthreads
-/tools/pthreads*.zip
-/tools/sequitur
-/tools/sequitur-g2p
-/tools/srilm.tgz
-/tools/liblbfgs-1.10.tar.gz
-/tools/liblbfgs-1.10/
-/tools/srilm-1.7.2-beta.tar.gz
-/tools/liblbfgs/
-/tools/sequitur-g2p/
-/tools/phonetisaurus-g2p/
-/tools/mmseg-1.3.0.tar.gz
-/tools/mmseg-1.3.0/
-/kaldiwin_vs*
-/tools/cub-1.8.0.zip
-/tools/cub-1.8.0/
-/tools/cub
-/tools/python/
-/tools/ngram-1.3.7.tar.gz
-/tools/ngram-1.3.7/
-
-# These CMakeLists.txt files are all genareted on the fly at the moment.
-# They are added here to avoid accidently checkin.
+# CMakeLists.txt files are currently autogenerated, must not be committed.
 /src/**/CMakeLists.txt
 /build*

.gitpod.Dockerfile

@@ -0,0 +1,22 @@
+FROM gitpod/workspace-full
+
+USER songmeixu
+
+# Install custom tools, runtime, etc. using apt-get
+# For example, the command below would install "bastet" - a command line tetris clone:
+#
+# RUN sudo apt-get -q update && #     sudo apt-get install -yq bastet && #     sudo rm -rf /var/lib/apt/lists/*
+#
+# More information: https://www.gitpod.io/docs/config-docker/
+
+FROM gitpod/workspace-full
+
+RUN sudo apt-get update \
+ && sudo apt-get install -y \
+    sox gfortran \
+ && sudo rm -rf /var/lib/apt/lists/*
+
+RUN cd ~/GitHub/r-with-intel-mkl/ \
+ && wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB | sudo apt-key add - \
+ && sudo sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' \
+ && sudo apt-get update && sudo apt-get install intel-mkl-64bit

.gitpod.yml

@@ -0,0 +1,7 @@
+image:
+  file: .gitpod.dockerfile
+
+tasks:
+  - init: echo "Replace me with a build script for the project."
+    command: echo "Replace me with something that should run on every start, or just
+      remove me entirely."

CMakeLists.txt

@@ -1,25 +1,35 @@
 cmake_minimum_required(VERSION 3.5)
 project(kaldi)
 
+if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+  set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/src CACHE PATH "Install path prefix." FORCE)
+endif(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake;${CMAKE_MODULE_PATH}")
 include(GNUInstallDirs)
 include(Utils)
+
 include(third_party/get_third_party)
 
-# Should update cmake to a more recent version which supports FindPython3.
+include(cmake/third_party/openfst.cmake)
+
 find_package(PythonInterp)
-if(NOT PYTHON_EXECUTABLE OR PYTHON_VERSION_MAJOR LESS 3)
-    message(WARNING "Needs python3 to auto-generate most CMake files, but not found. "
-                    "Will try `python3` directly...")
-    set(PYTHON_EXECUTABLE "python3")
+if(NOT PYTHON_EXECUTABLE)
+    message(FATAL_ERROR "Needs python to auto-generate most CMake files, but not found.")
 endif()
 
 message(STATUS "Running gen_cmake_skeleton.py")
+option(BUILD_SHARED_LIBS "Build shared Kaldi libraries." OFF)
+set(IS_LIB_SHARE "")
+if(BUILD_SHARED_LIBS)
+    set(IS_LIB_SHARE "--shared")
+endif()
 execute_process(COMMAND ${PYTHON_EXECUTABLE}
     "${CMAKE_CURRENT_SOURCE_DIR}/cmake/gen_cmake_skeleton.py"
     "${CMAKE_CURRENT_SOURCE_DIR}/src"
     "--quiet"
+    ${IS_LIB_SHARE}
 )
+unset(IS_LIB_SHARE)
 
 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_EXTENSIONS OFF)
@@ -32,7 +42,7 @@ if(BUILD_SHARED_LIBS)
     elseif(APPLE)
         set(CMAKE_INSTALL_RPATH "@loader_path")
     else()
-        set(CMAKE_INSTALL_RPATH "$ORIGIN;$ORIGIN/../lib")
+        set(CMAKE_INSTALL_RPATH "$ORIGIN;$ORIGIN/../lib;$ORIGIN/../../tools/openfst/lib")
     endif()
 endif()
 
@@ -63,12 +73,16 @@ if(MATHLIB STREQUAL "OpenBLAS")
     include_directories(${CMAKE_CURRENT_SOURCE_DIR}/tools/CLAPACK)
     link_libraries(${BLAS_LIBRARIES} ${LAPACK_LIBRARIES})
 elseif(MATHLIB STREQUAL "MKL")
-    set(BLA_VENDOR "Intel10_64lp")
-    # find_package(BLAS REQUIRED)
+    if(NOT DEFINED ENV{MKLROOT} OR "$ENV{MKLROOT}" STREQUAL "")
+        message(FATAL_ERROR "Environment variable MKLROOT is not defined")
+    else()
+        message(STATUS "Finding MKL from \"$ENV{MKLROOT}\"")
+    endif()
     normalize_env_path(ENV{MKLROOT})
+    set(BLA_VENDOR "Intel10_64lp_seq") # use the single threaded MKL by default
     find_package(LAPACK REQUIRED)
     add_definitions(-DHAVE_MKL=1)
-    include_directories($ENV{MKLROOT}/include) # TODO: maybe not use env, idk, find_package doesnt handle includes...
+    include_directories($ENV{MKLROOT}/include)
     link_libraries(${BLAS_LIBRARIES} ${LAPACK_LIBRARIES})
 elseif(MATHLIB STREQUAL "Accelerate")
     execute_process(COMMAND sw_vers -productVersion
@@ -113,8 +127,6 @@ endif()
 
 find_package(CUDA)
 if(CUDA_FOUND)
-    set(CUB_ROOT_DIR "${PROJECT_SOURCE_DIR}/tools/cub")
-
     set(CUDA_PROPAGATE_HOST_FLAGS ON)
     set(KALDI_CUDA_NVCC_FLAGS "--default-stream=per-thread;-std=c++${CMAKE_CXX_STANDARD}")
     if(MSVC)
@@ -146,6 +158,8 @@ if(CUDA_FOUND)
     include_directories(${NvToolExt_INCLUDE_DIR})
     link_libraries(${NvToolExt_LIBRARIES})
 
+    get_third_party(cub)
+    set(CUB_ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/cub")
     find_package(CUB REQUIRED)
     include_directories(${CUB_INCLUDE_DIR})
 endif()
@@ -158,9 +172,18 @@ if(${KALDI_USE_PATCH_NUMBER})
     set(KALDI_VERSION "${KALDI_VERSION}.${KALDI_PATCH_NUMBER}")
 endif()
 
-get_third_party(openfst)
-set(OPENFST_ROOT_DIR ${CMAKE_CURRENT_BINARY_DIR}/openfst)
-include(third_party/openfst_lib_target)
+# get_third_party(openfst)
+# set(OPENFST_ROOT_DIR ${CMAKE_CURRENT_BINARY_DIR}/openfst)
+# include(third_party/openfst_lib_target)
+#find_library(OpenFST_LIBRARY
+#            NAMES fst
+#            PATHS ${CMAKE_CURRENT_SOURCE_DIR}/tools/openfst/lib
+#            REQUIRED)
+#find_path(OpenFST_INCLUDE_DIR
+#            NAMES "fst/fst.h"
+#            PATHS "${CMAKE_CURRENT_SOURCE_DIR}/tools/openfst/include"
+#            REQUIRED)
+
 link_libraries(fst)
 
 # add all native libraries
@@ -230,15 +253,15 @@ include(CMakePackageConfigHelpers)
 # maybe we should put this into subfolder?
 configure_package_config_file(
     ${CMAKE_CURRENT_SOURCE_DIR}/cmake/kaldi-config.cmake.in
-    ${CMAKE_BINARY_DIR}/cmake/kaldi-config.cmake
+    ${CMAKE_CURRENT_BINARY_DIR}/cmake/kaldi-config.cmake
     INSTALL_DESTINATION lib/cmake/kaldi
 )
 write_basic_package_version_file(
-    ${CMAKE_BINARY_DIR}/cmake/kaldi-config-version.cmake
+    ${CMAKE_CURRENT_BINARY_DIR}/cmake/kaldi-config-version.cmake
     VERSION ${KALDI_VERSION}
     COMPATIBILITY AnyNewerVersion
 )
-install(FILES ${CMAKE_BINARY_DIR}/cmake/kaldi-config.cmake ${CMAKE_BINARY_DIR}/cmake/kaldi-config-version.cmake
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/cmake/kaldi-config.cmake ${CMAKE_CURRENT_BINARY_DIR}/cmake/kaldi-config-version.cmake
     DESTINATION lib/cmake/kaldi
 )
 install(EXPORT kaldi-targets DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/cmake/kaldi)

README.md

@@ -1,4 +1,5 @@
 [![Build Status](https://travis-ci.com/kaldi-asr/kaldi.svg?branch=master)](https://travis-ci.com/kaldi-asr/kaldi)
+[![Gitpod Ready-to-Code](https://img.shields.io/badge/Gitpod-Ready--to--Code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/kaldi-asr/kaldi) 
 Kaldi Speech Recognition Toolkit
 ================================
 
@@ -65,3 +66,10 @@ Platform specific notes
   OpenBLAS.
 - See [this blog post](http://jcsilva.github.io/2017/03/18/compile-kaldi-android/)
   for details.
+
+### Web Assembly
+
+- Kaldi supports cross compiling for Web Assembly for in-browser execution
+  using [emscripten](https://emscripten.org/) and CLAPACK.
+- See [this post](https://gitlab.inria.fr/kaldi.web/kaldi-wasm/-/wikis/build_details.md)
+  for a step-by-step description of the build process.