2.2.5 (#589)

* Fix bug in defaults for auto_server setting

* Rework indices and docker file, and bugfixes

* Update changelog

* Update dockerfile

.dockerignore

@@ -0,0 +1,7 @@
+
+docs/*
+build/*
+.tox/*
+.github/*
+.pytest_cache/*
+tests/*

.github/workflows/publish_docker.yml

@@ -0,0 +1,45 @@
+name: Publish Docker image
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  push_to_registries:
+    name: Push Docker image to multiple registries
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: |
+            mmcauliffe/montreal-forced-aligner
+            ghcr.io/${{ github.repository }}
+
+      - name: Build and push Docker images
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -1,2 +1,17 @@
-FROM continuumio/miniconda3
-RUN conda create -n aligner -c conda-forge montreal-forced-aligner
+FROM condaforge/mambaforge:22.11.1-4 as build
+
+COPY docker_environment.yaml .
+RUN mkdir -p /mfa
+RUN mamba env create -p /env -f docker_environment.yaml && conda clean -afy
+
+COPY . /pkg
+RUN conda run -p /env python -m pip install --no-deps /pkg
+
+RUN useradd -ms /bin/bash mfauser
+RUN chown -R mfauser /mfa
+RUN chown -R mfauser /env
+USER mfauser
+ENV MFA_ROOT_ENVIRONMENT_VARIABLE=/mfa
+RUN conda run -p /env mfa server init
+
+ENTRYPOINT ["conda", "run", "-p", "/env", "mfa"]

docker_environment.yaml

@@ -0,0 +1,41 @@
+channels:
+  - conda-forge
+  - pytorch
+  - nvidia
+dependencies:
+  - python>=3.8
+  - numpy
+  - librosa
+  - tqdm
+  - requests
+  - pyyaml
+  - dataclassy
+  - kaldi=*=*cpu*
+  - sox
+  - ffmpeg
+  - pynini
+  - openfst
+  - hdbscan
+  - baumwelch
+  - ngram
+  - praatio=6.0.0
+  - biopython=1.79
+  - sqlalchemy>=2.0
+  - git
+  - pgvector
+  - pgvector-python
+  - postgresql
+  - psycopg2
+  - click
+  - pytorch
+  - torchaudio
+  - setuptools_scm
+  - numba
+  - kneed
+  - matplotlib
+  - seaborn
+  - pip
+  - rich
+  - rich-click
+  - pip:
+      - speechbrain

docs/source/changelog/changelog_2.2.rst

@@ -5,6 +5,14 @@
 2.2 Changelog
 *************
 
+2.2.5
+=====
+
+- Fixed a bug in where piping to :code:`mfa g2p` could hang if there was no known characters
+- Fixed a bug where the default auto_server setting was to be disabled rather than enabled
+- Removed indices on utterance text to address issues in longer files
+- Revamped docker image to address issues with initializing postgresql databases as root
+
 2.2.4
 =====
 

entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/bash --login
+# The --login ensures the bash configuration is loaded,
+# enabling Conda.
+
+# Enable strict mode.
+set -euo pipefail
+# ... Run whatever commands ...
+
+# Temporarily disable strict mode and activate conda:
+set +euo pipefail
+conda activate myenv
+
+# Re-enable strict mode:
+set -euo pipefail
+
+# exec the final command:
+exec python run.py

montreal_forced_aligner/abc.py

@@ -32,7 +32,11 @@
 
 from montreal_forced_aligner.config import GLOBAL_CONFIG
 from montreal_forced_aligner.db import CorpusWorkflow, MfaSqlBase
-from montreal_forced_aligner.exceptions import KaldiProcessingError, MultiprocessingError
+from montreal_forced_aligner.exceptions import (
+    DatabaseError,
+    KaldiProcessingError,
+    MultiprocessingError,
+)
 from montreal_forced_aligner.helper import comma_join, load_configuration, mfa_open
 
 if TYPE_CHECKING:
@@ -238,19 +242,25 @@ def initialize_database(self) -> None:
         try:
             check_databases(self.identifier)
         except Exception:
-            subprocess.check_call(
-                [
-                    "createdb",
-                    f"--host={GLOBAL_CONFIG.database_socket}",
-                    self.identifier,
-                ],
-                stderr=subprocess.DEVNULL,
-                stdout=subprocess.DEVNULL,
-            )
+            try:
+                subprocess.check_call(
+                    [
+                        "createdb",
+                        f"--host={GLOBAL_CONFIG.database_socket}",
+                        self.identifier,
+                    ],
+                    stderr=subprocess.DEVNULL,
+                    stdout=subprocess.DEVNULL,
+                )
+            except Exception:
+                raise DatabaseError(
+                    f"There was an error connecting to the {GLOBAL_CONFIG.current_profile_name} MFA database server. "
+                    "Please ensure the server is initialized (mfa server init) or running (mfa server start)"
+                )
             exist_check = False
         self.database_initialized = True
         if exist_check:
-            if GLOBAL_CONFIG.current_profile.clean:
+            if GLOBAL_CONFIG.current_profile.clean or getattr(self, "dirty", False):
                 self.clean_working_directory()
                 self.delete_database()
             else:
@@ -668,29 +678,16 @@ def _validate_previous_configuration(self, conf: MetaDict) -> bool:
         """
         from montreal_forced_aligner.utils import get_mfa_version
 
-        clean = True
+        self.dirty = False
         current_version = get_mfa_version()
-        if conf["dirty"]:
-            logger.debug("Previous run ended in an error (maybe ctrl-c?)")
-            clean = False
-        if conf.get("version", current_version) != current_version:
+        if (
+            not GLOBAL_CONFIG.current_profile.debug
+            and conf.get("version", current_version) != current_version
+        ):
             logger.debug(
                 f"Previous run was on {conf['version']} version (new run: {current_version})"
             )
-            clean = False
-        for key in [
-            "corpus_directory",
-            "dictionary_path",
-            "acoustic_model_path",
-            "g2p_model_path",
-            "language_model_path",
-        ]:
-            if conf.get(key, None) != getattr(self, key, None):
-                logger.debug(
-                    f"Previous run used a different {key.replace('_', ' ')} than {getattr(self, key, None)} (was {conf.get(key, None)})"
-                )
-                clean = False
-        return clean
+            self.dirty = True
 
     def check_previous_run(self) -> bool:
         """
@@ -705,7 +702,7 @@ def check_previous_run(self) -> bool:
             return True
         conf = load_configuration(self.worker_config_path)
         clean = self._validate_previous_configuration(conf)
-        if not GLOBAL_CONFIG.current_profile.clean and not clean:
+        if not GLOBAL_CONFIG.current_profile.clean and self.dirty:
             logger.warning(
                 "The previous run had a different configuration than the current, which may cause issues."
                 " Please see the log for details or use --clean flag if issues are encountered."

montreal_forced_aligner/alignment/base.py

@@ -25,8 +25,8 @@
 from montreal_forced_aligner.alignment.multiprocessing import (
     AlignmentExtractionArguments,
     AlignmentExtractionFunction,
-    CalculateSpeechPostArguments,
-    CalculateSpeechPostFunction,
+    AnalyzeAlignmentsArguments,
+    AnalyzeAlignmentsFunction,
     ExportTextGridArguments,
     ExportTextGridProcessWorker,
     FineTuneArguments,
@@ -152,9 +152,9 @@ def score_options(self) -> MetaDict:
             "word_insertion_penalty": getattr(self, "word_insertion_penalty", 0.5),
         }
 
-    def calculate_speech_post_arguments(self) -> List[CalculateSpeechPostArguments]:
+    def analyze_alignments_arguments(self) -> List[AnalyzeAlignmentsArguments]:
         return [
-            CalculateSpeechPostArguments(
+            AnalyzeAlignmentsArguments(
                 j.id,
                 getattr(self, "db_string", ""),
                 self.working_log_directory.joinpath(f"calculate_speech_post.{j.id}.log"),
@@ -181,11 +181,11 @@ def analyze_alignments(self):
             bulk_update(session, Phone, update_mappings)
             session.commit()
 
-            arguments = self.calculate_speech_post_arguments()
+            arguments = self.analyze_alignments_arguments()
             update_mappings = []
             with tqdm(total=self.num_current_utterances, disable=GLOBAL_CONFIG.quiet) as pbar:
                 for utt_id, speech_log_likelihood, duration_deviation in run_kaldi_function(
-                    CalculateSpeechPostFunction, arguments, pbar.update
+                    AnalyzeAlignmentsFunction, arguments, pbar.update
                 ):
                     update_mappings.append(
                         {
@@ -482,7 +482,7 @@ def compute_pronunciation_probabilities(self):
                         Word.word,
                         Pronunciation.pronunciation,
                         Pronunciation.id,
-                        Pronunciation.base_pronunciation_id,
+                        Pronunciation.generated_by_rule,
                     )
                     .join(Pronunciation.word)
                     .filter(Word.dictionary_id == d_id)
@@ -492,8 +492,8 @@ def compute_pronunciation_probabilities(self):
                 pron_mapping = {}
                 pronunciations = [
                     (w, p, p_id)
-                    for w, p, p_id, b_id in pronunciations
-                    if p_id == b_id or p in counter.word_pronunciation_counts[w]
+                    for w, p, p_id, generated in pronunciations
+                    if not generated or p in counter.word_pronunciation_counts[w]
                 ]
                 for w, p, p_id in pronunciations:
                     pron_mapping[(w, p)] = {"id": p_id}
@@ -728,8 +728,7 @@ def compute_pronunciation_probabilities(self):
                 ).delete()
                 session.flush()
                 session.query(Pronunciation).filter(
-                    Pronunciation.id != Pronunciation.base_pronunciation_id,
-                    Pronunciation.count == 0,
+                    Pronunciation.count == 0, Pronunciation.generated_by_rule == True  # noqa
                 ).delete()
                 session.commit()
                 pronunciation_counts = {

montreal_forced_aligner/alignment/multiprocessing.py

@@ -82,7 +82,9 @@
     "AlignmentExtractionArguments",
     "ExportTextGridArguments",
     "AlignFunction",
+    "AnalyzeAlignmentsFunction",
     "AlignArguments",
+    "AnalyzeAlignmentsArguments",
     "AccStatsFunction",
     "AccStatsArguments",
     "compile_information_func",
@@ -375,9 +377,9 @@ class AlignArguments(MfaArguments):
 
 
 @dataclass
-class CalculateSpeechPostArguments(MfaArguments):
+class AnalyzeAlignmentsArguments(MfaArguments):
     """
-    Arguments for :class:`~montreal_forced_aligner.alignment.multiprocessing.AlignFunction`
+    Arguments for :class:`~montreal_forced_aligner.alignment.multiprocessing.AnalyzeAlignmentsFunction`
 
     Parameters
     ----------
@@ -955,9 +957,9 @@ def _run(self) -> typing.Generator[typing.Tuple[int, float]]:
                 self.check_call(align_proc)
 
 
-class CalculateSpeechPostFunction(KaldiFunction):
+class AnalyzeAlignmentsFunction(KaldiFunction):
     """
-    Multiprocessing function for alignment.
+    Multiprocessing function for analyzing alignments.
 
     See Also
     --------
@@ -980,7 +982,7 @@ class CalculateSpeechPostFunction(KaldiFunction):
         r"^LOG.*Log-like per frame for utterance (?P<utterance>.*) is (?P<loglike>[-\d.]+) over (?P<num_frames>\d+) frames."
     )
 
-    def __init__(self, args: CalculateSpeechPostArguments):
+    def __init__(self, args: AnalyzeAlignmentsArguments):
         super().__init__(args)
         self.model_path = args.model_path
         self.align_options = args.align_options
@@ -1005,7 +1007,7 @@ def _run(self) -> typing.Generator[typing.Tuple[int, float]]:
                 for k, m, sd in session.query(
                     Phone.id, Phone.mean_duration, Phone.sd_duration
                 ).filter(
-                    Phone.phone_type == PhoneType.non_silence,
+                    Phone.phone_type.in_([PhoneType.non_silence, PhoneType.oov]),
                     Phone.sd_duration != None,  # noqa
                     Phone.sd_duration != 0,
                 )

montreal_forced_aligner/command_line/__init__.py

@@ -29,7 +29,7 @@
     "adapt",
     "align",
     "anchor",
-    "diarize_speakers.py",
+    "diarize_speakers",
     "create_segments",
     "g2p",
     "mfa",

montreal_forced_aligner/command_line/g2p.py

@@ -128,6 +128,8 @@ def g2p_cli(context, **kwargs) -> None:
                     if not word:
                         continue
                     pronunciations = g2p.rewriter(word)
+                    if not pronunciations:
+                        output.write(f"{word}\t\n")
                     for p in pronunciations:
                         output.write(f"{word}\t{p}\n")
                     output.flush()