fixed slow preprocessing due to non-cached/compiled regexes, fixed cr…

…ash when deleting model with running pipes, fixed languages for language group models with no yaml file, fixed bug with Tatoeba model list parsing

OpusCatMTEngine/App.xaml.cs

@@ -11,6 +11,7 @@
 using System.Net;
 using System.Reflection;
 using System.ServiceModel;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using System.Windows;
 using System.Windows.Threading;
@@ -119,6 +120,7 @@ private void Application_Startup(object sender, StartupEventArgs e)
             this.InitializePythonEngine();
 
             this.CheckForUpdatesAsync();
+
         }
 
         private async void CheckForUpdatesAsync()

OpusCatMTEngine/MTModel.cs

@@ -160,7 +160,7 @@ internal void Shutdown()
                 foreach (var langpair in this.marianProcesses.Keys)
                 {
                     this.marianProcesses[langpair].ShutdownMtPipe();
-                    this.marianProcesses[langpair] = null;
+                    //this.marianProcesses[langpair] = null;
                 }
             }
             this.marianProcesses = null;
@@ -285,13 +285,7 @@ internal void DownloadProgressChanged(object sender, DownloadProgressChangedEven
         {
             this.InstallProgress = e.ProgressPercentage;
         }
-
-        public List<IsoLanguage> TargetLanguages
-        {
-            get => targetLanguages;
-            set => targetLanguages = value;
-        }
-
+
         public MTModelStatus Status { get => status; set { status = value; NotifyPropertyChanged(); } }
 
         //This creates a zip package of the model that can be moved to another computer
@@ -460,7 +454,25 @@ internal void ExitHandler(object sender, EventArgs e)
             }
         }
 
-        public List<IsoLanguage> SourceLanguages { get => sourceLanguages; set => sourceLanguages = value; }
+        public List<IsoLanguage> SourceLanguages
+        {
+            get => sourceLanguages;
+            set
+            {
+                sourceLanguages = value;
+                NotifyPropertyChanged("SourceLanguageString");
+            }
+        }
+
+        public List<IsoLanguage> TargetLanguages
+        {
+            get => targetLanguages;
+            set
+            {
+                targetLanguages = value;
+                NotifyPropertyChanged("TargetLanguageString");
+            }
+        }
 
         public string Name { get => name; set => name = value; }
 
@@ -633,14 +645,23 @@ public MTModel(
             this.UpdateModelYamlPath();
 
             this.SupportsWordAlignment = this.decoderSettings.models[0].Contains("-align");
-
-            if (this.modelYaml == null)
-            {
-                this.ParseModelPathForLanguages(modelPath);
-            }
 
             this.ParseModelConfig();
 
+            if (this.modelYaml == null)
+            {
+                if (this.modelConfig.SourceLanguageCodes == null ||
+                    this.modelConfig.TargetLanguageCodes == null)
+                {
+                    this.ParseModelPathForLanguages(modelPath);
+                }
+                else
+                {
+                    this.SourceLanguages = this.modelConfig.SourceLanguageCodes.Select(x => new IsoLanguage(x)).ToList();
+                    this.TargetLanguages = this.modelConfig.TargetLanguageCodes.Select(x => new IsoLanguage(x)).ToList();
+                }
+            }
+
             //
             this.AutoPreEditRuleCollections = new ObservableCollection<AutoEditRuleCollection>(
                 this.ModelConfig.AutoPreEditRuleCollectionGuids.Select(x => autoPreEditRuleCollections.SingleOrDefault(
@@ -654,6 +675,17 @@ public MTModel(
 
         internal void SaveModelConfig()
         {
+            if (this.ModelConfig == null)
+            {
+                this.ModelConfig = new MTModelConfig();
+            }
+
+            if (this.SourceLanguages != null && this.TargetLanguages != null)
+            {
+                this.ModelConfig.SourceLanguageCodes = new ObservableCollection<String>(this.SourceLanguages.Select(x => x.OriginalCode));
+                this.ModelConfig.TargetLanguageCodes = new ObservableCollection<String>(this.TargetLanguages.Select(x => x.OriginalCode));
+            }
+
             //The directory might not exists yet in case of customized models (i.e. copying of the base model
             //is not complete)
             if (Directory.Exists(this.InstallDir))
@@ -797,7 +829,6 @@ public MTModel(
         //This is used for online models, model uri is included for later download of models
         public MTModel(string modelPath, Uri modelUri, string yamlString = null)
         {
-
             this.ModelPath = modelPath;
             this.modelYaml = yamlString;
             if (yamlString != null)
@@ -1006,7 +1037,7 @@ public string TargetCodesString
         }
 
         public string ModelPath { get; internal set; }
-        public string InstallDir { get; }
+        public string InstallDir { get; set; }
         public bool Prioritized { get => _prioritized; set { _prioritized = value; NotifyPropertyChanged(); } }
 
         private MarianLog TrainingLog;

OpusCatMTEngine/MTModelConfig.cs

@@ -52,6 +52,12 @@ private void NotifyPropertyChanged([CallerMemberName] string propertyName = "")
         [YamlMember(Alias = "auto-post-edit-rule-collection-guids", ApplyNamingConventions = false)]
         public ObservableCollection<string> AutoPostEditRuleCollectionGuids { get; internal set; }
 
+        [YamlMember(Alias = "source-languages", ApplyNamingConventions = false)]
+        public ObservableCollection<string> SourceLanguageCodes { get; internal set; }
+
+        [YamlMember(Alias = "target-languages", ApplyNamingConventions = false)]
+        public ObservableCollection<string> TargetLanguageCodes { get; internal set; }
+
         private bool finetuningComplete;
 
         public MTModelConfig()

OpusCatMTEngine/ModelManager.cs

@@ -520,7 +520,7 @@ private void TatoebaModelListDownloaded(object sender, DownloadStringCompletedEv
             //Use distinct to remove duplicate entries
             foreach (var line in modelList.Split('\n').Distinct())
             {
-                var split = line.Split('\t');
+                var split = line.Split(new char[] { '\t' });
                 if (split.Length >= 4)
                 {
                     var modelPath = split[0];
@@ -529,17 +529,17 @@ private void TatoebaModelListDownloaded(object sender, DownloadStringCompletedEv
                     IEnumerable<string> sourceLangs = split[2].Split(',');
                     IEnumerable<string> targetLangs;
 
-                    //Multilingual models have a use-target-labels field, which contains the target
+                    //Multilingual models have a use-target-labels field (fifth column in tsv), which contains the target
                     //labels of the model, use that instead of target languages, as it will include
                     //script info (e.g. Latn or Cyrl).
-                    if (split.Length > 4)
+                    if (String.IsNullOrWhiteSpace(split[4]))
                     {
-                        //Remove the target code delimiters >>code<< -> code
-                        targetLangs = split[4].Replace("<","").Replace(">","").Split(',');
+                        targetLangs = split[3].Split(',');
                     }
                     else
                     {
-                        targetLangs = split[3].Split(',');
+                        //Remove the target code delimiters >>code<< -> code
+                        targetLangs = split[4].Replace("<", "").Replace(">", "").Split(',');
                     }
 
                     //Some entries might have empty source and target languages
@@ -549,8 +549,10 @@ private void TatoebaModelListDownloaded(object sender, DownloadStringCompletedEv
                     }
                     var model = new MTModel(modelPath.Replace(".zip", ""), modelUri);
                     model.ModelType = modelType;
-                    model.SourceLanguages = sourceLangs.Select(x => new IsoLanguage(x)).ToList();
+
                     model.TargetLanguages = targetLangs.Select(x => new IsoLanguage(x)).ToList();
+                    model.SourceLanguages = sourceLangs.Select(x => new IsoLanguage(x)).ToList();
+
                     this.onlineModels.Add(model);
                 }
             }