fix issue 4322, enable lda summary output

src/Microsoft.ML.Transforms/Text/LdaTransform.cs

@@ -103,7 +103,7 @@ internal sealed class Options : TransformInputBase
             public bool ResetRandomGenerator = LatentDirichletAllocationEstimator.Defaults.ResetRandomGenerator;
 
             [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to output the topic-word summary in text format", ShortName = "summary")]
-            public bool OutputTopicWordSummary;
+            public bool OutputTopicWordSummary = LatentDirichletAllocationEstimator.Defaults.OutputTopicWordSummary;
         }
 
         internal sealed class Column : OneToOneColumn
@@ -141,6 +141,9 @@ internal sealed class Column : OneToOneColumn
             [Argument(ArgumentType.AtMostOnce, HelpText = "Reset the random number generator for each document", ShortName = "reset")]
             public bool? ResetRandomGenerator;
 
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to output the topic-word summary in text format", ShortName = "summary")]
+            public bool? OutputTopicWordSummary;
+
             internal static Column Parse(string str)
             {
                 Contracts.AssertNonEmpty(str);
@@ -206,13 +209,17 @@ internal ModelParameters(IReadOnlyList<IReadOnlyList<WordItemScore>> wordScoresP
             }
         }
 
-        [BestFriend]
-        internal ModelParameters GetLdaDetails(int iinfo)
+        /// <summary>
+        /// Method to provide details about the topic discovered by LightLDA
+        /// </summary>
+        /// <param name="topicIndex">index of topic</param>
+        /// <returns></returns>
+        public ModelParameters GetLdaDetails(int topicIndex)
         {
-            Contracts.Assert(0 <= iinfo && iinfo < _ldas.Length);
+            Contracts.Assert(0 <= topicIndex && topicIndex < _ldas.Length);
 
-            var ldaState = _ldas[iinfo];
-            var mapping = _columnMappings[iinfo];
+            var ldaState = _ldas[topicIndex];
+            var mapping = _columnMappings[topicIndex];
 
             return ldaState.GetLdaSummary(mapping);
         }
@@ -760,9 +767,48 @@ private protected override void SaveModel(ModelSaveContext ctx)
             for (int i = 0; i < _ldas.Length; i++)
             {
                 _ldas[i].Save(ctx);
+
+                if(_columns[i].OutputTopicWordSummary)
+                    SaveTopicWordSummary(ctx, i);
             }
         }
 
+        private void SaveTopicWordSummary(ModelSaveContext ctx, int i)
+        {
+            var summary = GetLdaDetails(i);
+
+            ctx.SaveTextStream(WordTopicModelFilename, writer =>
+            {
+                if (summary.WordScoresPerTopic != null)
+                {
+                    int topId = 0;
+                    foreach (var wordScores in summary.WordScoresPerTopic)
+                    {
+                        foreach (var wordScore in wordScores)
+                        {
+                            writer.WriteLine($"Top[{topId}]: {wordScore.Word}\t{wordScore.Score}");
+                        }
+
+                        topId++;
+                    }
+                }
+
+                if (summary.ItemScoresPerTopic != null)
+                {
+                    int topId = 0;
+                    foreach (var itemScores in summary.ItemScoresPerTopic)
+                    {
+                        foreach (var itemScore in itemScores)
+                        {
+                            writer.WriteLine($"Top[{topId}]: {itemScore.Item}\t{itemScore.Score}");
+                        }
+
+                        topId++;
+                    }
+                }
+            });
+        }
+
         private static int GetFrequency(double value)
         {
             int result = (int)value;
@@ -994,6 +1040,7 @@ internal static class Defaults
             public const int NumberOfSummaryTermsPerTopic = 10;
             public const int NumberOfBurninIterations = 10;
             public const bool ResetRandomGenerator = false;
+            public const bool OutputTopicWordSummary = false;
         }
 
         private readonly IHost _host;
@@ -1100,6 +1147,10 @@ internal sealed class ColumnOptions
             /// Reset the random number generator for each document.
             /// </summary>
             public readonly bool ResetRandomGenerator;
+            /// <summary>
+            /// Whether to output the topic-word summary in text format
+            /// </summary>
+            public readonly bool OutputTopicWordSummary;
 
             /// <summary>
             /// Describes how the transformer handles one column pair.
@@ -1117,6 +1168,7 @@ internal sealed class ColumnOptions
             /// <param name="numberOfSummaryTermsPerTopic">The number of words to summarize the topic.</param>
             /// <param name="numberOfBurninIterations">The number of burn-in iterations.</param>
             /// <param name="resetRandomGenerator">Reset the random number generator for each document.</param>
+            /// <param name="outputTopicWordSummary">Whether to output the topic-word summary in text format.</param>
             public ColumnOptions(string name,
                 string inputColumnName = null,
                 int numberOfTopics = LatentDirichletAllocationEstimator.Defaults.NumberOfTopics,
@@ -1129,7 +1181,8 @@ public ColumnOptions(string name,
                 int maximumTokenCountPerDocument = LatentDirichletAllocationEstimator.Defaults.MaximumTokenCountPerDocument,
                 int numberOfSummaryTermsPerTopic = LatentDirichletAllocationEstimator.Defaults.NumberOfSummaryTermsPerTopic,
                 int numberOfBurninIterations = LatentDirichletAllocationEstimator.Defaults.NumberOfBurninIterations,
-                bool resetRandomGenerator = LatentDirichletAllocationEstimator.Defaults.ResetRandomGenerator)
+                bool resetRandomGenerator = LatentDirichletAllocationEstimator.Defaults.ResetRandomGenerator,
+                bool outputTopicWordSummary = LatentDirichletAllocationEstimator.Defaults.OutputTopicWordSummary)
             {
                 Contracts.CheckValue(name, nameof(name));
                 Contracts.CheckValueOrNull(inputColumnName);
@@ -1155,6 +1208,7 @@ public ColumnOptions(string name,
                 NumberOfSummaryTermsPerTopic = numberOfSummaryTermsPerTopic;
                 NumberOfBurninIterations = numberOfBurninIterations;
                 ResetRandomGenerator = resetRandomGenerator;
+                OutputTopicWordSummary = outputTopicWordSummary;
             }
 
             internal ColumnOptions(LatentDirichletAllocationTransformer.Column item, LatentDirichletAllocationTransformer.Options options) :
@@ -1170,7 +1224,8 @@ internal ColumnOptions(LatentDirichletAllocationTransformer.Column item, LatentD
                     item.NumMaxDocToken ?? options.NumMaxDocToken,
                     item.NumSummaryTermPerTopic ?? options.NumSummaryTermPerTopic,
                     item.NumBurninIterations ?? options.NumBurninIterations,
-                    item.ResetRandomGenerator ?? options.ResetRandomGenerator)
+                    item.ResetRandomGenerator ?? options.ResetRandomGenerator,
+                    item.OutputTopicWordSummary ?? options.OutputTopicWordSummary)
             {
             }
 

test/BaselineOutput/Common/EntryPoints/core_manifest.json

@@ -20900,6 +20900,18 @@
                   "IsNullable": true,
                   "Default": null
                 },
+                {
+                  "Name": "OutputTopicWordSummary",
+                  "Type": "Bool",
+                  "Desc": "Whether to output the topic-word summary in text format",
+                  "Aliases": [
+                    "summary"
+                  ],
+                  "Required": false,
+                  "SortOrder": 150.0,
+                  "IsNullable": true,
+                  "Default": null
+                },
                 {
                   "Name": "Name",
                   "Type": "String",

test/Microsoft.ML.TestFramework/DataPipe/TestDataPipe.cs

@@ -5,6 +5,8 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.IO.Compression;
+using System.Linq;
 using Microsoft.ML.CommandLine;
 using Microsoft.ML.Data;
 using Microsoft.ML.Internal.Utilities;
@@ -1047,6 +1049,18 @@ public void SavePipeLda()
                     "loader=Text{col=F1V:Num:0-2}",
                     "xf=Lda{col={name=Result src=F1V numtopic=3 alphasum=3 ns=3 reset=+ t=1} summary=+}",
                 }, forceDense: true);
+
+            // topic summary text file saved inside the model.zip file.
+            string name = TestName + ".zip";
+            string modelPath = GetOutputPath("SavePipe", name);
+            using (var file = Env.OpenInputFile(modelPath))
+            using (var strm = file.OpenReadStream())
+            using (var zip = new ZipArchive(strm, ZipArchiveMode.Read))
+            {
+                var entry = zip.Entries.First(source => source.Name == "word_topic_summary.txt");
+                Assert.True(entry != null);
+            }
+
             Done();
         }