fix issue 4322, enable lda summary output

src/Microsoft.ML.Transforms/Text/LdaTransform.cs

@@ -102,8 +102,8 @@ internal sealed class Options : TransformInputBase
             [Argument(ArgumentType.AtMostOnce, HelpText = "Reset the random number generator for each document", ShortName = "reset")]
             public bool ResetRandomGenerator = LatentDirichletAllocationEstimator.Defaults.ResetRandomGenerator;
 
-            [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to output the topic-word summary in text format", ShortName = "summary")]
-            public bool OutputTopicWordSummary;
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to output the topic-word summary in text format when saving the model to disk", ShortName = "summary")]
+            public bool OutputTopicWordSummary = LatentDirichletAllocationEstimator.Defaults.OutputTopicWordSummary;
         }
 
         internal sealed class Column : OneToOneColumn
@@ -141,6 +141,9 @@ internal sealed class Column : OneToOneColumn
             [Argument(ArgumentType.AtMostOnce, HelpText = "Reset the random number generator for each document", ShortName = "reset")]
             public bool? ResetRandomGenerator;
 
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to output the topic-word summary in text format when saving the model to disk", ShortName = "summary")]
+            public bool? OutputTopicWordSummary;
+
             internal static Column Parse(string str)
             {
                 Contracts.AssertNonEmpty(str);
@@ -206,13 +209,17 @@ internal ModelParameters(IReadOnlyList<IReadOnlyList<WordItemScore>> wordScoresP
             }
         }
 
-        [BestFriend]
-        internal ModelParameters GetLdaDetails(int iinfo)
+        /// <summary>
+        /// Method to provide details about the topics discovered by LightLDA
+        /// </summary>
+        /// <param name="columnIndex">index of column options pair</param>
+        /// <returns></returns>
+        public ModelParameters GetLdaDetails(int columnIndex)
         {
-            Contracts.Assert(0 <= iinfo && iinfo < _ldas.Length);
+            Contracts.Assert(0 <= columnIndex && columnIndex < _ldas.Length);
 
-            var ldaState = _ldas[iinfo];
-            var mapping = _columnMappings[iinfo];
+            var ldaState = _ldas[columnIndex];
+            var mapping = _columnMappings[columnIndex];
 
             return ldaState.GetLdaSummary(mapping);
         }
@@ -630,7 +637,7 @@ private static VersionInfo GetVersionInfo()
         private readonly List<VBuffer<ReadOnlyMemory<char>>> _columnMappings;
 
         private const string RegistrationName = "LightLda";
-        private const string WordTopicModelFilename = "word_topic_summary.txt";
+        private const string WordTopicModelFilename = "word_topic_summary-{0}.txt";
         internal const string Summary = "The LDA transform implements LightLDA, a state-of-the-art implementation of Latent Dirichlet Allocation.";
         internal const string UserName = "Latent Dirichlet Allocation Transform";
         internal const string ShortName = "LightLda";
@@ -760,9 +767,50 @@ private protected override void SaveModel(ModelSaveContext ctx)
             for (int i = 0; i < _ldas.Length; i++)
             {
                 _ldas[i].Save(ctx);
+
+                if(_columns[i].OutputTopicWordSummary)
+                    SaveTopicWordSummary(ctx, i);
             }
         }
 
+        private void SaveTopicWordSummary(ModelSaveContext ctx, int i)
+        {
+            var summary = GetLdaDetails(i);
+
+            var columnName = _columns[i].Name;
+
+            ctx.SaveTextStream(String.Format(WordTopicModelFilename, columnName), writer =>
+            {
+                if (summary.WordScoresPerTopic != null)
+                {
+                    int topId = 0;
+                    foreach (var wordScores in summary.WordScoresPerTopic)
+                    {
+                        foreach (var wordScore in wordScores)
+                        {
+                            writer.WriteLine($"Topic[{topId}]: {wordScore.Word}\t{wordScore.Score}");
+                        }
+
+                        topId++;
+                    }
+                }
+
+                if (summary.ItemScoresPerTopic != null)
+                {
+                    int topId = 0;
+                    foreach (var itemScores in summary.ItemScoresPerTopic)
+                    {
+                        foreach (var itemScore in itemScores)
+                        {
+                            writer.WriteLine($"Topic[{topId}]: {itemScore.Item}\t{itemScore.Score}");
+                        }
+
+                        topId++;
+                    }
+                }
+            });
+        }
+
         private static int GetFrequency(double value)
         {
             int result = (int)value;
@@ -994,6 +1042,7 @@ internal static class Defaults
             public const int NumberOfSummaryTermsPerTopic = 10;
             public const int NumberOfBurninIterations = 10;
             public const bool ResetRandomGenerator = false;
+            public const bool OutputTopicWordSummary = false;
         }
 
         private readonly IHost _host;
@@ -1014,6 +1063,7 @@ internal static class Defaults
         /// <param name="likelihoodInterval">Compute log likelihood over local dataset on this iteration interval.</param>
         /// <param name="numberOfBurninIterations">The number of burn-in iterations.</param>
         /// <param name="resetRandomGenerator">Reset the random number generator for each document.</param>
+        /// <param name="outputTopicWordSummary">Whether to output the topic-word summary in text format when saving the model to disk.</param>
         internal LatentDirichletAllocationEstimator(IHostEnvironment env,
             string outputColumnName, string inputColumnName = null,
             int numberOfTopics = Defaults.NumberOfTopics,
@@ -1026,10 +1076,11 @@ internal LatentDirichletAllocationEstimator(IHostEnvironment env,
             int numberOfSummaryTermsPerTopic = Defaults.NumberOfSummaryTermsPerTopic,
             int likelihoodInterval = Defaults.LikelihoodInterval,
             int numberOfBurninIterations = Defaults.NumberOfBurninIterations,
-            bool resetRandomGenerator = Defaults.ResetRandomGenerator)
+            bool resetRandomGenerator = Defaults.ResetRandomGenerator,
+            bool outputTopicWordSummary = Defaults.OutputTopicWordSummary)
             : this(env, new[] { new ColumnOptions(outputColumnName, inputColumnName ?? outputColumnName,
                 numberOfTopics, alphaSum, beta, samplingStepCount, maximumNumberOfIterations, likelihoodInterval, numberOfThreads, maximumTokenCountPerDocument,
-                numberOfSummaryTermsPerTopic, numberOfBurninIterations, resetRandomGenerator) })
+                numberOfSummaryTermsPerTopic, numberOfBurninIterations, resetRandomGenerator, outputTopicWordSummary) })
         { }
 
         /// <include file='doc.xml' path='doc/members/member[@name="LightLDA"]/*' />
@@ -1100,6 +1151,10 @@ internal sealed class ColumnOptions
             /// Reset the random number generator for each document.
             /// </summary>
             public readonly bool ResetRandomGenerator;
+            /// <summary>
+            /// Whether to output the topic-word summary in text format when saving the model to disk.
+            /// </summary>
+            public readonly bool OutputTopicWordSummary;
 
             /// <summary>
             /// Describes how the transformer handles one column pair.
@@ -1117,6 +1172,7 @@ internal sealed class ColumnOptions
             /// <param name="numberOfSummaryTermsPerTopic">The number of words to summarize the topic.</param>
             /// <param name="numberOfBurninIterations">The number of burn-in iterations.</param>
             /// <param name="resetRandomGenerator">Reset the random number generator for each document.</param>
+            /// <param name="outputTopicWordSummary">Whether to output the topic-word summary in text format when saving the model to disk.</param>
             public ColumnOptions(string name,
                 string inputColumnName = null,
                 int numberOfTopics = LatentDirichletAllocationEstimator.Defaults.NumberOfTopics,
@@ -1129,7 +1185,8 @@ public ColumnOptions(string name,
                 int maximumTokenCountPerDocument = LatentDirichletAllocationEstimator.Defaults.MaximumTokenCountPerDocument,
                 int numberOfSummaryTermsPerTopic = LatentDirichletAllocationEstimator.Defaults.NumberOfSummaryTermsPerTopic,
                 int numberOfBurninIterations = LatentDirichletAllocationEstimator.Defaults.NumberOfBurninIterations,
-                bool resetRandomGenerator = LatentDirichletAllocationEstimator.Defaults.ResetRandomGenerator)
+                bool resetRandomGenerator = LatentDirichletAllocationEstimator.Defaults.ResetRandomGenerator,
+                bool outputTopicWordSummary = LatentDirichletAllocationEstimator.Defaults.OutputTopicWordSummary)
             {
                 Contracts.CheckValue(name, nameof(name));
                 Contracts.CheckValueOrNull(inputColumnName);
@@ -1155,6 +1212,7 @@ public ColumnOptions(string name,
                 NumberOfSummaryTermsPerTopic = numberOfSummaryTermsPerTopic;
                 NumberOfBurninIterations = numberOfBurninIterations;
                 ResetRandomGenerator = resetRandomGenerator;
+                OutputTopicWordSummary = outputTopicWordSummary;
             }
 
             internal ColumnOptions(LatentDirichletAllocationTransformer.Column item, LatentDirichletAllocationTransformer.Options options) :
@@ -1170,7 +1228,8 @@ internal ColumnOptions(LatentDirichletAllocationTransformer.Column item, LatentD
                     item.NumMaxDocToken ?? options.NumMaxDocToken,
                     item.NumSummaryTermPerTopic ?? options.NumSummaryTermPerTopic,
                     item.NumBurninIterations ?? options.NumBurninIterations,
-                    item.ResetRandomGenerator ?? options.ResetRandomGenerator)
+                    item.ResetRandomGenerator ?? options.ResetRandomGenerator,
+                    item.OutputTopicWordSummary ?? options.OutputTopicWordSummary)
             {
             }
 

test/BaselineOutput/Common/EntryPoints/core_manifest.json

@@ -20900,6 +20900,18 @@
                   "IsNullable": true,
                   "Default": null
                 },
+                {
+                  "Name": "OutputTopicWordSummary",
+                  "Type": "Bool",
+                  "Desc": "Whether to output the topic-word summary in text format when saving the model to disk",
+                  "Aliases": [
+                    "summary"
+                  ],
+                  "Required": false,
+                  "SortOrder": 150.0,
+                  "IsNullable": true,
+                  "Default": null
+                },
                 {
                   "Name": "Name",
                   "Type": "String",
@@ -21112,7 +21124,7 @@
         {
           "Name": "OutputTopicWordSummary",
           "Type": "Bool",
-          "Desc": "Whether to output the topic-word summary in text format",
+          "Desc": "Whether to output the topic-word summary in text format when saving the model to disk",
           "Aliases": [
             "summary"
           ],

test/Microsoft.ML.Functional.Tests/IntrospectiveTraining.cs

@@ -182,15 +182,24 @@ public void InspectLdaModelParameters()
 
             // Define the pipeline.
             var pipeline = mlContext.Transforms.Text.ProduceWordBags("SentimentBag", "SentimentText")
-                .Append(mlContext.Transforms.Text.LatentDirichletAllocation("Features", "SentimentBag", numberOfTopics: numTopics, maximumNumberOfIterations: 10));
+                .Append(mlContext.Transforms.Text.LatentDirichletAllocation("Features", "SentimentBag", 
+                numberOfTopics: numTopics, maximumNumberOfIterations: 10));
 
             // Fit the pipeline.
             var model = pipeline.Fit(data);
 
             // Get the trained LDA model.
-            // TODO #2197: Get the topics and summaries from the model.
             var ldaTransform = model.LastTransformer;
 
+            // Get the topics and summaries from the model.
+            var ldaDetails = ldaTransform.GetLdaDetails(0);
+            Assert.False(ldaDetails.ItemScoresPerTopic == null && ldaDetails.WordScoresPerTopic == null);
+            if(ldaDetails.ItemScoresPerTopic != null)
+                Assert.Equal(numTopics, ldaDetails.ItemScoresPerTopic.Count);
+            if (ldaDetails.WordScoresPerTopic != null)
+                Assert.Equal(numTopics, ldaDetails.WordScoresPerTopic.Count);
+
+
             // Transform the data.
             var transformedData = model.Transform(data);
 

test/Microsoft.ML.TestFramework/DataPipe/TestDataPipe.cs

@@ -5,6 +5,8 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.IO.Compression;
+using System.Linq;
 using Microsoft.ML.CommandLine;
 using Microsoft.ML.Data;
 using Microsoft.ML.Internal.Utilities;
@@ -1047,6 +1049,18 @@ public void SavePipeLda()
                     "loader=Text{col=F1V:Num:0-2}",
                     "xf=Lda{col={name=Result src=F1V numtopic=3 alphasum=3 ns=3 reset=+ t=1} summary=+}",
                 }, forceDense: true);
+
+            // topic summary text file saved inside the model.zip file.
+            string name = TestName + ".zip";
+            string modelPath = GetOutputPath("SavePipe", name);
+            using (var file = Env.OpenInputFile(modelPath))
+            using (var strm = file.OpenReadStream())
+            using (var zip = new ZipArchive(strm, ZipArchiveMode.Read))
+            {
+                var entry = zip.Entries.First(source => source.Name == "word_topic_summary-Result.txt");
+                Assert.True(entry != null);
+            }
+
             Done();
         }