Add hosting config support to control 3.x log disablement

src/WebJobs.Script.WebHost/Diagnostics/ILoggingBuilderExtensions.cs

@@ -8,12 +8,12 @@ namespace Microsoft.Extensions.Logging
 {
     public static class ILoggingBuilderExtensions
     {
-        public static void AddWebJobsSystem<T>(this ILoggingBuilder builder) where T : SystemLoggerProvider
+        public static void AddWebJobsSystem<T>(this ILoggingBuilder builder, bool restrictHostLogs = false) where T : SystemLoggerProvider
         {
             builder.Services.AddSingleton<ILoggerProvider, T>();
 
             // Log all logs to SystemLogger
-            builder.AddDefaultWebJobsFilters<T>(LogLevel.Trace);
+            builder.AddDefaultWebJobsFilters<T>(LogLevel.Trace, restrictHostLogs);
         }
     }
 }

src/WebJobs.Script.WebHost/WebScriptHostBuilderExtension.cs

@@ -75,9 +75,12 @@ public static IHostBuilder AddWebScriptHost(this IHostBuilder builder, IServiceP
                 })
                 .ConfigureLogging(loggingBuilder =>
                 {
+                    var hostingConfigOptions = rootServiceProvider.GetService<IOptions<FunctionsHostingConfigOptions>>();
+                    var restrictHostLogs = RestrictHostLogs(hostingConfigOptions.Value);
+
                     loggingBuilder.Services.AddSingleton<ILoggerFactory, ScriptLoggerFactory>();
+                    loggingBuilder.AddWebJobsSystem<SystemLoggerProvider>(restrictHostLogs);
 
-                    loggingBuilder.AddWebJobsSystem<SystemLoggerProvider>();
                     if (environment.IsAzureMonitorEnabled())
                     {
                         loggingBuilder.Services.AddSingleton<ILoggerProvider, AzureMonitorDiagnosticLoggerProvider>();
@@ -170,6 +173,12 @@ private static void ConfigureRegisteredBuilders<TBuilder>(TBuilder builder, ISer
             }
         }
 
+        private static bool RestrictHostLogs(FunctionsHostingConfigOptions options)
+        {
+            // Feature flag should take precedence over the host configuration
+            return !FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagEnableHostLogs) && options.RestrictHostLogs;
+        }
+
         /// <summary>
         /// Used internally to register Newtonsoft formatters with our ScriptHost.
         /// </summary>

src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs

@@ -85,6 +85,22 @@ public bool SwtIssuerEnabled
             }
         }
 
+        /// <summary>
+        /// Gets or sets a value indicating whether non-critical logs should be disabled in the host.
+        /// </summary>
+        public bool RestrictHostLogs
+        {
+            get
+            {
+                return GetFeatureOrDefault(ScriptConstants.HostingConfigRestrictHostLogs, "1") == "1";
+            }
+
+            set
+            {
+                _features[ScriptConstants.HostingConfigRestrictHostLogs] = value ? "1" : "0";
+            }
+        }
+
         /// <summary>
         /// Gets feature by name.
         /// </summary>

src/WebJobs.Script/Extensions/ScriptLoggingBuilderExtensions.cs

@@ -6,7 +6,6 @@
 using System.Linq;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Azure.WebJobs.Script;
-using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Configuration;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
@@ -15,17 +14,25 @@ namespace Microsoft.Extensions.Logging
 {
     public static class ScriptLoggingBuilderExtensions
     {
-        private static ConcurrentDictionary<string, bool> _filteredCategoryCache = new ConcurrentDictionary<string, bool>();
+        private static ConcurrentDictionary<string, bool> _filteredCategoryCache = new ();
+        private static ImmutableArray<string> _allowedLogCategoryPrefixes = new ();
 
-        public static ILoggingBuilder AddDefaultWebJobsFilters(this ILoggingBuilder builder)
+        // For testing only
+        internal static ImmutableArray<string> AllowedSystemLogPrefixes => _allowedLogCategoryPrefixes;
+
+        public static ILoggingBuilder AddDefaultWebJobsFilters(this ILoggingBuilder builder, bool restrictHostLogs = false)
         {
+            SetSystemLogCategoryPrefixes(restrictHostLogs);
+
             builder.SetMinimumLevel(LogLevel.None);
             builder.AddFilter((c, l) => Filter(c, l, LogLevel.Information));
             return builder;
         }
 
-        public static ILoggingBuilder AddDefaultWebJobsFilters<T>(this ILoggingBuilder builder, LogLevel level) where T : ILoggerProvider
+        public static ILoggingBuilder AddDefaultWebJobsFilters<T>(this ILoggingBuilder builder, LogLevel level, bool restrictHostLogs = false) where T : ILoggerProvider
         {
+            SetSystemLogCategoryPrefixes(restrictHostLogs);
+
             builder.AddFilter<T>(null, LogLevel.None);
             builder.AddFilter<T>((c, l) => Filter(c, l, level));
             return builder;
@@ -38,11 +45,18 @@ internal static bool Filter(string category, LogLevel actualLevel, LogLevel minL
 
         private static bool IsFiltered(string category)
         {
-            ImmutableArray<string> systemLogCategoryPrefixes = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagEnableHostLogs)
-                                                                ? ScriptConstants.SystemLogCategoryPrefixes
-                                                                : ScriptConstants.RestrictedSystemLogCategoryPrefixes;
+            return _filteredCategoryCache.GetOrAdd(category, c => _allowedLogCategoryPrefixes.Any(p => c.StartsWith(p)));
+        }
+
+        private static void SetSystemLogCategoryPrefixes(bool restrictHostLogs)
+        {
+            var previous = _allowedLogCategoryPrefixes;
+            _allowedLogCategoryPrefixes = restrictHostLogs ? ScriptConstants.RestrictedSystemLogCategoryPrefixes : ScriptConstants.SystemLogCategoryPrefixes;
 
-            return _filteredCategoryCache.GetOrAdd(category, c => systemLogCategoryPrefixes.Any(p => category.StartsWith(p)));
+            if (!previous.IsDefault && !previous.SequenceEqual(_allowedLogCategoryPrefixes))
+            {
+                _filteredCategoryCache.Clear();
+            }
         }
 
         public static void AddConsoleIfEnabled(this ILoggingBuilder builder, HostBuilderContext context)

src/WebJobs.Script/ScriptConstants.cs

@@ -127,6 +127,7 @@ public static class ScriptConstants
         public const string FeatureFlagEnableHostLogs = "EnableHostLogs";
         public const string HostingConfigSwtAuthenticationEnabled = "SwtAuthenticationEnabled";
         public const string HostingConfigSwtIssuerEnabled = "SwtIssuerEnabled";
+        public const string HostingConfigRestrictHostLogs = "RestrictHostLogs";
 
         public const string SiteAzureFunctionsUriFormat = "https://{0}.azurewebsites.net/azurefunctions";
         public const string ScmSiteUriFormat = "https://{0}.scm.azurewebsites.net";

test/WebJobs.Script.Tests.Shared/TestHostBuilderExtensions.cs

@@ -29,13 +29,13 @@ namespace Microsoft.WebJobs.Script.Tests
 {
     public static class TestHostBuilderExtensions
     {
-        public static IHostBuilder ConfigureDefaultTestWebScriptHost(this IHostBuilder builder, Action<ScriptApplicationHostOptions> configure = null, bool runStartupHostedServices = false)
+        public static IHostBuilder ConfigureDefaultTestWebScriptHost(this IHostBuilder builder, Action<ScriptApplicationHostOptions> configure = null, bool runStartupHostedServices = false, IEnvironment environment = null)
         {
-            return builder.ConfigureDefaultTestWebScriptHost(null, configure, runStartupHostedServices);
+            return builder.ConfigureDefaultTestWebScriptHost(null, configure, runStartupHostedServices, environment: environment);
         }
 
         public static IHostBuilder ConfigureDefaultTestWebScriptHost(this IHostBuilder builder, Action<IWebJobsBuilder> configureWebJobs,
-            Action<ScriptApplicationHostOptions> configure = null, bool runStartupHostedServices = false, Action<IServiceCollection> configureRootServices = null)
+            Action<ScriptApplicationHostOptions> configure = null, bool runStartupHostedServices = false, Action<IServiceCollection> configureRootServices = null, IEnvironment environment = null)
         {
             var webHostOptions = new ScriptApplicationHostOptions()
             {
@@ -48,9 +48,18 @@ public static IHostBuilder ConfigureDefaultTestWebScriptHost(this IHostBuilder b
 
             // Register root services
             var services = new ServiceCollection();
+
+            if (environment is not null)
+            {
+                services.AddSingleton<IEnvironment>(environment);
+            }
+            else
+            {
+                services.AddSingleton<IEnvironment>(SystemEnvironment.Instance);
+            }
+
             AddMockedSingleton<IDebugStateProvider>(services);
             AddMockedSingleton<IScriptHostManager>(services);
-            AddMockedSingleton<IEnvironment>(services);
             AddMockedSingleton<IScriptWebHostEnvironment>(services);
             AddMockedSingleton<IEventGenerator>(services);
             AddMockedSingleton<IFunctionInvocationDispatcherFactory>(services);

test/WebJobs.Script.Tests/Configuration/DefaultDependencyValidatorTests.cs

@@ -36,27 +36,28 @@ public async Task Validator_AllValid()
         [Fact]
         public async Task Validator_InvalidServices_LogsError()
         {
-            SystemEnvironment.Instance.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableHostLogs);
-
-            LogMessage invalidServicesMessage = await RunTest(configureJobHost: s =>
+            using (new TestScopedEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableHostLogs))
             {
-                s.AddSingleton<IHostedService, MyHostedService>();
-                s.AddSingleton<IScriptEventManager, MyScriptEventManager>();
-                s.AddSingleton<IMetricsLogger>(new MyMetricsLogger());
-
-                // Try removing system logger
-                var descriptor = s.Single(p => p.ImplementationType == typeof(SystemLoggerProvider));
-                s.Remove(descriptor);
-            });
-
-            Assert.NotNull(invalidServicesMessage);
-
-            IEnumerable<string> messageLines = invalidServicesMessage.Exception.Message.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries).Select(p => p.Trim());
-            Assert.Equal(5, messageLines.Count());
-            Assert.Contains(messageLines, p => p.StartsWith("[Invalid]") && p.EndsWith(typeof(MyHostedService).AssemblyQualifiedName));
-            Assert.Contains(messageLines, p => p.StartsWith("[Invalid]") && p.EndsWith(typeof(MyScriptEventManager).AssemblyQualifiedName));
-            Assert.Contains(messageLines, p => p.StartsWith("[Invalid]") && p.EndsWith(typeof(MyMetricsLogger).AssemblyQualifiedName));
-            Assert.Contains(messageLines, p => p.StartsWith("[Missing]") && p.EndsWith(typeof(SystemLoggerProvider).AssemblyQualifiedName));
+                LogMessage invalidServicesMessage = await RunTest(configureJobHost: s =>
+                {
+                    s.AddSingleton<IHostedService, MyHostedService>();
+                    s.AddSingleton<IScriptEventManager, MyScriptEventManager>();
+                    s.AddSingleton<IMetricsLogger>(new MyMetricsLogger());
+
+                    // Try removing system logger
+                    var descriptor = s.Single(p => p.ImplementationType == typeof(SystemLoggerProvider));
+                    s.Remove(descriptor);
+                });
+
+                Assert.NotNull(invalidServicesMessage);
+
+                IEnumerable<string> messageLines = invalidServicesMessage.Exception.Message.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries).Select(p => p.Trim());
+                Assert.Equal(5, messageLines.Count());
+                Assert.Contains(messageLines, p => p.StartsWith("[Invalid]") && p.EndsWith(typeof(MyHostedService).AssemblyQualifiedName));
+                Assert.Contains(messageLines, p => p.StartsWith("[Invalid]") && p.EndsWith(typeof(MyScriptEventManager).AssemblyQualifiedName));
+                Assert.Contains(messageLines, p => p.StartsWith("[Invalid]") && p.EndsWith(typeof(MyMetricsLogger).AssemblyQualifiedName));
+                Assert.Contains(messageLines, p => p.StartsWith("[Missing]") && p.EndsWith(typeof(SystemLoggerProvider).AssemblyQualifiedName));
+            }
         }
 
         [Fact]

test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs

@@ -135,14 +135,35 @@ public void SwtIssuerEnabled_ReturnsExpectedValue()
             Assert.False(options.SwtIssuerEnabled);
         }
 
-        internal static IHostBuilder GetScriptHostBuilder(string fileName, string fileContent)
+        [Fact]
+        public void RestrictHostLogs_ReturnsExpectedValue()
+        {
+            FunctionsHostingConfigOptions options = new FunctionsHostingConfigOptions();
+
+            // defaults to true
+            Assert.True(options.RestrictHostLogs);
+
+            // returns true when explicitly enabled
+            options.Features[ScriptConstants.HostingConfigRestrictHostLogs] = "1";
+            Assert.True(options.RestrictHostLogs);
+
+            // returns false when disabled
+            options.Features[ScriptConstants.HostingConfigRestrictHostLogs] = "0";
+            Assert.False(options.RestrictHostLogs);
+        }
+
+        internal static IHostBuilder GetScriptHostBuilder(string fileName, string fileContent, IEnvironment environment = null)
         {
             if (!string.IsNullOrEmpty(fileContent))
             {
                 File.WriteAllText(fileName, fileContent);
             }
 
-            TestEnvironment environment = new TestEnvironment();
+            if (environment is null)
+            {
+                environment = new TestEnvironment();
+            }
+
             environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionsPlatformConfigFilePath, fileName);
 
             IHost webHost = new HostBuilder()
@@ -161,7 +182,7 @@ internal static IHostBuilder GetScriptHostBuilder(string fileName, string fileCo
                 {
                     services.AddSingleton<TestService>();
                 })
-                .ConfigureDefaultTestWebScriptHost(null, configureRootServices: (services) =>
+                .ConfigureDefaultTestWebScriptHost(null, environment: environment, configureRootServices: (services) =>
                 {
                     services.AddSingleton(webHost.Services.GetService<IOptions<FunctionsHostingConfigOptions>>());
                     services.AddSingleton(webHost.Services.GetService<IOptionsMonitor<FunctionsHostingConfigOptions>>());
@@ -170,7 +191,7 @@ internal static IHostBuilder GetScriptHostBuilder(string fileName, string fileCo
 
         public class TestService
         {
-            public TestService(IOptions<Config.FunctionsHostingConfigOptions> options, IOptionsMonitor<Config.FunctionsHostingConfigOptions> monitor)
+            public TestService(IOptions<FunctionsHostingConfigOptions> options, IOptionsMonitor<FunctionsHostingConfigOptions> monitor)
             {
                 Options = options;
                 Monitor = monitor;
@@ -180,9 +201,9 @@ public TestService(IOptions<Config.FunctionsHostingConfigOptions> options, IOpti
                 });
             }
 
-            public IOptions<Config.FunctionsHostingConfigOptions> Options { get; set; }
+            public IOptions<FunctionsHostingConfigOptions> Options { get; set; }
 
-            public IOptionsMonitor<Config.FunctionsHostingConfigOptions> Monitor { get; set; }
+            public IOptionsMonitor<FunctionsHostingConfigOptions> Monitor { get; set; }
         }
     }
 }

test/WebJobs.Script.Tests/Configuration/RestrictHostLogsTests.cs

@@ -0,0 +1,75 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System.Collections.Immutable;
+using System.IO;
+using System.Reflection;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs.Script.Config;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using WebJobs.Script.Tests;
+using Xunit;
+using static Microsoft.Azure.WebJobs.Script.Tests.Configuration.FunctionsHostingConfigOptionsTest;
+
+namespace Microsoft.Azure.WebJobs.Script.Tests.Configuration
+{
+    public class RestrictHostLogsTests : IAsyncLifetime
+    {
+        public Task InitializeAsync()
+        {
+            return Task.CompletedTask;
+        }
+
+        public Task DisposeAsync()
+        {
+            // Reset the static _allowedLogCategoryPrefixes field after each test to the default value
+            typeof(ScriptLoggingBuilderExtensions)
+                .GetField("_allowedLogCategoryPrefixes", BindingFlags.Static | BindingFlags.NonPublic)
+                .SetValue(null, default(ImmutableArray<string>));
+
+            SystemEnvironment.Instance.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, null);
+            return Task.CompletedTask;
+        }
+
+        [Theory]
+        [InlineData(true, false, true)] // RestrictHostLogs is true, FeatureFlag is not set, should result in **restricted** logs. This is the default behaviour of the host.
+        [InlineData(false, true, false)] // RestrictHostLogs is false, FeatureFlag is set, should result in unrestricted logs
+        [InlineData(true, true, false)] // RestrictHostLogs is true, FeatureFlag is set, should result in unrestricted logs
+        [InlineData(false, false, false)] // RestrictHostLogs is false, FeatureFlag is not set, should result in unrestricted log
+        public async Task RestirctHostLogs_SetsCorrectSystemLogPrefix(bool restrictHostLogs, bool setFeatureFlag, bool shouldResultInRestrictedSystemLogs)
+        {
+            using (TempDirectory tempDir = new TempDirectory())
+            {
+                string fileName = Path.Combine(tempDir.Path, "settings.txt");
+                string fileContent = restrictHostLogs ? string.Empty : $"{ScriptConstants.HostingConfigRestrictHostLogs}=false";
+
+                if (setFeatureFlag)
+                {
+                    SystemEnvironment.Instance.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableHostLogs);
+                }
+
+                IHost host = GetScriptHostBuilder(fileName, fileContent).Build();
+                var testService = host.Services.GetService<TestService>();
+
+                await host.StartAsync();
+                await Task.Delay(1000);
+
+                Assert.Equal(restrictHostLogs, testService.Options.Value.RestrictHostLogs);
+                Assert.Equal(setFeatureFlag, FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagEnableHostLogs, SystemEnvironment.Instance));
+
+                if (shouldResultInRestrictedSystemLogs)
+                {
+                    Assert.Equal(ScriptConstants.RestrictedSystemLogCategoryPrefixes, ScriptLoggingBuilderExtensions.AllowedSystemLogPrefixes);
+                }
+                else
+                {
+                    Assert.Equal(ScriptConstants.SystemLogCategoryPrefixes, ScriptLoggingBuilderExtensions.AllowedSystemLogPrefixes);
+                }
+
+                await host.StopAsync();
+            }
+        }
+    }
+}