Configure services to output to Azure Monitor AB#16831 (#6343)

* Configure logging and telemetry for Azure Monitor

* Enrich tracing with additional request properties

* Update lists of activity sources used in each project

* Fix duplicate package reference

Apps/Admin/Server/appsettings.json

@@ -149,13 +149,21 @@
     "OpenTelemetry": {
         "Enabled": false,
         "Sources": [
-            "ClinicalDocumentService",
-            "PatientService",
+            "RestImmunizationAdminDelegate",
+            "RestVaccineStatusDelegate",
+            "BroadcastService",
+
+            "PatientQueryStrategy",
             "ClientRegistriesDelegate",
+
             "DbAuditLogger",
-            "RedisAuditLogger"
+            "RedisAuditLogger",
+            "RestTokenSwapDelegate",
+            "ClientRegistriesDelegate",
+            "AccessTokenService",
+            "PatientService"
         ],
-        "ServiceName": "BlazorAdmin",
+        "ServiceName": "Admin",
         "TraceConsoleExporterEnabled": false,
         "ZipkinEnabled": false,
         "ZipkinUri": "",

Apps/ClinicalDocument/src/Startup.cs

@@ -100,6 +100,7 @@ public void Configure(IApplicationBuilder app)
             this.startupConfig.UseSwagger(app);
             this.startupConfig.UseHttp(app);
             this.startupConfig.UseAuth(app);
+            this.startupConfig.EnrichTracing(app);
             this.startupConfig.UseRest(app);
         }
     }

Apps/ClinicalDocument/src/appsettings.json

@@ -26,9 +26,7 @@
         ]
     },
     "RequestLogging": {
-        "ExcludedPaths": [
-            "/health"
-        ]
+        "ExcludedPaths": ["/health"]
     },
     "OpenIdConnect": {
         "Authority": "https://loginproxy.gov.bc.ca/auth/realms/health-gateway-gold",
@@ -65,18 +63,20 @@
         "Enabled": false,
         "Sources": [
             "ClinicalDocumentService",
-            "PatientService",
-            "ClientRegistriesDelegate",
+
             "DbAuditLogger",
-            "RedisAuditLogger"
+            "RedisAuditLogger",
+            "RestTokenSwapDelegate",
+            "ClientRegistriesDelegate",
+            "AccessTokenService",
+            "PatientService",
+            "PersonalAccountsService"
         ],
         "ServiceName": "ClinicalDocumentService",
         "TraceConsoleExporterEnabled": false,
         "ZipkinEnabled": false,
         "ZipkinUri": "",
-        "IgnorePathPrefixes": [
-            "/health"
-        ]
+        "IgnorePathPrefixes": ["/health"]
     },
     "PhsaV2": {
         "TokenBaseUrl": "https://phsa-ccd.azure-api.net/identity",

Apps/Common/src/AspNetConfiguration/Modules/Observability.cs

@@ -17,11 +17,15 @@
 namespace HealthGateway.Common.AspNetConfiguration.Modules
 {
     using System;
+    using System.Diagnostics;
     using System.Diagnostics.CodeAnalysis;
     using System.Globalization;
     using System.Net;
     using System.Reflection;
+    using Azure.Monitor.OpenTelemetry.Exporter;
+    using HealthGateway.Common.Constants;
     using HealthGateway.Common.Models;
+    using HealthGateway.Common.Utils;
     using Microsoft.AspNetCore.Builder;
     using Microsoft.AspNetCore.Http;
     using Microsoft.Extensions.Configuration;
@@ -77,10 +81,9 @@ public static IApplicationBuilder UseDefaultHttpRequestLogging(this IApplication
                 opts =>
                 {
                     opts.IncludeQueryInRequestPath = true;
-                    opts.GetLevel = (httpCtx, _, exception) => ExcludePaths(httpCtx, exception, excludePaths ?? Array.Empty<string>());
+                    opts.GetLevel = (httpCtx, _, exception) => ExcludePaths(httpCtx, exception, excludePaths ?? []);
                     opts.EnrichDiagnosticContext = (diagCtx, httpCtx) =>
                     {
-                        diagCtx.Set("User", httpCtx.User.Identity?.Name ?? string.Empty);
                         diagCtx.Set("Host", httpCtx.Request.Host.Value);
                         diagCtx.Set("ContentLength", httpCtx.Response.ContentLength?.ToString(CultureInfo.InvariantCulture) ?? string.Empty);
                         diagCtx.Set("Protocol", httpCtx.Request.Protocol);
@@ -134,11 +137,21 @@ public static IServiceCollection AddOpenTelemetryDefaults(this IServiceCollectio
                             .AddNpgsql()
                             ;
 
+                        foreach (string source in otlpConfig.Sources)
+                        {
+                            builder.AddSource(source);
+                        }
+
                         if (otlpConfig.TraceConsoleExporterEnabled)
                         {
                             builder.AddConsoleExporter();
                         }
 
+                        if (!string.IsNullOrEmpty(otlpConfig.AzureConnectionString))
+                        {
+                            builder.AddAzureMonitorTraceExporter(options => options.ConnectionString = otlpConfig.AzureConnectionString);
+                        }
+
                         if (otlpConfig.Endpoint != null)
                         {
                             builder.AddOtlpExporter(
@@ -163,6 +176,11 @@ public static IServiceCollection AddOpenTelemetryDefaults(this IServiceCollectio
                             builder.AddConsoleExporter();
                         }
 
+                        if (!string.IsNullOrEmpty(otlpConfig.AzureConnectionString))
+                        {
+                            builder.AddAzureMonitorMetricExporter(options => options.ConnectionString = otlpConfig.AzureConnectionString);
+                        }
+
                         if (otlpConfig.Endpoint != null)
                         {
                             builder.AddOtlpExporter(
@@ -180,15 +198,57 @@ public static IServiceCollection AddOpenTelemetryDefaults(this IServiceCollectio
         }
 #pragma warning restore CA1506
 
+        /// <summary>
+        /// Adds middleware to enrich tracing telemetry with additional properties.
+        /// </summary>
+        /// <param name="app">The application builder provider.</param>
+        /// <param name="configuration">The configuration to use.</param>
+        public static void EnrichTracing(IApplicationBuilder app, IConfiguration configuration)
+        {
+            OpenTelemetryConfig openTelemetryConfig = new();
+            configuration.GetSection("OpenTelemetry").Bind(openTelemetryConfig);
+
+            if (openTelemetryConfig.Enabled)
+            {
+                app.Use(
+                    async (context, next) =>
+                    {
+                        string subject = GetRequestHdid(context);
+                        EnrichActivityWithBaggage("Subject", subject, Activity.Current);
+
+                        string user = context.User.Identity?.Name ?? string.Empty;
+                        EnrichActivityWithBaggage("User", user, Activity.Current);
+
+                        await next();
+                    });
+            }
+        }
+
+        private static string GetRequestHdid(HttpContext context)
+        {
+            return HttpContextHelper.GetResourceHdid(context, FhirSubjectLookupMethod.Route)
+                   ?? HttpContextHelper.GetResourceHdid(context, FhirSubjectLookupMethod.Parameter)
+                   ?? string.Empty;
+        }
+
+        private static void EnrichActivityWithBaggage(string key, string value, Activity? activity)
+        {
+            if (string.IsNullOrEmpty(value))
+            {
+                return;
+            }
+
+            activity?.AddBaggage(key, value);
+        }
+
         private static void ConfigureDefaultLogging(this LoggerConfiguration loggerConfiguration, IConfiguration configuration, string serviceName)
         {
             loggerConfiguration
                 .Enrich.WithMachineName()
                 .Enrich.FromLogContext()
                 .Enrich.WithExceptionDetails()
-                .Enrich.WithProperty("service", serviceName)
+                .Enrich.WithProperty("Application", serviceName)
                 .Enrich.WithEnvironmentName()
-                .Enrich.WithEnvironmentUserName()
                 .Enrich.WithCorrelationId()
                 .Enrich.WithCorrelationIdHeader()
                 .Enrich.WithRequestHeader("User-Agent")

Apps/Common/src/AspNetConfiguration/StartupConfiguration.cs

@@ -255,6 +255,15 @@ public void UseRest(IApplicationBuilder app)
             HttpWeb.UseRest(app, this.Logger);
         }
 
+        /// <summary>
+        /// Configures the app to use middleware to enrich tracing telemetry with additional properties.
+        /// </summary>
+        /// <param name="app">The application builder provider.</param>
+        public void EnrichTracing(IApplicationBuilder app)
+        {
+            Observability.EnrichTracing(app, this.Configuration);
+        }
+
         /// <summary>
         /// Configures the app to use messaging.
         /// </summary>

Apps/Common/src/Common.csproj

@@ -7,6 +7,7 @@
     <PackageReference Include="Asp.Versioning.Mvc.ApiExplorer" Version="8.1.0" />
     <PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="12.0.1" />
     <PackageReference Include="Azure.Messaging.ServiceBus" Version="7.18.1" />
+    <PackageReference Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.3.0" />
     <PackageReference Include="Hangfire.AspNetCore" Version="1.8.14" />
     <PackageReference Include="Hangfire.PostgreSql" Version="1.20.8" />
     <PackageReference Include="Macross.Json.Extensions" Version="3.0.0" />
@@ -27,6 +28,7 @@
     <PackageReference Include="Serilog.Enrichers.Process" Version="3.0.0" />
     <PackageReference Include="Serilog.Enrichers.Span" Version="3.1.0" />
     <PackageReference Include="Serilog.Exceptions" Version="8.4.0" />
+    <PackageReference Include="Serilog.Sinks.ApplicationInsights" Version="4.0.0" />
     <PackageReference Include="Serilog.Sinks.Async" Version="2.0.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
     <PackageReference Include="Serilog.Sinks.Seq" Version="8.0.0" />

Apps/Common/src/Models/OpenTelemetryConfig.cs

@@ -16,11 +16,12 @@
 namespace HealthGateway.Common.Models
 {
     using System;
+    using System.Collections.Generic;
     using System.Diagnostics.CodeAnalysis;
     using OpenTelemetry.Exporter;
 
     /// <summary>
-    /// Provides configuration data for the Notifications Settings PHSA API.
+    /// Provides configuration data for OpenTelemetry.
     /// </summary>
     public class OpenTelemetryConfig
     {
@@ -29,6 +30,11 @@ public class OpenTelemetryConfig
         /// </summary>
         public bool Enabled { get; set; } = true;
 
+        /// <summary>
+        /// Gets or sets the names of activity sources to monitor.
+        /// </summary>
+        public IList<string> Sources { get; set; } = [];
+
         /// <summary>
         /// Gets or sets the name to use for the service.
         /// </summary>
@@ -43,7 +49,12 @@ public class OpenTelemetryConfig
         /// Gets or sets the path prefixes that tracing will ignore.
         /// </summary>
         [SuppressMessage("Performance", "CA1819:Properties should not return arrays", Justification = "Configuration Binding")]
-        public string[] IgnorePathPrefixes { get; set; } = Array.Empty<string>();
+        public string[] IgnorePathPrefixes { get; set; } = [];
+
+        /// <summary>
+        /// Gets or sets the connection string for exporting to Azure Monitor.
+        /// </summary>
+        public string AzureConnectionString { get; set; } = string.Empty;
 
         /// <summary>
         /// Gets or sets OpenTelemetry collector endpoint.
@@ -56,12 +67,12 @@ public class OpenTelemetryConfig
         public OtlpExportProtocol ExportProtocol { get; set; } = OtlpExportProtocol.HttpProtobuf;
 
         /// <summary>
-        /// Gets or sets a value indicating whether  OpenTelemetry console export for tracing, default to false.
+        /// Gets or sets a value indicating whether the console exporter for tracing is enabled, default to false.
         /// </summary>
         public bool TraceConsoleExporterEnabled { get; set; }
 
         /// <summary>
-        /// Gets or sets a value indicating whether OpenTelemetry console export for metrics, default to false.
+        /// Gets or sets a value indicating whether the console exporter for metrics is enabled, default to false.
         /// </summary>
         public bool MetricsConsoleExporterEnabled { get; set; }
     }

Apps/Encounter/src/Startup.cs

@@ -106,6 +106,7 @@ public void Configure(IApplicationBuilder app)
             this.startupConfig.UseSwagger(app);
             this.startupConfig.UseHttp(app);
             this.startupConfig.UseAuth(app);
+            this.startupConfig.EnrichTracing(app);
             this.startupConfig.UseRest(app);
         }
     }

Apps/Encounter/src/appsettings.json

@@ -80,12 +80,16 @@
     "OpenTelemetry": {
         "Enabled": false,
         "Sources": [
+            "RestHospitalVisitDelegate",
+            "RestMspVisitDelegate",
             "EncounterService",
-            "RestMSPVisitDelegate",
-            "PatientService",
-            "ClientRegistriesDelegate",
+
             "DbAuditLogger",
-            "RedisAuditLogger"
+            "RedisAuditLogger",
+            "RestTokenSwapDelegate",
+            "ClientRegistriesDelegate",
+            "AccessTokenService",
+            "PatientService"
         ],
         "ServiceName": "EncounterService",
         "TraceConsoleExporterEnabled": false,

Apps/GatewayApi/src/Startup.cs

@@ -165,6 +165,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
             this.startupConfig.UseSwagger(app);
             this.startupConfig.UseHttp(app, false);
             this.startupConfig.UseAuth(app);
+            this.startupConfig.EnrichTracing(app);
             this.startupConfig.UseRest(app);
 
             DisableTraceMethod(app);

Apps/GatewayApi/src/appsettings.json

@@ -81,10 +81,20 @@
     "OpenTelemetry": {
         "Enabled": false,
         "Sources": [
-            "PatientService",
+            "PatientDetailsService",
+            "WebAlertService",
+
+            "PatientQueryStrategy",
             "ClientRegistriesDelegate",
+
             "DbAuditLogger",
-            "RedisAuditLogger"
+            "RedisAuditLogger",
+            "RestTokenSwapDelegate",
+            "CDogsDelegate",
+            "ClientRegistriesDelegate",
+            "AccessTokenService",
+            "PatientService",
+            "PersonalAccountsService"
         ],
         "ServiceName": "GatewayApi",
         "TraceConsoleExporterEnabled": false,

Apps/Immunization/src/Startup.cs

@@ -103,6 +103,7 @@ public void Configure(IApplicationBuilder app)
             this.startupConfig.UseSwagger(app);
             this.startupConfig.UseHttp(app);
             this.startupConfig.UseAuth(app);
+            this.startupConfig.EnrichTracing(app);
             this.startupConfig.UseRest(app);
         }
     }

Apps/Immunization/src/appsettings.json

@@ -68,10 +68,14 @@
         "Enabled": false,
         "Sources": [
             "RestImmunizationDelegate",
-            "PatientService",
-            "ClientRegistriesDelegate",
+            "RestVaccineStatusDelegate",
+
             "DbAuditLogger",
-            "RedisAuditLogger"
+            "RedisAuditLogger",
+            "RestTokenSwapDelegate",
+            "ClientRegistriesDelegate",
+            "AccessTokenService",
+            "PatientService"
         ],
         "ServiceName": "ImmunizationService",
         "TraceConsoleExporterEnabled": false,

Apps/JobScheduler/src/Startup.cs

@@ -163,6 +163,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
             this.startupConfig.UseHttp(app);
             this.startupConfig.UseContentSecurityPolicy(app);
             this.startupConfig.UseAuth(app);
+            this.startupConfig.EnrichTracing(app);
 
             app.UseEndpoints(
                 endpoints =>

Apps/JobScheduler/src/appsettings.json

@@ -59,12 +59,7 @@
     },
     "OpenTelemetry": {
         "Enabled": false,
-        "Sources": [
-            "PatientService",
-            "ClientRegistriesDelegate",
-            "DbAuditLogger",
-            "RedisAuditLogger"
-        ],
+        "Sources": ["RestNotificationSettingsDelegate"],
         "ServiceName": "JobScheduler",
         "TraceConsoleExporterEnabled": false,
         "ZipkinEnabled": false,

Apps/Laboratory/src/Startup.cs

@@ -104,6 +104,7 @@ public void Configure(IApplicationBuilder app)
             this.startupConfig.UseSwagger(app);
             this.startupConfig.UseHttp(app);
             this.startupConfig.UseAuth(app);
+            this.startupConfig.EnrichTracing(app);
             this.startupConfig.UseRest(app);
         }
     }