Feature/prod 2955/add traces to hz cache

HzMemoryCache/Diagnostics/Activities.cs

@@ -0,0 +1,75 @@
+using System.Collections.Generic;
+using System.Diagnostics;
+
+namespace HzCache.Diagnostics
+{
+    public static class Activities
+    {
+        public static ActivitySource? Source => new(HzCacheDiagnostics.ActivitySourceName, HzCacheDiagnostics.HzCacheVersion);
+
+        public static class Names
+        {
+            public const string Set = "set from cache";
+            public const string SetRedis = "set to redis";
+            public const string Get = "get from cache";
+            public const string GetRedis = "get from redis";
+            public const string GetOrSet = "get or set from cache";
+            public const string GetOrSetBatch = "get or set batch from cache";
+            public const string GetBatchRedis = "get batch from redis";
+            public const string Remove = "remove";
+            public const string RemoveByPattern = "remove by pattern";
+            public const string RemoveRedis = "remove from redis";
+            public const string RemoveByPatternRedis = "remove by pattern from redis";
+            public const string RemoveItem = "remove item";
+            public const string Clear = "clear";
+            public const string ExecuteFactory = "execute factory";
+
+            public const string NotifyItemChange = "notify item change";
+            public const string Subscribe = "subscribe";
+            public const string ValueChanged = "value changed";
+            public const string EvictExpired = "evict expired";
+            public const string GetStatistics = "get statistics";
+            public const string ProcessExpiredEviction = "process expired eviction";
+
+            public const string AcquireLock = "acquire lock";
+            public const string GetSemaphore = "get semaphore";
+            public const string ReleaseLock = "release lock";
+
+
+        }
+
+        public static class Area
+        {
+            public const string HzMemoryCache = "HzMemoryCache";
+            public const string HzCacheMemoryLocker = "HzCacheMemoryLocker"; 
+            public const string RedisBackedHzCache = "RedisBackedHzCache";
+            public const string Redis = "RedisCache";
+        }
+
+        private static IEnumerable<KeyValuePair<string, object>> GetCommonTags(string? key, string project, bool async, string? pattern, bool? sendNotification)
+        {
+            var res = new List<KeyValuePair<string, object?>>
+            {
+                new KeyValuePair<string, object?>(Tags.Names.OperationKey, key),
+                new KeyValuePair<string, object?>(Tags.Names.Project, project),
+                new KeyValuePair<string, object?>(Tags.Names.Async, async),
+                new KeyValuePair<string, object?>(Tags.Names.Pattern, pattern),
+                new KeyValuePair<string, object?>(Tags.Names.SendNotification, sendNotification),
+            };
+
+            return res;
+        }
+
+        public static Activity? StartActivityWithCommonTags(this ActivitySource source, string activityName, string project, bool async = false, string? key = null, string? pattern = null, bool? sendNotification = null)
+        {
+            if (source.HasListeners() == false || !HzCacheTracesInstrumentationOptions.Instance.IsActive(activityName, project, key))
+                return null;
+
+            return source.StartActivity(
+                ActivityKind.Internal,
+                tags: GetCommonTags(key, project, async, pattern, sendNotification),
+                name: activityName
+            );
+        }
+    }
+}

HzMemoryCache/Diagnostics/HzCacheDiagnostics.cs

@@ -0,0 +1,8 @@
+namespace HzCache.Diagnostics
+{
+    public static class HzCacheDiagnostics
+    {
+        public const string HzCacheVersion = "0.0.9";
+        public const string ActivitySourceName = "HzMemoryCache";
+    }
+}

HzMemoryCache/Diagnostics/HzCacheTracesInstrumentationOptions.cs

@@ -0,0 +1,13 @@
+using System;
+using static HzCache.Diagnostics.Activities;
+
+namespace HzCache.Diagnostics
+{
+    public class HzCacheTracesInstrumentationOptions
+    {
+        public static HzCacheTracesInstrumentationOptions Instance { get; } = new();
+        public Func<string, string, string, bool> Active { private get; set; }
+
+        public bool IsActive(string activityName,string project, string? key) => Active(activityName, project, key);
+    }
+}

HzMemoryCache/Diagnostics/Tags.cs

@@ -0,0 +1,16 @@
+namespace HzCache.Diagnostics
+{
+    internal static class Tags
+    {
+        internal static class Names
+        {
+            public const string OperationKey = "hzcache.operation.key";
+            public const string Pattern = "hzcache.pattern";
+            public const string SendNotification = "hzcache.sendNotification";
+            public const string Project = "hzcache.project";
+            public const string Async = "hzcache.operation.async";
+            public const string AcquiredLock = "hzcache.result.lock.acquired";
+            public const string ReleasedLock = "hzcache.result.lock.released";
+        }
+    }
+}

HzMemoryCache/Diagnostics/TracerProviderBuilderExtensions.cs

@@ -0,0 +1,22 @@
+using OpenTelemetry.Trace;
+using System;
+
+namespace HzCache.Diagnostics
+{
+    public static class TracerProviderBuilderBuilderExtensions
+    {
+        public static TracerProviderBuilder AddHzCacheMemoryInstrumentation(this TracerProviderBuilder builder,
+            Action<HzCacheTracesInstrumentationOptions>? configure = null)
+        {
+            if (builder is null)
+                throw new ArgumentNullException(nameof(builder));
+
+            var options = HzCacheTracesInstrumentationOptions.Instance;
+            configure?.Invoke(options);
+
+            builder.AddSource(HzCacheDiagnostics.ActivitySourceName);
+
+            return builder;
+        }
+    }
+}

HzMemoryCache/HzCacheMemoryLocker.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
+using HzCache.Diagnostics;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
 
@@ -35,6 +36,8 @@ private uint GetLockIndex(string key)
 
         private SemaphoreSlim GetSemaphore(string cacheName, string cacheInstanceId, string key, ILogger? logger)
         {
+            using var activity = Activities.Source.StartActivityWithCommonTags(Activities.Names.GetSemaphore, Activities.Area.HzCacheMemoryLocker, key: key);
+
             object? _semaphore;
 
             if (lockCache.TryGetValue(key, out _semaphore))
@@ -79,6 +82,7 @@ private SemaphoreSlim GetSemaphore(string cacheName, string cacheInstanceId, str
         public async ValueTask<object> AcquireLockAsync(string cacheName, string cacheInstanceId, string operationId, string key, TimeSpan timeout, ILogger? logger,
             CancellationToken token)
         {
+            using var activity = Activities.Source.StartActivityWithCommonTags(Activities.Names.GetSemaphore, Activities.Area.HzCacheMemoryLocker, key: key);
             var semaphore = GetSemaphore(cacheName, cacheInstanceId, key, logger);
 
             if (logger?.IsEnabled(LogLevel.Trace) ?? false)
@@ -107,13 +111,16 @@ public async ValueTask<object> AcquireLockAsync(string cacheName, string cacheIn
                         operationId, key);
                 }
             }
+            activity?.AddTag(Tags.Names.AcquiredLock, acquired);
 
             return acquired ? semaphore : null;
         }
 
         /// <inheritdoc />
         public object? AcquireLock(string cacheName, string cacheInstanceId, string operationId, string key, TimeSpan timeout, ILogger? logger, CancellationToken token)
         {
+            using var activity = Activities.Source.StartActivityWithCommonTags(Activities.Names.AcquireLock, Activities.Area.HzCacheMemoryLocker, key: key);
+
             var semaphore = GetSemaphore(cacheName, cacheInstanceId, key, logger);
 
             if (logger?.IsEnabled(LogLevel.Trace) ?? false)
@@ -142,13 +149,15 @@ public async ValueTask<object> AcquireLockAsync(string cacheName, string cacheIn
                         operationId, key);
                 }
             }
-
+            activity?.AddTag(Tags.Names.AcquiredLock, acquired);
             return acquired ? semaphore : null;
         }
 
         /// <inheritdoc />
         public void ReleaseLock(string cacheName, string cacheInstanceId, string operationId, string key, object? lockObj, ILogger? logger)
         {
+            using var activity = Activities.Source.StartActivityWithCommonTags(Activities.Names.ReleaseLock, Activities.Area.HzCacheMemoryLocker, key: key);
+
             if (lockObj is null)
             {
                 return;
@@ -157,6 +166,7 @@ public void ReleaseLock(string cacheName, string cacheInstanceId, string operati
             try
             {
                 ((SemaphoreSlim)lockObj).Release();
+                activity?.AddTag(Tags.Names.ReleasedLock, true);
             }
             catch (Exception exc)
             {
@@ -166,6 +176,7 @@ public void ReleaseLock(string cacheName, string cacheInstanceId, string operati
                         "FUSION [N={CacheName} I={CacheInstanceId}] (O={CacheOperationId} K={CacheKey}): an error occurred while trying to release a SemaphoreSlim in the memory locker",
                         cacheName, cacheInstanceId, operationId, key);
                 }
+                activity?.AddTag(Tags.Names.ReleasedLock, false);
             }
         }
     }

HzMemoryCache/HzMemoryCache.cs

@@ -5,11 +5,13 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Reactive.Concurrency;
+using System.Reactive.Joins;
 using System.Reactive.Linq;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
+using HzCache.Diagnostics;
 using Microsoft.Extensions.Logging;
 
 namespace HzCache
@@ -47,6 +49,8 @@ public HzMemoryCache(HzCacheOptions? options = null)
 
         public void RemoveByPattern(string pattern, bool sendNotification = true)
         {
+            using var activity = Activities.Source.StartActivityWithCommonTags(Activities.Names.RemoveByPattern, Activities.Area.HzMemoryCache, pattern: pattern, sendNotification: sendNotification);
+
             var myPattern = pattern;
             if (pattern[0] != '*')
             {
@@ -67,6 +71,8 @@ public void RemoveByPattern(string pattern, bool sendNotification = true)
 
         public void EvictExpired()
         {
+            using var activity = Activities.Source.StartActivityWithCommonTags(Activities.Names.EvictExpired, Activities.Area.HzMemoryCache);
+
             if (Monitor.TryEnter(cleanUpTimer)) //use the timer-object for our lock, it's local, private and instance-type, so its ok
             {
                 try
@@ -98,6 +104,8 @@ public void Clear()
         /// </summary>
         public T? Get<T>(string key)
         {
+            using var activity = Activities.Source.StartActivityWithCommonTags(Activities.Names.Get, Activities.Area.HzMemoryCache);
+
             var defaultValue = default(T);
 
             if (!dictionary.TryGetValue(key, out var ttlValue))
@@ -136,6 +144,8 @@ public void Set<T>(string key, T? value)
         /// </summary>
         public void Set<T>(string key, T? value, TimeSpan ttl)
         {
+            using var activity = Activities.Source.StartActivityWithCommonTags(Activities.Names.Set, Activities.Area.HzMemoryCache);
+
             var v = new TTLValue(key, value, ttl, updateChecksumAndSerializeQueue, options.notificationType,
                 (tv, objectData) => NotifyItemChange(key, CacheItemChangeType.AddOrUpdate, tv, objectData));
             dictionary[key] = v;
@@ -146,6 +156,8 @@ public void Set<T>(string key, T? value, TimeSpan ttl)
         /// </summary>
         public T? GetOrSet<T>(string key, Func<string, T> valueFactory, TimeSpan ttl, long maxMsToWaitForFactory = 10000)
         {
+            using var activity = Activities.Source.StartActivityWithCommonTags(Activities.Names.GetOrSet, Activities.Area.HzMemoryCache, key: key);
+
             var value = Get<T>(key);
             if (!IsNullOrDefault(value))
             {
@@ -162,14 +174,22 @@ public void Set<T>(string key, T? value, TimeSpan ttl)
                 throw new Exception($"Could not acquire lock for key {key}");
             }
 
+
             try
             {
                 value = Get<T>(key);
                 if (!IsNullOrDefault(value))
                 {
                     return value;
                 }
-                value = valueFactory(key);
+                using (var executeActivity =
+                       Activities.Source.StartActivityWithCommonTags(Activities.Names.ExecuteFactory,
+                           Activities.Area.HzMemoryCache, key: key))
+                {
+
+                    value = valueFactory(key);
+                }
+
                 var ttlValue = new TTLValue(key, value, ttl, updateChecksumAndSerializeQueue, options.notificationType, (tv, objectData) =>
                 {
                     NotifyItemChange(key, CacheItemChangeType.AddOrUpdate, tv, objectData);
@@ -191,6 +211,9 @@ public IList<T> GetOrSetBatch<T>(IList<string> keys, Func<IList<string>, List<Ke
 
         public IList<T> GetOrSetBatch<T>(IList<string> keys, Func<IList<string>, List<KeyValuePair<string, T>>> valueFactory, TimeSpan ttl)
         {
+            using var activity = Activities.Source.StartActivityWithCommonTags(Activities.Names.GetOrSetBatch, Activities.Area.HzMemoryCache, key: string.Join(",",keys??new List<string>()));
+
+
             var cachedItems = keys.Select(key => new KeyValuePair<string, T?>(key, Get<T>(key)));
             var missingKeys = cachedItems.Where(kvp => IsNullOrDefault(kvp.Value)).Select(kvp => kvp.Key).ToList();
             var factoryRetrievedItems = valueFactory(missingKeys).ToDictionary(kv => kv.Key, kv => kv.Value);
@@ -306,6 +329,7 @@ private void StartUpdateChecksumAndNotify()
 
         private async Task ProcessExpiredEviction()
         {
+            using var activity = Activities.Source.StartActivityWithCommonTags(Activities.Names.ProcessExpiredEviction, Activities.Area.HzMemoryCache);
             await globalStaticLock.WaitAsync().ConfigureAwait(false);
             try
             {
@@ -316,6 +340,8 @@ private async Task ProcessExpiredEviction()
 
         private bool RemoveItem(string key, CacheItemChangeType changeType, bool sendNotification, Func<string, bool>? areEqualFunc = null)
         {
+            using var activity = Activities.Source.StartActivityWithCommonTags(Activities.Names.RemoveItem, Activities.Area.HzMemoryCache, key: key);
+
             var result = !(!dictionary.TryGetValue(key, out TTLValue ttlValue) || (areEqualFunc != null && areEqualFunc.Invoke(ttlValue.checksum)));
 
             if (result)
@@ -337,6 +363,8 @@ private bool RemoveItem(string key, CacheItemChangeType changeType, bool sendNot
 
         private void NotifyItemChange(string key, CacheItemChangeType changeType, TTLValue ttlValue, byte[]? objectData = null, bool isPattern = false)
         {
+            using var activity = Activities.Source.StartActivityWithCommonTags(Activities.Names.NotifyItemChange, Activities.Area.HzMemoryCache, key: key);
+
             options.valueChangeListener(key, changeType, ttlValue, objectData, isPattern);
         }
 

HzMemoryCache/HzMemoryCache.csproj

@@ -34,6 +34,7 @@
     <PackageReference Include="K4os.Compression.LZ4" Version="1.3.8" />
     <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="6.0.1" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0" />
+    <PackageReference Include="OpenTelemetry.Api" Version="1.10.0" />
     <PackageReference Include="System.Reactive" Version="6.0.0" />
     <PackageReference Include="System.Threading.Tasks.Dataflow" Version="8.0.0" />
     <PackageReference Include="Utf8Json" Version="1.3.7" />