feat: add apm clickhouse query api

.github/workflows/Codecov.yml

@@ -94,9 +94,9 @@ jobs:
       uses: actions/setup-dotnet@v1
       with:
         dotnet-version: |
-          6.0.x
-          7.0.x
           8.0.x
+          7.0.x
+          6.0.x
 
         include-prerelease: true
 

.github/workflows/ci_test_reports_to_oss.yml

@@ -27,9 +27,9 @@ jobs:
       uses: actions/setup-dotnet@v1
       with:
         dotnet-version: |
-          6.0.x
-          7.0.x
           8.0.x
+          7.0.x
+          6.0.x
 
         include-prerelease: true
     # - name: dependencies

.github/workflows/package_push_nuget.org.yml

@@ -19,9 +19,9 @@ jobs:
       uses: actions/setup-dotnet@v1
       with:
         dotnet-version: |
-          6.0.x
-          7.0.x
           8.0.x
+          7.0.x
+          6.0.x
         include-prerelease: true
 
     - name: restore

.github/workflows/pr_run_test_ci.yml

@@ -38,9 +38,10 @@ jobs:
       uses: actions/setup-dotnet@v1
       with:
         dotnet-version: |
-          6.0.x
-          7.0.x
           8.0.x
+          7.0.x
+          6.0.x
+
         include-prerelease: true
 
 #     - name: Start Redis
@@ -57,7 +58,7 @@ jobs:
     - name: Set up JDK 11
       uses: actions/setup-java@v1
       with:
-        java-version: 1.11
+        java-version: 1.17
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis

Masa.Framework.sln

@@ -715,7 +715,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Utils.DynamicsCrm.Enti
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Contrib.StackSdks.Tsc.Clickhouse", "src\Contrib\StackSdks\Masa.Contrib.StackSdks.Tsc.Clickhouse\Masa.Contrib.StackSdks.Tsc.Clickhouse.csproj", "{43389D12-17E1-4F07-9A42-5CFCC24D08B2}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests", "src\Contrib\StackSdks\Tests\Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests\Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests.csproj", "{289BF8C8-968F-4105-A65E-C1C6FD8857F2}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests", "src\Contrib\StackSdks\Tests\Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests\Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests.csproj", "{289BF8C8-968F-4105-A65E-C1C6FD8857F2}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse", "src\Contrib\StackSdks\Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse\Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj", "{9B4F7AC1-97FD-400A-9810-7F7672FBAF65}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests", "src\Contrib\StackSdks\Tests\Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests\Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj", "{7E11AD82-5E5C-43E9-B7C3-77A127B628E8}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -2589,6 +2593,22 @@ Global
 		{289BF8C8-968F-4105-A65E-C1C6FD8857F2}.Release|Any CPU.Build.0 = Release|Any CPU
 		{289BF8C8-968F-4105-A65E-C1C6FD8857F2}.Release|x64.ActiveCfg = Release|Any CPU
 		{289BF8C8-968F-4105-A65E-C1C6FD8857F2}.Release|x64.Build.0 = Release|Any CPU
+		{9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Debug|x64.Build.0 = Debug|Any CPU
+		{9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Release|x64.ActiveCfg = Release|Any CPU
+		{9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Release|x64.Build.0 = Release|Any CPU
+		{7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Debug|x64.Build.0 = Debug|Any CPU
+		{7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Release|x64.ActiveCfg = Release|Any CPU
+		{7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -2943,6 +2963,8 @@ Global
 		{8A51A2A9-FBF4-40DC-AD89-AD3B9D3A50DC} = {64B54122-44F1-4379-9422-953EF706A3A6}
 		{43389D12-17E1-4F07-9A42-5CFCC24D08B2} = {6042AE23-A07E-4F6F-B1C3-F17617AEB722}
 		{289BF8C8-968F-4105-A65E-C1C6FD8857F2} = {E4AD67C8-9255-4013-A3C4-962694399770}
+		{9B4F7AC1-97FD-400A-9810-7F7672FBAF65} = {6042AE23-A07E-4F6F-B1C3-F17617AEB722}
+		{7E11AD82-5E5C-43E9-B7C3-77A127B628E8} = {E4AD67C8-9255-4013-A3C4-962694399770}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {40383055-CC50-4600-AD9A-53C14F620D03}

src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Extensions/DictionaryExtenistions.cs

@@ -7,7 +7,8 @@ public static class DictionaryExtenistions
 {
     private static readonly JsonSerializerOptions _serializerOptions = new()
     {
-        PropertyNamingPolicy = JsonNamingPolicy.CamelCase
+        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+        NumberHandling = JsonNumberHandling.AllowReadingFromString
     };
 
     public static Dictionary<string, T> GroupByKeyPrefix<T>(this Dictionary<string, object> source, string prefix, Func<object, T>? convertFunc = null)

src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceDatabaseResponseDto.cs

@@ -21,6 +21,7 @@ public class TraceDatabaseResponseDto
     public virtual string PeerName { get; set; }
 
     [JsonPropertyName("net.peer.port")]
+    [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]
     public virtual int PeerPort { get; set; }
 
     [JsonPropertyName("net.transport")]
@@ -42,6 +43,7 @@ public class TraceDatabaseResponseDto
     public virtual string Operation { get; set; }
 
     [JsonPropertyName("db.redis.database_index")]
+    [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]
     public virtual int RedisDatabaseIndex { get; set; }
 
     [JsonPropertyName("db.mongodb.collection")]
@@ -53,6 +55,7 @@ public class TraceDatabaseResponseDto
     #region Cassandra
 
     [JsonPropertyName("db.cassandra.page_size")]
+    [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]
     public virtual int CassandraPageSize { get; set; }
 
     [JsonPropertyName("db.cassandra.consistency_level")]

src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceHttpResponseDto.cs

@@ -25,6 +25,7 @@ public class TraceHttpResponseDto
     public virtual string Scheme { get; set; }
 
     [JsonPropertyName("http.status_code")]
+    [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]
     public virtual int StatusCode { get; set; }
 
     [JsonPropertyName("http.flavor")]
@@ -34,28 +35,34 @@ public class TraceHttpResponseDto
     public virtual string UserAgent { get; set; }
 
     [JsonPropertyName("http.request_content_length")]
+    [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]
     public virtual int RequestContentLength { get; set; }
 
     [JsonPropertyName("http.request_content_length_uncompressed")]
+    [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]
     public virtual int RequestContentLengthUncompressed { get; set; }
 
     public Dictionary<string, IEnumerable<string>> RequestHeaders { get; set; }
 
     [JsonPropertyName("http.response_content_length")]
+    [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]
     public virtual int ResponseContentLength { get; set; }
 
     [JsonPropertyName("http.response_content_length_uncompressed")]
+    [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]
     public virtual int ResponseContentLengthUncompressed { get; set; }    
 
     public Dictionary<string, IEnumerable<string>> ReponseHeaders { get; set; }
 
     [JsonPropertyName("http.retry_count")]
+    [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]
     public virtual int RetryCount { get; set; }
 
     [JsonPropertyName("net.peer.ip")]
     public virtual string PeerIp { get; set; }
 
     [JsonPropertyName("net.peer.port")]
+    [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]
     public virtual int? PeerPort { get; set; }
 
     [JsonPropertyName("net.peer.name")]

src/Contrib/Data/Contracts/Masa.Contrib.Data.Contracts/MasaDbContextBuilderExtensions.cs

@@ -3,7 +3,7 @@
 
 // ReSharper disable once CheckNamespace
 
-namespace Masa.BuildingBlocks.Data;
+namespace Microsoft.EntityFrameworkCore;
 
 public static class MasaDbContextBuilderExtensions
 {

src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs

@@ -0,0 +1,77 @@
+// Copyright (c) MASA Stack All rights reserved.
+// Licensed under the MIT License. See LICENSE.txt in the project root for license information.
+
+namespace Microsoft.Extensions.DependencyInjection;
+
+public static class ApmClickhouseServiceExtensions
+{
+    internal static ILogger Logger { get; private set; }
+
+    public static IServiceCollection AddMASAStackApmClickhouse(this IServiceCollection services, string connectionStr, string logTable, string traceTable, string? logSourceTable = null, string? traceSourceTable = null, Action<IDbConnection>? configer = null)
+    {
+        services.AddMASAStackClickhouse(connectionStr, logTable, traceTable, logSourceTable, traceSourceTable, con =>
+         {
+             Constants.Init(MasaStackClickhouseConnection.LogTable.Split('.')[0], MasaStackClickhouseConnection.LogTable.Split('.')[1], MasaStackClickhouseConnection.TraceTable.Split('.')[1], "otel_errors");
+             Init(services, con);
+             configer?.Invoke(con);
+         });
+        services.AddScoped<IApmService, ClickhouseApmService>();
+        return services;
+    }
+
+    private static void Init(IServiceCollection services, IDbConnection connection)
+    {
+        var serviceProvider = services.BuildServiceProvider();
+        var logfactory = serviceProvider.GetRequiredService<ILoggerFactory>();
+        Logger = logfactory.CreateLogger("Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse");
+        InitTable(connection);
+    }
+
+    private static void InitTable(IDbConnection connection)
+    {
+        if (Convert.ToInt32(connection.ExecuteScalar($"select count() from system.tables where database ='{Constants.Database}' and name in ['{Constants.ErrorTable}','{Constants.ErrorTable}_v']")) > 0)
+            return;
+        var createTableSqls = new string[]{
+            @$"CREATE TABLE {Constants.Database}.{Constants.ErrorTable}
+(
+    `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+    `TraceId` String CODEC(ZSTD(1)),
+    `SpanId` String CODEC(ZSTD(1)),
+    `Attributes.exception.message` String CODEC(ZSTD(1)),
+    `Attributes.exception.type` String CODEC(ZSTD(1)),
+    `ServiceName` String CODEC(ZSTD(1)),
+    `Resource.service.namespace` String CODEC(ZSTD(1)),
+    `Attributes.http.target` String CODEC(ZSTD(1)),
+    INDEX idx_log_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_log_spanid SpanId TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_log_environment `Resource.service.namespace` TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_log_servicename ServiceName TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_log_type `Attributes.exception.type` TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_log_endpoint `Attributes.http.target` TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_string_message `Attributes.exception.message` TYPE tokenbf_v1(30720, 2, 0) GRANULARITY 1
+)
+ENGINE = MergeTree
+PARTITION BY toDate(Timestamp)
+ORDER BY (Timestamp,
+ ServiceName,
+ `Resource.service.namespace`,
+ `Attributes.exception.type`,
+`Attributes.http.target`)
+TTL toDateTime(Timestamp) + toIntervalDay(30)
+SETTINGS index_granularity = 8192,
+ ttl_only_drop_parts = 1;
+",
+$@"CREATE MATERIALIZED VIEW {Constants.Database}.{Constants.ErrorTable}_v TO {Constants.ErrorTableFull}
+AS
+SELECT
+Timestamp,TraceId,SpanId, Body AS `Attributes.exception.message`,LogAttributes['exception.type'] AS `Attributes.exception.type`,
+    ServiceName,ResourceAttributes['service.namespace'] AS `Resource.service.namespace`, LogAttributes['RequestPath'] AS `Attributes.http.target`
+FROM {MasaStackClickhouseConnection.LogSourceTable}
+WHERE mapContains(LogAttributes, 'exception.type')
+"};
+        foreach (var sql in createTableSqls)
+        {
+            connection.ExecuteSql(sql);
+        }
+    }
+}