From b974dc067d6d08553076567cbc858b51f399feea Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Thu, 4 Jan 2024 14:46:32 +0800 Subject: [PATCH 01/24] feat: add apm clickhouse query api --- Masa.Framework.sln | 24 +- .../ApmClickhouseServiceExtensions.cs | 78 +++ .../Cliclhouse/ClickhouseApmService.cs | 468 ++++++++++++++++++ .../Config/Constants.cs | 31 ++ .../Interfaces/IAPMService.cs | 49 ++ ...ontrib.StackSdks.Tsc.Apm.Clickhouse.csproj | 15 + .../Models/APMEnums.cs | 10 + .../Models/Request/ApmEndpointRequestDto.cs | 9 + .../Request/ApmTraceLatencyRequestDto.cs | 13 + .../Models/Request/BaseApmRequestDto.cs | 31 ++ .../Models/Response/EndpointChartDto.cs | 28 ++ .../EndpointLatencyDistributionDto.cs | 11 + .../Models/Response/EndpointListDto.cs | 21 + .../Models/Response/ServiceListDto.cs | 50 ++ .../Models/Response/TraceDetailDto.cs | 9 + .../_Imports.cs | 19 + .../Extensions/IDbConnectionExtensitions.cs | 16 +- ...ns.cs => MasaTscCliclhouseExtensitions.cs} | 11 +- .../Model/MASAStackClickhouseConnection.cs | 4 +- .../_Imports.cs | 1 + .../Options/OpenTelemetryAttributeName.cs | 2 + .../ClickhouseApmServiceTests.cs | 130 +++++ .../Data/otel_trace_data.txt | 50 ++ .../GlobalUsings.cs | 8 + ....StackSdks.Tsc.Apm.Clickhouse.Tests.csproj | 33 ++ .../TestUtils.cs | 10 + .../Consts.cs | 4 +- 27 files changed, 1119 insertions(+), 16 deletions(-) create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Config/Constants.cs create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IAPMService.cs create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/APMEnums.cs create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmEndpointRequestDto.cs create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmTraceLatencyRequestDto.cs create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointLatencyDistributionDto.cs create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointListDto.cs create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/ServiceListDto.cs create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/TraceDetailDto.cs create mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs rename src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/{ServiceExtensitions.cs => MasaTscCliclhouseExtensitions.cs} (96%) create mode 100644 src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs create mode 100644 src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Data/otel_trace_data.txt create mode 100644 src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/GlobalUsings.cs create mode 100644 src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj create mode 100644 src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs diff --git a/Masa.Framework.sln b/Masa.Framework.sln index e5b607c0d..bae963363 100644 --- a/Masa.Framework.sln +++ b/Masa.Framework.sln @@ -715,7 +715,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Utils.DynamicsCrm.Enti EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Contrib.StackSdks.Tsc.Clickhouse", "src\Contrib\StackSdks\Masa.Contrib.StackSdks.Tsc.Clickhouse\Masa.Contrib.StackSdks.Tsc.Clickhouse.csproj", "{43389D12-17E1-4F07-9A42-5CFCC24D08B2}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests", "src\Contrib\StackSdks\Tests\Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests\Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests.csproj", "{289BF8C8-968F-4105-A65E-C1C6FD8857F2}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests", "src\Contrib\StackSdks\Tests\Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests\Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests.csproj", "{289BF8C8-968F-4105-A65E-C1C6FD8857F2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse", "src\Contrib\StackSdks\Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse\Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj", "{9B4F7AC1-97FD-400A-9810-7F7672FBAF65}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests", "src\Contrib\StackSdks\Tests\Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests\Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj", "{7E11AD82-5E5C-43E9-B7C3-77A127B628E8}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -2589,6 +2593,22 @@ Global {289BF8C8-968F-4105-A65E-C1C6FD8857F2}.Release|Any CPU.Build.0 = Release|Any CPU {289BF8C8-968F-4105-A65E-C1C6FD8857F2}.Release|x64.ActiveCfg = Release|Any CPU {289BF8C8-968F-4105-A65E-C1C6FD8857F2}.Release|x64.Build.0 = Release|Any CPU + {9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Debug|x64.ActiveCfg = Debug|Any CPU + {9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Debug|x64.Build.0 = Debug|Any CPU + {9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Release|Any CPU.Build.0 = Release|Any CPU + {9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Release|x64.ActiveCfg = Release|Any CPU + {9B4F7AC1-97FD-400A-9810-7F7672FBAF65}.Release|x64.Build.0 = Release|Any CPU + {7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Debug|x64.ActiveCfg = Debug|Any CPU + {7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Debug|x64.Build.0 = Debug|Any CPU + {7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Release|Any CPU.Build.0 = Release|Any CPU + {7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Release|x64.ActiveCfg = Release|Any CPU + {7E11AD82-5E5C-43E9-B7C3-77A127B628E8}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -2943,6 +2963,8 @@ Global {8A51A2A9-FBF4-40DC-AD89-AD3B9D3A50DC} = {64B54122-44F1-4379-9422-953EF706A3A6} {43389D12-17E1-4F07-9A42-5CFCC24D08B2} = {6042AE23-A07E-4F6F-B1C3-F17617AEB722} {289BF8C8-968F-4105-A65E-C1C6FD8857F2} = {E4AD67C8-9255-4013-A3C4-962694399770} + {9B4F7AC1-97FD-400A-9810-7F7672FBAF65} = {6042AE23-A07E-4F6F-B1C3-F17617AEB722} + {7E11AD82-5E5C-43E9-B7C3-77A127B628E8} = {E4AD67C8-9255-4013-A3C4-962694399770} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {40383055-CC50-4600-AD9A-53C14F620D03} diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs new file mode 100644 index 000000000..6ac65eea3 --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs @@ -0,0 +1,78 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Microsoft.Extensions.DependencyInjection; + +public static class ApmClickhouseServiceExtensions +{ + internal static ILogger Logger { get; private set; } + + public static IServiceCollection AddMASAStackApmClickhouse(this IServiceCollection services, string connectionStr, string logTable, string traceTable, string? logSourceTable = null, string? traceSourceTable = null, Action? configer = null) + { + return services.AddMASAStackClickhouse(connectionStr, logTable, traceTable, logSourceTable, traceSourceTable, con => + { + var connection = (MasaStackClickhouseConnection)con; + Constants.Init(MasaStackClickhouseConnection.LogTable.Split('.')[0], MasaStackClickhouseConnection.LogTable.Split('.')[1], MasaStackClickhouseConnection.TraceTable.Split('.')[1], "otel_errors"); + services.TryAddScoped(builder => new ClickhouseApmService(con, services.BuildServiceProvider().GetRequiredService())); + Init(services, con); + configer?.Invoke(con); + + }); + } + + private static void Init(IServiceCollection services, IDbConnection connection) + { + var serviceProvider = services.BuildServiceProvider(); + var logfactory = serviceProvider.GetRequiredService(); + Logger = logfactory.CreateLogger("Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse"); + InitTable(connection); + } + + private static void InitTable(IDbConnection connection) + { + if (Convert.ToInt32(connection.ExecuteScalar($"select count() from system.tables where database ='{Constants.Database}' and name in ['{Constants.ErrorTable}','{Constants.ErrorTable}_v']")) > 0) + return; + var createTableSqls = new string[]{ + @$"CREATE TABLE {Constants.Database}.{Constants.ErrorTable} +( + `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `TraceId` String CODEC(ZSTD(1)), + `SpanId` String CODEC(ZSTD(1)), + `Attributes.exception.message` String CODEC(ZSTD(1)), + `Attributes.exception.type` String CODEC(ZSTD(1)), + `ServiceName` String CODEC(ZSTD(1)), + `Resource.service.namespace` String CODEC(ZSTD(1)), + `Attributes.http.target` String CODEC(ZSTD(1)), + INDEX idx_log_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_spanid SpanId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_environment `Resource.service.namespace` TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_servicename ServiceName TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_type `Attributes.exception.type` TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_endpoint `Attributes.http.target` TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_string_message `Attributes.exception.message` TYPE tokenbf_v1(30720, 2, 0) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (Timestamp, + ServiceName, + `Resource.service.namespace`, + `Attributes.exception.type`, +`Attributes.http.target`) +TTL toDateTime(Timestamp) + toIntervalDay(30) +SETTINGS index_granularity = 8192, + ttl_only_drop_parts = 1; +", +$@"CREATE MATERIALIZED VIEW {Constants.Database}.{Constants.ErrorTable}_v TO {Constants.ErrorTableFull} +AS +SELECT +Timestamp,TraceId,SpanId, Body AS `Attributes.exception.message`,LogAttributes['exception.type'] AS `Attributes.exception.type`, + ServiceName,ResourceAttributes['service.namespace'] AS `Resource.service.namespace`, LogAttributes['RequestPath'] AS `Attributes.http.target` +FROM {MasaStackClickhouseConnection.LogSourceTable} +WHERE mapContains(LogAttributes, 'exception.type') +"}; + foreach (var sql in createTableSqls) + { + connection.ExecuteSql(sql); + } + } +} diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs new file mode 100644 index 000000000..0d9dc9ec9 --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -0,0 +1,468 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model; + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Cliclhouse; + +internal class ClickhouseApmService : IAPMService, IDisposable +{ + private readonly IDbConnection _dbConnection; + private readonly IDbCommand _command; + private readonly ITraceService _traceService; + + public ClickhouseApmService(IDbConnection dbConnection, ITraceService traceService) + { + _traceService = traceService; + _dbConnection = dbConnection; + if (_dbConnection.State == ConnectionState.Closed) + _dbConnection.Open(); + _command = dbConnection.CreateCommand(); + } + + public async Task> ServicePageAsync(BaseApmRequestDto query) + { + var parameters = new List(); + var where = AppendWhere(query, parameters); + var result = new PaginatedListBase(); + var groupby = "group by ServiceName"; + var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; + result.Total = Convert.ToInt64(Scalar(countSql, parameters)); + var orderBy = GetOrderBy(query, "ServiceName"); + var start = (query.Page - 1) * query.PageSize; + if (result.Total - start > 0) + { + var sql = $@"select * from( +select +ServiceName, +arrayStringConcat(groupUniqArray(`Resource.service.namespace`)) env, +floor(AVG(Duration)) latency, +round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) throughput, +round(sum(has(['{string.Join("','", query.ErrorStatusCodes)}'],`Attributes.http.status_code`))*100.0/count(1),2) failed +from {Constants.TraceTableFull} where {where} {groupby} {orderBy} limit {start},{query.PageSize})"; + using var reader = Query(sql, parameters); + result.Result = new(); + while (reader.NextResult()) + while (reader.Read()) + { + var item = new ServiceListDto() + { + Name = reader[0].ToString()!, + Envs = reader[1]?.ToString()?.Split(',') ?? Array.Empty(), + Latency = (long)Math.Floor(Convert.ToDouble(reader[2])), + Throughput = Math.Round(Convert.ToDouble(reader[3]), 2), + Failed = Math.Round(Convert.ToDouble(reader[4]), 2), + }; + result.Result.Add(item); + } + } + return result; + } + + public async Task> InstancePageAsync(BaseApmRequestDto query) + { + var parameters = new List(); + var where = AppendWhere(query, parameters); + + var result = new PaginatedListBase(); + var groupby = "group by ServiceName,`Attributes.http.target`,`method`"; + var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; + result.Total = Convert.ToInt64(Scalar(countSql, parameters)); + var orderBy = GetOrderBy(query); + var start = (query.Page - 1) * query.PageSize; + if (result.Total - start > 0) + { + var sql = $@"select * from( select `Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, +AVG(Duration) Latency, +count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)) throughput +sum(has(['{string.Join("','", query.ErrorStatusCodes)}'],`Attributes.http.status_code`))/count(1) failed +from {where} {groupby} {orderBy} limit{start},{query.PageSize})"; + using var reader = Query(sql, parameters); + result.Result = new(); + while (reader.NextResult()) + while (reader.Read()) + { + var item = new EndpointListDto() + { + Name = reader[0].ToString()!, + Service = reader[1]?.ToString()!, + Method = reader[2]?.ToString()!, + Latency = (long)Math.Floor(Convert.ToDouble(reader[3])), + Throughput = Math.Round(Convert.ToDouble(reader[4]), 2), + Failed = Math.Round(Convert.ToDouble(reader[5]), 2) + }; + result.Result.Add(item); + } + } + return result; + } + + public async Task> DependencyPageAsync(BaseApmRequestDto query) + { + var parameters = new List(); + var where = AppendWhere(query, parameters); + + var result = new PaginatedListBase(); + var groupby = "group by ServiceName,`Attributes.http.target`,`method`"; + var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; + result.Total = Convert.ToInt64(Scalar(countSql, parameters)); + var orderBy = GetOrderBy(query); + var start = (query.Page - 1) * query.PageSize; + if (result.Total - start > 0) + { + var sql = $@"select * from( select `Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, +AVG(Duration) Latency, +count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)) throughput +sum(has(['{string.Join(",'", query.ErrorStatusCodes)}'],`Attributes.http.status_code`))/count(1) failed +from {where} {groupby} {orderBy} limit{start},{query.PageSize})"; + using var reader = Query(sql, parameters); + result.Result = new(); + while (reader.NextResult()) + while (reader.Read()) + { + var item = new EndpointListDto() + { + Name = reader[0].ToString()!, + Service = reader[1]?.ToString()!, + Method = reader[2]?.ToString()!, + Latency = (long)Math.Floor(Convert.ToDouble(reader[3])), + Throughput = Math.Round(Convert.ToDouble(reader[4]), 2), + Failed = Math.Round(Convert.ToDouble(reader[5]), 2) + }; + result.Result.Add(item); + } + } + return result; + } + + public async Task> EndpointPageAsync(BaseApmRequestDto query) + { + var parameters = new List(); + var where = AppendWhere(query, parameters); + var result = new PaginatedListBase(); + var groupby = "group by ServiceName,`Attributes.http.target`,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')]"; + var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; + result.Total = Convert.ToInt64(Scalar(countSql, parameters)); + var orderBy = GetOrderBy(query); + var start = (query.Page - 1) * query.PageSize; + if (result.Total - start > 0) + { + var sql = $@"select * from( select `Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, +floor(AVG(Duration)) Latency, +round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) throughput, +round(sum(has(['{string.Join("','", query.ErrorStatusCodes)}'],`Attributes.http.status_code`))*100.0/count(1),2) failed +from {Constants.TraceTableFull} where {where} and Attributes.http.target!='' {groupby} {orderBy} limit {start},{query.PageSize})"; + using var reader = Query(sql, parameters); + result.Result = new(); + while (reader.NextResult()) + while (reader.Read()) + { + var item = new EndpointListDto() + { + Name = reader[0].ToString()!, + Service = reader[1]?.ToString()!, + Method = reader[2]?.ToString()!, + Latency = (long)Math.Floor(Convert.ToDouble(reader[3])), + Throughput = Math.Round(Convert.ToDouble(reader[4]), 2), + Failed = Math.Round(Convert.ToDouble(reader[5]), 2) + }; + result.Result.Add(item); + } + } + return result; + } + + public async Task> ChartDataAsync(BaseApmRequestDto query) + { + var parameters = new List(); + var where = AppendWhere(query, parameters); + var result = new List(); + var groupby = "group by ServiceName ,`time` order by ServiceName ,`time`"; + var sql = $@"select +ServiceName, +toStartOfInterval(`Timestamp` , INTERVAL 1 minute ) as `time`, +floor(avg(Duration)) latency, +floor(quantile(0.95)(Duration)) p95, +floor(quantile(0.99)(Duration)) p99, +round(sum(has(['{string.Join("','", query.ErrorStatusCodes)}'],`Attributes.http.status_code`))*100.0/count(1),2) failed, +round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) throughput +from {Constants.TraceTableFull} where {where} {groupby}"; + using var reader = Query(sql, parameters); + SetChartData(result, reader); + GetPreviousChartData(query, sql, parameters, result); + return result; + } + + private void GetPreviousChartData(BaseApmRequestDto query, string sql, List parameters, List result) + { + if (!query.ComparisonType.HasValue) + return; + + int day = 0; + switch (query.ComparisonType.Value) + { + case ComparisonTypes.DayBefore: + day = -1; + break; + case ComparisonTypes.WeekBefore: + day = -7; + break; + } + if (day == 0) + return; + + var paramStartTime = parameters.First(p => p.ParameterName == "startTime"); + paramStartTime.Value = ((DateTime)paramStartTime.Value!).AddDays(day); + + var paramEndTime = parameters.First(p => p.ParameterName == "endTime"); + paramEndTime.Value = ((DateTime)paramEndTime.Value!).AddDays(day); + + using var readerPrevious = Query(sql, parameters); + SetChartData(result, readerPrevious, isPrevious: true); + } + + private static void SetChartData(List result, IDataReader reader, bool isPrevious = false) + { + ChartLineDto? current = null; + while (reader.NextResult()) + while (reader.Read()) + { + var name = reader[0].ToString()!; + var time = new DateTimeOffset(Convert.ToDateTime(reader[1])).ToUnixTimeSeconds(); + if (current == null || current.Name != name) + { + if (isPrevious) + { + if (result.Exists(item => item.Name == name)) + { + current = result.First(item => item.Name == name); + } + else + { + current = new ChartLineDto + { + Name = name, + Previous = new List() + }; + result.Add(current); + } + } + else + { + current = new ChartLineDto + { + Name = name, + Currents = new List() + }; + result.Add(current); + } + } + + ((List)(isPrevious ? current.Previous : current.Currents)).Add( + new() + { + Latency = (long)Math.Floor(Convert.ToDouble(reader[2])), + P95 = Math.Round(Convert.ToDouble(reader[3]), 2, MidpointRounding.ToZero), + P99 = Math.Round(Convert.ToDouble(reader[4]), 2, MidpointRounding.ToZero), + Failed = Math.Round(Convert.ToDouble(reader[5]), 2, MidpointRounding.ToZero), + Throughput = Math.Round(Convert.ToDouble(reader[6]), 2, MidpointRounding.ToZero), + Time = time + }); + } + } + + public async Task EndpointLatencyDistributionAsync(ApmEndpointRequestDto query) + { + var parameters = new List(); + var where = AppendWhere(query, parameters); + + var result = new EndpointLatencyDistributionDto(); + var p95 = Convert.ToDouble(Scalar($"select floor(quantile(0.95)(Duration)) p95 from {Constants.TraceTableFull} where {where}", parameters)); + if (p95 is not double.NaN) + result.P95 = (long)Math.Floor(p95); + var sql = $@"select Duration,count(1) total from {Constants.TraceTableFull} where {where} group by Duration order by Duration"; + using var reader = Query(sql, parameters); + var list = new List(); + while (reader.NextResult()) + while (reader.Read()) + { + var item = new ChartPointDto() + { + X = reader[0].ToString()!, + Y = reader[1]?.ToString()! + }; + list.Add(item); + } + result.Latencies = list; + return result; + } + + public async Task> ErrorMessagePageAsync(ApmEndpointRequestDto query) + { + var parameters = new List(); + var where = AppendWhere(query, parameters); + + var result = new PaginatedListBase(); + var groupby = $"group by Type{(string.IsNullOrEmpty(query.Endpoint) ? "" : ",Endpoint")}"; + var countSql = $"select count(1) from (select Attributes.exception.type as Type,max(Timestamp) time,count(1) from {Constants.ErrorTableFull} where {where} {groupby})"; + result.Total = Convert.ToInt64(Scalar(countSql, parameters)); + var orderBy = GetOrderBy(query); + var start = (query.Page - 1) * query.PageSize; + if (result.Total - start > 0) + { + var sql = $@"select * from( select Attributes.exception.type as Type,max(Timestamp) time,count(1) total +from {Constants.ErrorTableFull} where {where} {groupby} {orderBy} limit {start},{query.PageSize})"; + using var reader = Query(sql, parameters); + result.Result = new(); + while (reader.NextResult()) + while (reader.Read()) + { + var item = new ErrorMessageDto() + { + Type = reader[0]?.ToString()!, + LastTime = Convert.ToDateTime(reader[1])!, + Total = Convert.ToInt32(reader[2]), + }; + result.Result.Add(item); + } + } + return result; + } + + private static string AppendWhere(TQuery query, List parameters) where TQuery : BaseApmRequestDto + { + var sql = new StringBuilder(); + sql.AppendLine(" Timestamp between @startTime and @endTime"); + parameters.Add(new ClickHouseParameter { ParameterName = "startTime", Value = query.Start, DbType = DbType.DateTime }); + parameters.Add(new ClickHouseParameter { ParameterName = "endTime", Value = query.End, DbType = DbType.DateTime }); + if (!string.IsNullOrEmpty(query.Env)) + { + sql.AppendLine(" and Resource.service.namespace=@environment"); + parameters.Add(new ClickHouseParameter { ParameterName = "environment", Value = query.Env }); + } + if (!string.IsNullOrEmpty(query.Service)) + { + sql.AppendLine(" and ServiceName=@serviceName"); + parameters.Add(new ClickHouseParameter { ParameterName = "serviceName", Value = query.Service }); + } + if (query.IsServer.HasValue) + { + sql.AppendLine(" and SpanKind=@spanKind"); + parameters.Add(new ClickHouseParameter { ParameterName = "serviceName", Value = query.IsServer.Value ? "SPAN_KIND_SERVER" : "SPAN_KIND_CLIENT" }); + } + + if (query is ApmEndpointRequestDto traceQuery && !string.IsNullOrEmpty(traceQuery.Endpoint)) + { + sql.AppendLine(" and Attributes.http.target=@endpoint"); + parameters.Add(new ClickHouseParameter { ParameterName = "endpoint", Value = traceQuery.Endpoint }); + } + + if (query is ApmTraceLatencyRequestDto durationQuery) + { + if (durationQuery.LatMin > 0 && durationQuery.LatMax > 0) + { + sql.AppendLine(" and Duration between @minDuration and @maxDuration"); + parameters.Add(new ClickHouseParameter { ParameterName = "minDuration", Value = durationQuery.LatMin }); + parameters.Add(new ClickHouseParameter { ParameterName = "maxDuration", Value = durationQuery.LatMax }); + } + else if (durationQuery.LatMin > 0) + { + sql.AppendLine(" and Duration >=@minDuration"); + parameters.Add(new ClickHouseParameter { ParameterName = "minDuration", Value = durationQuery.LatMin }); + } + else if (durationQuery.LatMax > 0) + { + sql.AppendLine(" and Duration <=@maxDuration"); + parameters.Add(new ClickHouseParameter { ParameterName = "maxDuration", Value = durationQuery.LatMax }); + } + } + + return sql.ToString(); + } + + public async Task> TraceLatencyDetailAsync(ApmTraceLatencyRequestDto query) + { + var queryDto = new BaseRequestDto + { + Start = query.Start, + End = query.End, + Endpoint = query.Endpoint, + Service = query.Service! + }; + var conditions = new List(); + if (!string.IsNullOrEmpty(query.Env)) + { + conditions.Add(new FieldConditionDto + { + Name = "Resource.service.namespace", + Type = ConditionTypes.Equal, + Value = query.Env + }); + } + if (query.LatMin.HasValue && query.LatMin.Value >= 0) + { + conditions.Add(new FieldConditionDto + { + Name = "Duration", + Type = ConditionTypes.GreatEqual, + Value = query.LatMin.Value, + }); + } + + if (query.LatMax.HasValue && query.LatMax.Value >= 0 && ( + !query.LatMin.HasValue + || query.LatMin.HasValue && query.LatMax - query.LatMin.Value > 0)) + conditions.Add(new FieldConditionDto + { + Name = "Duration", + Type = ConditionTypes.LessEqual, + Value = query.LatMax.Value, + }); + if (conditions.Any()) + queryDto.Conditions = conditions; + + return await _traceService.ListAsync(queryDto); + } + + private IDataReader Query(string sql, IEnumerable parameters) + { + _command.CommandText = sql; + SetParameters(parameters); + return _command.ExecuteReader(); + } + + private object Scalar(string sql, IEnumerable parameters) + { + _command.CommandText = sql; + SetParameters(parameters); + return _command.ExecuteScalar()!; + } + + private void SetParameters(IEnumerable parameters) + { + if (_command.Parameters.Count > 0) + _command.Parameters.Clear(); + if (parameters != null && parameters.Any()) + foreach (var param in parameters) + _command.Parameters.Add(param); + } + + private static string? GetOrderBy(BaseApmRequestDto query, string? defaultSort = null, bool isDesc = false) + { + if (string.IsNullOrEmpty(query.OrderField)) + { + if (string.IsNullOrEmpty(defaultSort)) + return null; + return $"order by {defaultSort}{(isDesc ? " desc" : "")}"; + } + if (!query.IsDesc.HasValue) + return $"order by {query.OrderField}"; + return $"order by {query.OrderField}{(query.IsDesc.Value ? "" : " desc")}"; + } + + public void Dispose() + { + _dbConnection.Close(); + _dbConnection.Dispose(); + } +} diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Config/Constants.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Config/Constants.cs new file mode 100644 index 000000000..70c326913 --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Config/Constants.cs @@ -0,0 +1,31 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Config; + +internal static class Constants +{ + public static string Database { get; private set; } + + public static string TraceTableFull => $"{Database}.{TraceTable}"; + + public static string ErrorTableFull => $"{Database}.{ErrorTable}"; + + public static string LogTableFull => $"{Database}.{LogTable}"; + + public static string LogTable { get; private set; } + + public static string TraceTable { get; private set; } + + public static string ErrorTable { get; private set; } + + public static void Init(string database, string logTable, string traceTable, string errorTable) + { + Database = database; + LogTable = logTable; + TraceTable = traceTable; + ErrorTable = errorTable; + } + + public static int[] DefaultErrorStatus = new int[] { 500, 501, 502, 503, 504, 505 }; +} diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IAPMService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IAPMService.cs new file mode 100644 index 000000000..176b2daa3 --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IAPMService.cs @@ -0,0 +1,49 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse; + +public interface IAPMService +{ + /// + /// 服务列表页，服务详情页endpoints和instance公用 + /// + /// + /// + Task> ServicePageAsync(BaseApmRequestDto query); + + /// + /// trace列表 + /// + /// + /// + Task> EndpointPageAsync(BaseApmRequestDto query); + + /// + /// 可共用，service和endpoint公用 + /// + /// + /// + Task> ChartDataAsync(BaseApmRequestDto query); + + /// + /// endpoint 加载耗时分布 + /// + /// + /// + Task EndpointLatencyDistributionAsync(ApmEndpointRequestDto query); + + /// + /// tendpoint trace tree line + /// + /// + /// + Task> TraceLatencyDetailAsync(ApmTraceLatencyRequestDto query); + + /// + /// 错误列表 + /// + /// + /// + Task> ErrorMessagePageAsync(ApmEndpointRequestDto query); +} diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj new file mode 100644 index 000000000..9df74467d --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj @@ -0,0 +1,15 @@ + + + + net6.0 + enable + enable + + + + + + + + + diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/APMEnums.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/APMEnums.cs new file mode 100644 index 000000000..00c23676e --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/APMEnums.cs @@ -0,0 +1,10 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models; + +public enum ComparisonTypes +{ + DayBefore = 1, + WeekBefore +} diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmEndpointRequestDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmEndpointRequestDto.cs new file mode 100644 index 000000000..6ec3e0178 --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmEndpointRequestDto.cs @@ -0,0 +1,9 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Request; + +public class ApmEndpointRequestDto : BaseApmRequestDto +{ + public string Endpoint { get; set; } +} diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmTraceLatencyRequestDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmTraceLatencyRequestDto.cs new file mode 100644 index 000000000..14b8264bf --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmTraceLatencyRequestDto.cs @@ -0,0 +1,13 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Request; + +public class ApmTraceLatencyRequestDto : ApmEndpointRequestDto +{ + public long? LatMin { get; set; } + + public long? LatMax { get; set; } + + public new int PageSize { get; } = 1; +} diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs new file mode 100644 index 000000000..051af9739 --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs @@ -0,0 +1,31 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Request; + +public class BaseApmRequestDto : RequestPageBase +{ + public string? Env { get; set; } + + public ComparisonTypes? ComparisonType { get; set; } + + public DateTime Start { get; set; } + + public DateTime End { get; set; } + + public string Period { get; set; } + + public string? Service { get; set; } + + public string? Queries { get; set; } + + public string? OrderField { get; set; } + + public bool? IsDesc { get; set; } + + public string StatusCodes { get; set; } + + internal int[] ErrorStatusCodes => string.IsNullOrEmpty(StatusCodes) ? Constants.DefaultErrorStatus : StatusCodes.Split(',').Select(s => Convert.ToInt32(s)).Where(num => num != 0).ToArray(); + + internal bool? IsServer { get; set; } +} diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs new file mode 100644 index 000000000..8609c5e62 --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs @@ -0,0 +1,28 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Response; + +public class EndpointChartDto +{ + public IEnumerable P99s { get; set; } + + public IEnumerable P95s { get; set; } + + public IEnumerable Latencies { get; set; } + + public IEnumerable Throughputs { get; set; } + + public IEnumerable Fails { get; set; } +} + +public class ErrorMessageDto +{ + public string Type { get; set; } + + //public string Message { get; set; } + + public DateTime LastTime { get; set; } + + public int Total { get; set; } +} diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointLatencyDistributionDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointLatencyDistributionDto.cs new file mode 100644 index 000000000..78146aa55 --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointLatencyDistributionDto.cs @@ -0,0 +1,11 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Response; + +public class EndpointLatencyDistributionDto +{ + public long? P95 { get; set; } + + public IEnumerable Latencies { get; set; } +} diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointListDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointListDto.cs new file mode 100644 index 000000000..a3de5941c --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointListDto.cs @@ -0,0 +1,21 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Response; + +public class EndpointListDto +{ + public string Name { get; set; } + + public string Method { get; set; } + + public string Service { get; set; } + + public string AgentType { get; set; } + + public long Latency { get; set; } + + public double Failed { get; set; } + + public double Throughput { get; set; } +} diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/ServiceListDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/ServiceListDto.cs new file mode 100644 index 000000000..6789b0fec --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/ServiceListDto.cs @@ -0,0 +1,50 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Response; + +public class ServiceListDto +{ + public string Service { get; set; } + + public string Name { get; set; } + + public IEnumerable Envs { get; set; } + + public long Latency { get; set; } + + public double Throughput { get; set; } + + public double Failed { get; set; } +} + +public class ChartLineDto +{ + public string Name { get; set; } + + public IEnumerable Currents { get; set; } + + public IEnumerable Previous { get; set; } +} + +public class ChartLineItemDto +{ + public long Time { get; set; } + + public long Latency { get; set; } + + public double P99 { get; set; } + + public double P95 { get; set; } + + public double Throughput { get; set; } + + public double Failed { get; set; } +} + +public class ChartPointDto +{ + public string X { get; set; } + + public string Y { get; set; } +} diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/TraceDetailDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/TraceDetailDto.cs new file mode 100644 index 000000000..a2bdb150c --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/TraceDetailDto.cs @@ -0,0 +1,9 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Response; + +public class TraceDetailDto +{ + +} diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs new file mode 100644 index 000000000..533095307 --- /dev/null +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs @@ -0,0 +1,19 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +global using ClickHouse.Ado; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Service; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Trace; +global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse; +global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Cliclhouse; +global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Config; +global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models; +global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Request; +global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Response; +global using Masa.Contrib.StackSdks.Tsc.Clickhouse; +global using Masa.Utils.Models; +global using Microsoft.Extensions.DependencyInjection.Extensions; +global using Microsoft.Extensions.Logging; +global using System.Data; +global using System.Data.Common; +global using System.Text; diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs index dbefd7258..c0eb79738 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs @@ -1,6 +1,6 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. - +[assembly: InternalsVisibleTo("Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse")] namespace System.Data.Common; internal static class IDbConnectionExtensitions @@ -106,7 +106,7 @@ public static (string where, List @parameters, List ors) if (isTrace && !string.IsNullOrEmpty(query.Endpoint)) { sql.Append(" and `Attributes.http.target`=@HttpTarget"); - @paramerters.Add(new ClickHouseParameter() { ParameterName = "HttpTarget", Value = query.Instance }); + @paramerters.Add(new ClickHouseParameter() { ParameterName = "HttpTarget", Value = query.Endpoint }); } var ors = AppendKeyword(query.Keyword, paramerters, isTrace); AppendConditions(query.Conditions, paramerters, sql, isTrace); @@ -263,7 +263,7 @@ private static void ParseWhere(StringBuilder sql, object value, List Query(this IDbConnection dbConnection, string sql, IDat } catch (Exception ex) { - ServiceExtensitions.Logger?.LogError(ex, "query sql error:{rawSql}, paramters:{parameters}", sql, parameters); + MasaTscCliclhouseExtensitions.Logger?.LogError(ex, "query sql error:{rawSql}, paramters:{parameters}", sql, parameters); throw; } } @@ -353,7 +353,7 @@ public static LogResponseDto ConvertLogDto(IDataReader reader) SeverityText = reader["SeverityText"].ToString()!, TraceFlags = Convert.ToInt32(reader["TraceFlags"]), SpanId = reader["SpanId"].ToString()!, - Timestamp = Convert.ToDateTime(reader["Timestamp"]).ToLocalTime(), + Timestamp = Convert.ToDateTime(reader["Timestamp"]), }; if (!string.IsNullOrEmpty(resource)) result.Resource = JsonSerializer.Deserialize>(resource)!; @@ -450,6 +450,9 @@ private static string GetName(string name, bool isLog) if (name.Equals("@timestamp", StringComparison.CurrentCultureIgnoreCase)) return "Timestamp"; + if (!isLog && name.Equals("duration", StringComparison.CurrentCultureIgnoreCase)) + return "Duration"; + if (!isLog && name.Equals("kind", StringComparison.InvariantCultureIgnoreCase)) return "SpanKind"; @@ -484,7 +487,8 @@ private static string GetAttributeName(string name, bool isLog) if (!isLog && (field.Equals("http.status_code", StringComparison.CurrentCultureIgnoreCase) || field.Equals("http.request_content_body", StringComparison.CurrentCultureIgnoreCase) || field.Equals("http.response_content_body", StringComparison.CurrentCultureIgnoreCase) - || field.Equals("exception.message", StringComparison.CurrentCultureIgnoreCase)) + || field.Equals("exception.message", StringComparison.CurrentCultureIgnoreCase) + || field.Equals("http.target", StringComparison.CurrentCultureIgnoreCase)) ) return $"Attributes.{field}"; diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/ServiceExtensitions.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/MasaTscCliclhouseExtensitions.cs similarity index 96% rename from src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/ServiceExtensitions.cs rename to src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/MasaTscCliclhouseExtensitions.cs index 715373ba0..c8c88b97e 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/ServiceExtensitions.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/MasaTscCliclhouseExtensitions.cs @@ -3,16 +3,17 @@ namespace Microsoft.Extensions.DependencyInjection; -public static class ServiceExtensitions +public static class MasaTscCliclhouseExtensitions { internal static ILogger? Logger { get; private set; } - public static IServiceCollection AddMASAStackClickhouse(this IServiceCollection services, string connectionStr, string logTable, string traceTable, string? logSourceTable = null, string? traceSourceTable = null) + public static IServiceCollection AddMASAStackClickhouse(this IServiceCollection services, string connectionStr, string logTable, string traceTable, string? logSourceTable = null, string? traceSourceTable = null, Action? configer = null) { services.AddScoped(services => new MasaStackClickhouseConnection(connectionStr, logTable, traceTable, logSourceTable, traceSourceTable)) .AddScoped() .AddScoped(); Init(services); + configer?.Invoke(services.BuildServiceProvider().GetRequiredService()!); return services; } @@ -122,7 +123,7 @@ TTL toDateTime(Timestamp) + toIntervalDay(30) `Attributes.http.status_code` String CODEC(ZSTD(1)), `Attributes.http.response_content_body` String CODEC(ZSTD(1)), `Attributes.http.request_content_body` String CODEC(ZSTD(1)), - `Attributes.http.Target` String CODEC(ZSTD(1)), + `Attributes.http.target` String CODEC(ZSTD(1)), `Attributes.exception.message` String CODEC(ZSTD(1)), `ResourceAttributesKeys` Array(String) CODEC(ZSTD(1)), @@ -234,10 +235,10 @@ select DISTINCT arraySort(mapKeys(ResourceAttributes)) as Name, 'log_resource' a (['Timestamp','TraceId','SpanId','ParentSpanId','TraceState','SpanKind','Duration'],'trace_basic'); " }; foreach (var sql in initSqls) - ExecuteSql(connection, sql); + connection.ExecuteSql(sql); } - private static void ExecuteSql(MasaStackClickhouseConnection connection, string sql) + internal static void ExecuteSql(this IDbConnection connection, string sql) { using var cmd = connection.CreateCommand(); if (connection.State != ConnectionState.Open) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs index f50a424f0..69fbce49d 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs @@ -1,9 +1,9 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. - +[assembly: InternalsVisibleTo("Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse")] namespace Masa.Contrib.StackSdks.Tsc.Clickhouse; -internal class MasaStackClickhouseConnection : ClickHouseConnection +internal sealed class MasaStackClickhouseConnection : ClickHouseConnection { public static string LogSourceTable { get; private set; } diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/_Imports.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/_Imports.cs index 7cb31407e..51f1b7ee3 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/_Imports.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/_Imports.cs @@ -14,6 +14,7 @@ global using System.Collections; global using System.Data; global using System.Data.Common; +global using System.Runtime.CompilerServices; global using System.Text; global using System.Text.Json; global using System.Text.RegularExpressions; diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.OpenTelemetry/Options/OpenTelemetryAttributeName.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.OpenTelemetry/Options/OpenTelemetryAttributeName.cs index 9757f838f..953d4becd 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.OpenTelemetry/Options/OpenTelemetryAttributeName.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.OpenTelemetry/Options/OpenTelemetryAttributeName.cs @@ -59,6 +59,8 @@ internal static class EndUser /// internal static class Http { + public const string STATUS_CODE = "status_code"; + /// /// The URI scheme identifying the used protocol. /// diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs new file mode 100644 index 000000000..d2e0fafe9 --- /dev/null +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs @@ -0,0 +1,130 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests; + +[TestClass] +public class ClickhouseApmServiceTests +{ + private static IAPMService _APMService; + private DateTime _start = DateTime.Parse("2024/01/03 14:44:18"); + + [ClassInitialize] + public static void Initialized(TestContext testContext) + { + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddConsole()); + services.AddMASAStackApmClickhouse(TestUtils.ConnectionString, "custom_log", "custom_trace"); + _APMService = services.BuildServiceProvider().GetRequiredService(); + } + + [TestMethod] + public async Task ServicePageAsync() + { + var query = new ApmEndpointRequestDto + { + Start = _start, + End = DateTime.Now,// _start.AddHours(1), + ComparisonType = ComparisonTypes.DayBefore, + StatusCodes = "401,402,503,500", + PageSize = 10, + Page = 1 + }; + var result = await _APMService.ServicePageAsync(query); + Assert.IsNotNull(result); + Assert.IsTrue(result.Total > 0); + Assert.IsNotNull(result.Result); + } + + [TestMethod] + public async Task EndpointPageAsync() + { + var query = new ApmEndpointRequestDto + { + Start = _start, + End = DateTime.Now, + ComparisonType = ComparisonTypes.DayBefore, + StatusCodes = "401,402,503,500", + PageSize = 10, + Page = 1, + Service = "tsc-service-iotdev" + }; + var result = await _APMService.EndpointPageAsync(query); + Assert.IsNotNull(result); + Assert.IsTrue(result.Total > 0); + Assert.IsNotNull(result.Result); + } + + [TestMethod] + public async Task ChartDataAsync() + { + var query = new ApmEndpointRequestDto + { + Start = _start, + End = DateTime.Now, + ComparisonType = ComparisonTypes.DayBefore, + StatusCodes = "401,402,503,500", + PageSize = 10, + Page = 1, + Service = "tsc-service-iotdev" + }; + var result = await _APMService.ChartDataAsync(query); + Assert.IsNotNull(result); + } + + [TestMethod] + public async Task EndpointLatencyDistributionAsync() + { + var query = new ApmEndpointRequestDto + { + Start = _start, + End = DateTime.Now, + ComparisonType = ComparisonTypes.DayBefore, + StatusCodes = "401,402,503,500", + PageSize = 10, + Page = 1, + Service = "tsc-service-iotdev", + Endpoint = "/api/trace/list" + }; + var result = await _APMService.EndpointLatencyDistributionAsync(query); + Assert.IsNotNull(result); + } + + [TestMethod] + public async Task ErrorMessagePageAsync() + { + var query = new ApmEndpointRequestDto + { + Start = _start, + End = DateTime.Now, + ComparisonType = ComparisonTypes.DayBefore, + StatusCodes = "401,402,503,500", + PageSize = 10, + Page = 1, + Service = "tsc-service-iotdev" + }; + var result = await _APMService.ErrorMessagePageAsync(query); + Assert.IsNotNull(result); + } + + [TestMethod] + public async Task TraceLatencyDetailAsync() + { + var query = new ApmTraceLatencyRequestDto + { + Start = _start, + End = DateTime.Now, + ComparisonType = ComparisonTypes.DayBefore, + StatusCodes = "401,402,503,500", + Page = 1, + Service = "tsc-service-iotdev", + Endpoint = "/api/trace/list" + }; + var result = await _APMService.TraceLatencyDetailAsync(query); + Assert.IsNotNull(result); + query.Env = "Development"; + query.LatMax = 1000000; + result = await _APMService.TraceLatencyDetailAsync(query); + Assert.IsNotNull(result); + } +} diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Data/otel_trace_data.txt b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Data/otel_trace_data.txt new file mode 100644 index 000000000..294dce286 --- /dev/null +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Data/otel_trace_data.txt @@ -0,0 +1,50 @@ + INSERT INTO otel_traces (`Timestamp`, TraceId, SpanId, ParentSpanId, TraceState, SpanName, SpanKind, ServiceName, ResourceAttributes, ScopeName, ScopeVersion, SpanAttributes, Duration, StatusCode, StatusMessage, `Events.Timestamp`, `Events.Name`, `Events.Attributes`, `Links.TraceId`, `Links.SpanId`, `Links.TraceState`, `Links.Attributes`) +VALUES + +2024-01-04 06:02:12.982357700 ecc21857f5032d057571b1230e8eb014 1dd1edd9a64dccd3 c819bede56087c3d HTTP GET SPAN_KIND_CLIENT tsc-service-iotdev 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev Development OpenTelemetry.Instrumentation.Http.HttpClient 1.0.0.0 http GET 10.130.0.73 http://10.130.0.73:30090/api/v1/query_range?end=1704348132&query=round(sum+(increase(http_server_duration_sum%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2fsum+(increase(http_server_duration_count%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2c1)&start=1704304932&step=172s http GET 10.130.0.73 http://10.130.0.73:30090/api/v1/query_range?end=1704348132&query=round(sum+(increase(http_server_duration_sum%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2fsum+(increase(http_server_duration_count%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2c1)&start=1704304932&step=172s 10.130.0.73:30090 30090 1.1 Lonsid-52 200 http://10.130.0.73:30090/api/v1/query_range?end=1704348132&query=round(sum+(increase(http_server_duration_sum%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2fsum+(increase(http_server_duration_count%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2c1)&start=1704304932&step=172s 30090 10.130.0.73 30090 150848800 STATUS_CODE_UNSET [] [] [] [] [] [] [] +2024-01-04 06:02:12.828538400 ecc21857f5032d057571b1230e8eb014 3fe92b307ca25331 c819bede56087c3d HTTP GET SPAN_KIND_CLIENT tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev OpenTelemetry.Instrumentation.Http.HttpClient 1.0.0.0 http 10.130.0.73 30090 http://10.130.0.73:30090/api/v1/query_range?end=1704348132&query=round(sum+(increase(http_server_duration_count%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2c1)&start=1704304932&step=172s 200 http 10.130.0.73 30090 http://10.130.0.73:30090/api/v1/query_range?end=1704348132&query=round(sum+(increase(http_server_duration_count%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2c1)&start=1704304932&step=172s 200 10.130.0.73:30090 GET 1.1 Lonsid-52 200 GET http://10.130.0.73:30090/api/v1/query_range?end=1704348132&query=round(sum+(increase(http_server_duration_count%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2c1)&start=1704304932&step=172s GET 128014800 STATUS_CODE_UNSET [] [] [] [] [] [] [] +2024-01-04 06:02:12.684341700 ecc21857f5032d057571b1230e8eb014 7e2d0530e1bf8106 c819bede56087c3d HTTP GET SPAN_KIND_CLIENT tsc-service-iotdev 1.5.1 masastack tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet OpenTelemetry.Instrumentation.Http.HttpClient 1.0.0.0 http GET 30090 1.1 Lonsid-52 http GET 30090 1.1 Lonsid-52 200 10.130.0.73:30090 10.130.0.73 http://10.130.0.73:30090/api/v1/label/__name__/values?lable=__name__ Lonsid-52 200 1.1 200 105716900 STATUS_CODE_UNSET [] [] [] [] [] [] [] +2024-01-04 06:02:12.656875700 ecc21857f5032d057571b1230e8eb014 c819bede56087c3d 1ad3ed80e6cd7563 /api/metric/multi-range SPAN_KIND_SERVER tsc-service-iotdev opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc OpenTelemetry.Instrumentation.AspNetCore 1.0.0.0 /api/metric/multi-range 328 200 GET /api/metric/multi-range 328 200 GET http://localhost:18010/api/metric/multi-range HTTP/1.1 application/json; charset=utf-8 Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ Lonsid-52 application/json; charset=utf-8 18010 http GET HTTP/1.1 {"Layer":null,"Service":null,"Instance":null,"EndPoint":null,"Start":"2024-01-03T18:02:12.1674699Z","End":"2024-01-04T06:02:12.1674699Z","Step":"172s","MetricNames":["round(sum (increase(http_server_duration_count[1m])),1)","round(sum (increase(http_server_duration_sum[1m]))/sum (increase(http_server_duration_count[1m])),1)"]} 200 HTTP/1.1 490802200 STATUS_CODE_UNSET [] [] [] [] [] [] [] +2024-01-04 06:02:12.391519700 bfad12923f64935cec97f5e3dd5b9a47 f7d9f646dbfc500d tsc_iotdev SPAN_KIND_CLIENT tsc-service-iotdev dotnet 1.5.1 masastack tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry OpenTelemetry.Instrumentation.EntityFrameworkCore 1.0.0.7 10.130.0.19,2959 Text "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] +FROM [tsc].[IntegrationEventLog] AS [i] +WHERE [i].[State] = 0 +ORDER BY [i].[CreationTime]" mssql tsc_iotdev 10.130.0.19,2959 Text "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] +FROM [tsc].[IntegrationEventLog] AS [i] +WHERE [i].[State] = 0 +ORDER BY [i].[CreationTime]" mssql tsc_iotdev tsc_iotdev mssql 145378900 STATUS_CODE_UNSET [] [] [] [] [] [] [] +2024-01-04 06:02:12.372382800 bca3e932f72628858f040df98172fa09 e7591443dde17f6e 2700571f3f7589d4 /api/trace/list SPAN_KIND_SERVER tsc-service-iotdev dotnet 1.5.1 masastack tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry OpenTelemetry.Instrumentation.AspNetCore 1.0.0.0 localhost Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ 200 18010 localhost Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ 200 18010 http://localhost:18010/api/trace/list?Start=2024/1/3%2018:02:12&End=2024/1/4%206:02:12&IsDesc=True&IsError=False&Page=1&PageSize=10 Lonsid-52 http /api/trace/list GET HTTP/1.1 application/json; charset=utf-8 18010 200 243969100 STATUS_CODE_UNSET [] [] [] [] [] [] [] +2024-01-04 06:02:12.186883900 15a58db826237e3c2f9ff872c10f1e27 55fdbbb0b5755535 28acb93474b6ddf7 /api/trace/attr-values SPAN_KIND_SERVER tsc-service-iotdev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev Development 1.0-Prev OpenTelemetry.Instrumentation.AspNetCore 1.0.0.0 {"Name":null,"Alias":null,"Type":6,"MaxCount":1000,"Interval":null,"AllValue":false,"TraceId":null,"Service":"","Instance":null,"Endpoint":null,"Keyword":null,"Start":"2024-01-03T18:02:12.1674699Z","End":"2024-01-04T06:02:12.1674699Z","RawQuery":null,"Conditions":null,"Sort":null,"Page":1,"PageSize":20} localhost GET /api/trace/attr-values HTTP/1.1 {"Name":null,"Alias":null,"Type":6,"MaxCount":1000,"Interval":null,"AllValue":false,"TraceId":null,"Service":"","Instance":null,"Endpoint":null,"Keyword":null,"Start":"2024-01-03T18:02:12.1674699Z","End":"2024-01-04T06:02:12.1674699Z","RawQuery":null,"Conditions":null,"Sort":null,"Page":1,"PageSize":20} localhost GET /api/trace/attr-values HTTP/1.1 http://localhost:18010/api/trace/attr-values application/json; charset=utf-8 Lonsid-52 18010 http Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ HTTP/1.1 http://localhost:18010/api/trace/attr-values 200 /api/trace/attr-values http://localhost:18010/api/trace/attr-values 126784100 STATUS_CODE_UNSET [] [] [] [] [] [] [] +2024-01-04 06:02:12.089789900 135eada26529589440f22dd55b6a6975 14864123964f693f 18ead6cf4f8b0d1d /api/trace/errorStatus SPAN_KIND_SERVER tsc-service-iotdev opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc OpenTelemetry.Instrumentation.AspNetCore 1.0.0.0 localhost /api/trace/errorStatus Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ localhost /api/trace/errorStatus Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ Lonsid-52 200 GET http http://localhost:18010/api/trace/errorStatus 18010 HTTP/1.1 application/json; charset=utf-8 200 Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ 200 2004400 STATUS_CODE_UNSET [] [] [] [] [] [] [] +2024-01-04 06:02:11.273064100 5f0982d9ca0972fe0c90e87376477ec8 f7f20f341b595a6c tsc_iotdev SPAN_KIND_CLIENT tsc-service-iotdev tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet 1.5.1 masastack OpenTelemetry.Instrumentation.EntityFrameworkCore 1.0.0.7 mssql tsc_iotdev 10.130.0.19,2959 Text "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] +FROM [tsc].[IntegrationEventLog] AS [i] +WHERE [i].[State] = 0 +ORDER BY [i].[CreationTime]" mssql tsc_iotdev 10.130.0.19,2959 Text "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] +FROM [tsc].[IntegrationEventLog] AS [i] +WHERE [i].[State] = 0 +ORDER BY [i].[CreationTime]" "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] +FROM [tsc].[IntegrationEventLog] AS [i] +WHERE [i].[State] = 0 +ORDER BY [i].[CreationTime]" Text 98267900 STATUS_CODE_UNSET [] [] [] [] [] [] [] +2024-01-04 06:02:10.179119500 6b0fde523c65d18862f50ed99aa6f85e 6ad9e25caa8f4f28 tsc_iotdev SPAN_KIND_CLIENT tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev OpenTelemetry.Instrumentation.EntityFrameworkCore 1.0.0.7 tsc_iotdev 10.130.0.19,2959 Text "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] +FROM [tsc].[IntegrationEventLog] AS [i] +WHERE [i].[State] = 0 +ORDER BY [i].[CreationTime]" mssql tsc_iotdev 10.130.0.19,2959 Text "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] +FROM [tsc].[IntegrationEventLog] AS [i] +WHERE [i].[State] = 0 +ORDER BY [i].[CreationTime]" mssql mssql "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] +FROM [tsc].[IntegrationEventLog] AS [i] +WHERE [i].[State] = 0 +ORDER BY [i].[CreationTime]" 91859100 STATUS_CODE_UNSET [] [] [] [] [] [] [] +2024-01-04 06:02:09.654368700 87a81d697be817ff9e48f7bc80f57301 65f6fdf66547dfad eb3a1ae54e324251 /api/log/list SPAN_KIND_SERVER tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev OpenTelemetry.Instrumentation.AspNetCore 1.0.0.0 18010 /api/log/list http://localhost:18010/api/log/list?Query=a='test'&Start=2024/1/3%2014:47:23&End=2024/1/4%202:47:23&IsDesc=True&Page=1&PageSize=10 Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ 18010 /api/log/list http://localhost:18010/api/log/list?Query=a='test'&Start=2024/1/3%2014:47:23&End=2024/1/4%202:47:23&IsDesc=True&Page=1&PageSize=10 Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ Lonsid-52 http GET 500 localhost text/plain; charset=utf-8 HTTP/1.1 Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ http http://localhost:18010/api/log/list?Query=a='test'&Start=2024/1/3%2014:47:23&End=2024/1/4%202:47:23&IsDesc=True&Page=1&PageSize=10 http 283681400 STATUS_CODE_ERROR [] [] [] [] [] [] [] +2024-01-04 06:02:09.630141400 d6ad209686047701802cbe32b8f3345e b243c61dabdd4676 630bb5b95c34b166 /api/log/list SPAN_KIND_SERVER tsc-service-iotdev 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev Development OpenTelemetry.Instrumentation.AspNetCore 1.0.0.0 text/plain; charset=utf-8 localhost http /api/log/list 500 text/plain; charset=utf-8 localhost http /api/log/list 500 Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ Lonsid-52 18010 GET HTTP/1.1 http://localhost:18010/api/log/list?Query=a='test'&Start=2024/1/3%2014:47:23&End=2024/1/4%202:47:23&IsDesc=True&Page=1&PageSize=10 500 Lonsid-52 /api/log/list Lonsid-52 309668700 STATUS_CODE_ERROR [] [] [] [] [] [] [] + + + + + + + + +('2023-11-02 09:39:30.865', '2821e37c871ee22bcc10b43ddd7ca657', '402f0ed4040433e6', '1c9a707a29a9e449', '', 'HTTP POST', 'SPAN_KIND_CLIENT', 'service', '{''service.instance.id'':''c56534c5-2f62-48e8-82a5-6d8ecdbdd7de'',''telemetry.sdk.name'':''opentelemetry'',''telemetry.sdk.language'':''dotnet'',''telemetry.sdk.version'':''1.5.1'',''service.layer'':''service'',''service.name'':''service'',''service.namespace'':''Test'',''service.version'':''1.0.0''}', 'OpenTelemetry.Instrumentation.Http.HttpClient', '1.0.0.0', '{''http.flavor'':''1.1'',''host.name'':''service-device-676656994-xbnnq'',''http.request_content_body'':''{"deviceInfoId":"5EF36830-438D-4B7F-BA15-AF3A138412BA","activeTime":"2022-03-26T03:15:56.878289","active1":"","active2":"","active3":"","active4":"","active5":"","life1":"","life2":"","life3":"","life4":"","life5":"","maxLife1":"","maxLife2":"","maxLife3":"","maxLife4":"","maxLife5":"","lock":""}'',''peer.service'':''127.0.0.1:3500'',''http.scheme'':''http'',''http.method'':''POST'',''net.peer.name'':''127.0.0.1'',''net.peer.port'':''3500'',''http.url'':''http://127.0.0.1:3500/v1.0/invoke/service/method/api/filter/ActivateDeviceFilter'',''http.status_code'':''200''}', 4828300, 'STATUS_CODE_UNSET', '', [], [], [], [], [], [], []), +('2023-11-02 09:39:30.865', '2821e37c871ee22bcc10b43ddd7ca657', '402f0ed4040433e6', '1c9a707a29a9e449', '', 'HTTP POST', 'SPAN_KIND_CLIENT', 'service', '{''service.instance.id'':''c56534c5-2f62-48e8-82a5-6d8ecdbdd7de'',''telemetry.sdk.name'':''opentelemetry'',''telemetry.sdk.language'':''dotnet'',''telemetry.sdk.version'':''1.5.1'',''service.layer'':''service'',''service.name'':''service'',''service.namespace'':''Test'',''service.version'':''1.0.0''}', 'OpenTelemetry.Instrumentation.Http.HttpClient', '1.0.0.0', '{''http.flavor'':''1.1'',''host.name'':''service-device-676656994-xbnnq'',''http.request_content_body'':''{"deviceInfoId":"C829B9EE-E911-44F8-8E74-1EF991DA6803","activeTime":"2022-03-26T03:15:56.878289","active1":"","active2":"","active3":"","active4":"","active5":"","life1":"","life2":"","life3":"","life4":"","life5":"","maxLife1":"","maxLife2":"","maxLife3":"","maxLife4":"","maxLife5":"","lock":""}'',''peer.service'':''127.0.0.1:3500'',''http.scheme'':''http'',''http.method'':''POST'',''net.peer.name'':''127.0.0.1'',''net.peer.port'':''3500'',''http.url'':''http://127.0.0.1:3500/v1.0/invoke/service/method/api/filter/ActivateDeviceFilter'',''http.status_code'':''200''}', 4828300, 'STATUS_CODE_UNSET', '', [], [], [], [], [], [], []), +('2023-11-02 09:39:30.865', '2821e37c871ee22bcc10b43ddd7ca657', '402f0ed4040433e6', '1c9a707a29a9e449', '', 'HTTP POST', 'SPAN_KIND_CLIENT', 'service', '{''service.instance.id'':''c56534c5-2f62-48e8-82a5-6d8ecdbdd7de'',''telemetry.sdk.name'':''opentelemetry'',''telemetry.sdk.language'':''dotnet'',''telemetry.sdk.version'':''1.5.1'',''service.layer'':''service'',''service.name'':''service'',''service.namespace'':''Test'',''service.version'':''1.0.0''}', 'OpenTelemetry.Instrumentation.Http.HttpClient', '1.0.0.0', '{''http.flavor'':''1.1'',''host.name'':''service-device-676656994-xbnnq'',''http.request_content_body'':''{"deviceInfoId":"8995f797-0385-4b27-9452-181eb0bb4079","activeTime":"2022-03-26T03:15:56.878289","active1":"","active2":"","active3":"","active4":"","active5":"","life1":"","life2":"","life3":"","life4":"","life5":"","maxLife1":"","maxLife2":"","maxLife3":"","maxLife4":"","maxLife5":"","lock":""}'',''peer.service'':''127.0.0.1:3500'',''http.scheme'':''http'',''http.method'':''POST'',''net.peer.name'':''127.0.0.1'',''net.peer.port'':''3500'',''http.url'':''http://127.0.0.1:3500/v1.0/invoke/service/method/api/filter/ActivateDeviceFilter'',''http.status_code'':''200''}', 4828300, 'STATUS_CODE_UNSET', '', [], [], [], [], [], [], []) \ No newline at end of file diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/GlobalUsings.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/GlobalUsings.cs new file mode 100644 index 000000000..89422827d --- /dev/null +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/GlobalUsings.cs @@ -0,0 +1,8 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models; +global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Request; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; +global using Microsoft.VisualStudio.TestTools.UnitTesting; diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj new file mode 100644 index 000000000..53b18cde4 --- /dev/null +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj @@ -0,0 +1,33 @@ + + + + net6.0 + enable + enable + + false + true + + + + + + + + + Always + + + + + + + + + + + + + + + diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs new file mode 100644 index 000000000..adfd4463b --- /dev/null +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs @@ -0,0 +1,10 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests; + +internal class TestUtils +{ + //public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=localhost;Port=9000;Database=default;User=default"; + public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=10000;Host=192.168.51.234;Port=19003;User=test;Password=123456;Database=otel_test"; +} diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Consts.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Consts.cs index 738be3cc6..300063521 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Consts.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Consts.cs @@ -5,6 +5,6 @@ namespace Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests; internal class Consts { - public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=localhost;Port=9000;Database=default;User=default"; - //public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=192.168.51.234;Port=19003;Database=default;User=default"; + //public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=localhost;Port=9000;Database=default;User=default"; + public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=192.168.51.234;Port=19003;Database=default;User=default"; } From d3128842ed5a81b92c93a55b94398547abe30d2f Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Fri, 5 Jan 2024 13:55:40 +0800 Subject: [PATCH 02/24] chore: add tests --- .../Cliclhouse/ClickhouseApmService.cs | 7 +- .../_Imports.cs | 2 + .../Model/MASAStackClickhouseConnection.cs | 1 + .../ClickhouseApmServiceTests.cs | 18 +- .../Data/otel_trace_data.json | 266 ++++++++++++++++++ .../Data/otel_trace_data.txt | 50 ---- .../GlobalUsings.cs | 2 + ....StackSdks.Tsc.Apm.Clickhouse.Tests.csproj | 17 +- .../TestUtils.cs | 4 +- .../Common.cs | 88 ++++-- .../LogServiceTests.cs | 15 +- .../TraceServiceTests.cs | 9 +- .../_Imports.cs | 3 + 13 files changed, 373 insertions(+), 109 deletions(-) create mode 100644 src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Data/otel_trace_data.json delete mode 100644 src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Data/otel_trace_data.txt diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs index 0d9dc9ec9..fef4e38c4 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -1,8 +1,6 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. -using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model; - namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Cliclhouse; internal class ClickhouseApmService : IAPMService, IDisposable @@ -252,6 +250,7 @@ private static void SetChartData(List result, IDataReader reader, current = new ChartLineDto { Name = name, + Previous = new List(), Currents = new List() }; result.Add(current); @@ -333,8 +332,8 @@ private static string AppendWhere(TQuery query, List p { var sql = new StringBuilder(); sql.AppendLine(" Timestamp between @startTime and @endTime"); - parameters.Add(new ClickHouseParameter { ParameterName = "startTime", Value = query.Start, DbType = DbType.DateTime }); - parameters.Add(new ClickHouseParameter { ParameterName = "endTime", Value = query.End, DbType = DbType.DateTime }); + parameters.Add(new ClickHouseParameter { ParameterName = "startTime", Value = MasaStackClickhouseConnection.ToTimeZone(query.Start), DbType = DbType.DateTime }); + parameters.Add(new ClickHouseParameter { ParameterName = "endTime", Value = MasaStackClickhouseConnection.ToTimeZone(query.End), DbType = DbType.DateTime }); if (!string.IsNullOrEmpty(query.Env)) { sql.AppendLine(" and Resource.service.namespace=@environment"); diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs index 533095307..0640ffc04 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See LICENSE.txt in the project root for license information. global using ClickHouse.Ado; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model; global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Service; global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Trace; global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse; @@ -16,4 +17,5 @@ global using Microsoft.Extensions.Logging; global using System.Data; global using System.Data.Common; +global using System.Runtime.CompilerServices; global using System.Text; diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs index 69fbce49d..9813fb5b1 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs @@ -1,6 +1,7 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. [assembly: InternalsVisibleTo("Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse")] +[assembly: InternalsVisibleTo("Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests")] namespace Masa.Contrib.StackSdks.Tsc.Clickhouse; internal sealed class MasaStackClickhouseConnection : ClickHouseConnection diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs index d2e0fafe9..b31b2e2b7 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs @@ -7,15 +7,19 @@ namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests; public class ClickhouseApmServiceTests { private static IAPMService _APMService; - private DateTime _start = DateTime.Parse("2024/01/03 14:44:18"); + private DateTime _start = DateTime.Parse("2024/01/03 14:00:00"); [ClassInitialize] public static void Initialized(TestContext testContext) { + var connection = new ClickHouseConnection(TestUtils.ConnectionString); + Common.InitTable(false, connection); + Common.InitTable(true, connection); var services = new ServiceCollection(); services.AddLogging(builder => builder.AddConsole()); services.AddMASAStackApmClickhouse(TestUtils.ConnectionString, "custom_log", "custom_trace"); _APMService = services.BuildServiceProvider().GetRequiredService(); + Common.InitTableJsonData(false, AppDomain.CurrentDomain.BaseDirectory, connection); } [TestMethod] @@ -24,7 +28,7 @@ public async Task ServicePageAsync() var query = new ApmEndpointRequestDto { Start = _start, - End = DateTime.Now,// _start.AddHours(1), + End = _start.AddHours(1), ComparisonType = ComparisonTypes.DayBefore, StatusCodes = "401,402,503,500", PageSize = 10, @@ -42,7 +46,7 @@ public async Task EndpointPageAsync() var query = new ApmEndpointRequestDto { Start = _start, - End = DateTime.Now, + End = _start.AddHours(1), ComparisonType = ComparisonTypes.DayBefore, StatusCodes = "401,402,503,500", PageSize = 10, @@ -61,7 +65,7 @@ public async Task ChartDataAsync() var query = new ApmEndpointRequestDto { Start = _start, - End = DateTime.Now, + End = _start.AddHours(1), ComparisonType = ComparisonTypes.DayBefore, StatusCodes = "401,402,503,500", PageSize = 10, @@ -78,7 +82,7 @@ public async Task EndpointLatencyDistributionAsync() var query = new ApmEndpointRequestDto { Start = _start, - End = DateTime.Now, + End = _start.AddHours(1), ComparisonType = ComparisonTypes.DayBefore, StatusCodes = "401,402,503,500", PageSize = 10, @@ -96,7 +100,7 @@ public async Task ErrorMessagePageAsync() var query = new ApmEndpointRequestDto { Start = _start, - End = DateTime.Now, + End = _start.AddHours(1), ComparisonType = ComparisonTypes.DayBefore, StatusCodes = "401,402,503,500", PageSize = 10, @@ -113,7 +117,7 @@ public async Task TraceLatencyDetailAsync() var query = new ApmTraceLatencyRequestDto { Start = _start, - End = DateTime.Now, + End = _start.AddHours(1), ComparisonType = ComparisonTypes.DayBefore, StatusCodes = "401,402,503,500", Page = 1, diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Data/otel_trace_data.json b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Data/otel_trace_data.json new file mode 100644 index 000000000..0057f7a1f --- /dev/null +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Data/otel_trace_data.json @@ -0,0 +1,266 @@ +[ + { + "Timestamp" : "2024-01-03T22:02:12.828Z", + "TraceId" : "ecc21857f5032d057571b1230e8eb014", + "SpanId" : "3fe92b307ca25331", + "ParentSpanId" : "c819bede56087c3d", + "TraceState" : "", + "SpanName" : "HTTP GET", + "SpanKind" : "SPAN_KIND_CLIENT", + "ServiceName" : "tsc-service-iotdev", + "ResourceAttributes" : "{'service.namespace':'Development','service.version':'1.0-Prev','service.instance.id':'2efa4321-1f36-441a-bb1c-8eb25bc894dc','telemetry.sdk.name':'opentelemetry','telemetry.sdk.language':'dotnet','telemetry.sdk.version':'1.5.1','service.layer':'masastack','service.name':'tsc-service-iotdev'}", + "ScopeName" : "OpenTelemetry.Instrumentation.Http.HttpClient", + "ScopeVersion" : "1.0.0.0", + "SpanAttributes" : "{'http.scheme':'http','net.peer.name':'127.0.0.1','net.peer.port':'30090','http.url':'http:\/\/127.0.0.1:9090\/api\/v1\/query_range?end=1704348132&query=round(sum+(increase(http_server_duration_count%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2c1)&start=1704304932&step=172s','http.status_code':'200','peer.service':'127.0.0.1:9090','http.method':'GET','http.flavor':'1.1','host.name':'Lonsid-52','user_agent.original':''}", + "Duration" : 128014800, + "StatusCode" : "STATUS_CODE_UNSET", + "StatusMessage" : "", + "Events.Timestamp" : "[]", + "Events.Name" : "[]", + "Events.Attributes" : "[]", + "Links.TraceId" : "[]", + "Links.SpanId" : "[]", + "Links.TraceState" : "[]", + "Links.Attributes" : "[]" + }, + { + "Timestamp" : "2024-01-03T22:02:12.684Z", + "TraceId" : "ecc21857f5032d057571b1230e8eb014", + "SpanId" : "7e2d0530e1bf8106", + "ParentSpanId" : "c819bede56087c3d", + "TraceState" : "", + "SpanName" : "HTTP GET", + "SpanKind" : "SPAN_KIND_CLIENT", + "ServiceName" : "tsc-service-iotdev", + "ResourceAttributes" : "{'telemetry.sdk.version':'1.5.1','service.layer':'masastack','service.name':'tsc-service-iotdev','service.namespace':'Development','service.version':'1.0-Prev','service.instance.id':'2efa4321-1f36-441a-bb1c-8eb25bc894dc','telemetry.sdk.name':'opentelemetry','telemetry.sdk.language':'dotnet'}", + "ScopeName" : "OpenTelemetry.Instrumentation.Http.HttpClient", + "ScopeVersion" : "1.0.0.0", + "SpanAttributes" : "{'http.scheme':'http','http.method':'GET','net.peer.port':'30090','http.flavor':'1.1','host.name':'Lonsid-52','user_agent.original':'','http.status_code':'200','peer.service':'127.0.0.1:9090','net.peer.name':'127.0.0.1','http.url':'http:\/\/127.0.0.1:9090\/api\/v1\/label\/__name__\/values?lable=__name__'}", + "Duration" : 105716900, + "StatusCode" : "STATUS_CODE_UNSET", + "StatusMessage" : "", + "Events.Timestamp" : "[]", + "Events.Name" : "[]", + "Events.Attributes" : "[]", + "Links.TraceId" : "[]", + "Links.SpanId" : "[]", + "Links.TraceState" : "[]", + "Links.Attributes" : "[]" + }, + { + "Timestamp" : "2024-01-03T22:02:12.656Z", + "TraceId" : "ecc21857f5032d057571b1230e8eb014", + "SpanId" : "c819bede56087c3d", + "ParentSpanId" : "1ad3ed80e6cd7563", + "TraceState" : "", + "SpanName" : "\/api\/metric\/multi-range", + "SpanKind" : "SPAN_KIND_SERVER", + "ServiceName" : "tsc-service-iotdev", + "ResourceAttributes" : "{'telemetry.sdk.name':'opentelemetry','telemetry.sdk.language':'dotnet','telemetry.sdk.version':'1.5.1','service.layer':'masastack','service.name':'tsc-service-iotdev','service.namespace':'Development','service.version':'1.0-Prev','service.instance.id':'2efa4321-1f36-441a-bb1c-8eb25bc894dc'}", + "ScopeName" : "OpenTelemetry.Instrumentation.AspNetCore", + "ScopeVersion" : "1.0.0.0", + "SpanAttributes" : "{'http.client_ip':'','http.target':'\/api\/metric\/multi-range','http.request_content_length':'328','http.status_code':'200','http.method':'GET','http.url':'http:\/\/localhost:18010\/api\/metric\/multi-range','http.flavor':'HTTP\/1.1','http.request_content_type':'application\/json; charset=utf-8','authorization':'Bearer token','host.name':'Lonsid-52','http.response_content_type':'application\/json; charset=utf-8','net.host.port':'18010','http.scheme':'http','user_agent.original':'','http.request_content_body':'{\"Layer\": null,\"Service\": null,\"Instance\": null,\"EndPoint\": null,\"Start\":\"2024-01-03T18:02:12.1674699Z\",\"End\":\"2024-01-04T06:02:12.1674699Z\",\"Step\":\"172s\",\"MetricNames\":[\"round(sum (increase(http_server_duration_count[1m])),1)\",\"round(sum (increase(http_server_duration_sum[1m]))\/sum (increase(http_server_duration_count[1m])),1)\"]}','net.host.name':'localhost'}", + "Duration" : 490802200, + "StatusCode" : "STATUS_CODE_UNSET", + "StatusMessage" : "", + "Events.Timestamp" : "[]", + "Events.Name" : "[]", + "Events.Attributes" : "[]", + "Links.TraceId" : "[]", + "Links.SpanId" : "[]", + "Links.TraceState" : "[]", + "Links.Attributes" : "[]" + }, + { + "Timestamp" : "2024-01-03T22:02:12.391Z", + "TraceId" : "bfad12923f64935cec97f5e3dd5b9a47", + "SpanId" : "f7d9f646dbfc500d", + "ParentSpanId" : "", + "TraceState" : "", + "SpanName" : "tsc_iotdev", + "SpanKind" : "SPAN_KIND_CLIENT", + "ServiceName" : "tsc-service-iotdev", + "ResourceAttributes" : "{'telemetry.sdk.language':'dotnet','telemetry.sdk.version':'1.5.1','service.layer':'masastack','service.name':'tsc-service-iotdev','service.namespace':'Development','service.version':'1.0-Prev','service.instance.id':'2efa4321-1f36-441a-bb1c-8eb25bc894dc','telemetry.sdk.name':'opentelemetry'}", + "ScopeName" : "OpenTelemetry.Instrumentation.EntityFrameworkCore", + "ScopeVersion" : "1.0.0.7", + "SpanAttributes" : "{'peer.service':'127.0.0.1,1433','db.statement_type':'Text','db.statement':'SELECT TOP(@ __p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId]\\r\\nFROM [tsc].[IntegrationEventLog] AS [i]\\r\\nWHERE [i].[State] = 0\\r\\nORDER BY [i].[CreationTime]','db.system':'mssql','db.name':'tsc_iotdev'}", + "Duration" : 145378900, + "StatusCode" : "STATUS_CODE_UNSET", + "StatusMessage" : "", + "Events.Timestamp" : "[]", + "Events.Name" : "[]", + "Events.Attributes" : "[]", + "Links.TraceId" : "[]", + "Links.SpanId" : "[]", + "Links.TraceState" : "[]", + "Links.Attributes" : "[]" + }, + { + "Timestamp" : "2024-01-03T22:02:12.372Z", + "TraceId" : "bca3e932f72628858f040df98172fa09", + "SpanId" : "e7591443dde17f6e", + "ParentSpanId" : "2700571f3f7589d4", + "TraceState" : "", + "SpanName" : "\/api\/trace\/list", + "SpanKind" : "SPAN_KIND_SERVER", + "ServiceName" : "tsc-service-iotdev", + "ResourceAttributes" : "{'telemetry.sdk.language':'dotnet','telemetry.sdk.version':'1.5.1','service.layer':'masastack','service.name':'tsc-service-iotdev','service.namespace':'Development','service.version':'1.0-Prev','service.instance.id':'2efa4321-1f36-441a-bb1c-8eb25bc894dc','telemetry.sdk.name':'opentelemetry'}", + "ScopeName" : "OpenTelemetry.Instrumentation.AspNetCore", + "ScopeVersion" : "1.0.0.0", + "SpanAttributes" : "{'net.host.name':'localhost','authorization':'Bearer token','user_agent.original':'','http.status_code':'200','net.host.port':'18010','http.url':'http:\/\/localhost:18010\/api\/trace\/list?Start=2024\/1\/3%2018:02:12&End=2024\/1\/4%206:02:12&IsDesc=True&IsError=False&Page=1&PageSize=10','http.client_ip':'','host.name':'Lonsid-52','http.scheme':'http','http.target':'\/api\/trace\/list','http.method':'GET','http.flavor':'HTTP\/1.1','http.response_content_type':'application\/json; charset=utf-8'}", + "Duration" : 243969100, + "StatusCode" : "STATUS_CODE_UNSET", + "StatusMessage" : "", + "Events.Timestamp" : "[]", + "Events.Name" : "[]", + "Events.Attributes" : "[]", + "Links.TraceId" : "[]", + "Links.SpanId" : "[]", + "Links.TraceState" : "[]", + "Links.Attributes" : "[]" + }, + { + "Timestamp" : "2024-01-03T22:02:12.186Z", + "TraceId" : "15a58db826237e3c2f9ff872c10f1e27", + "SpanId" : "55fdbbb0b5755535", + "ParentSpanId" : "28acb93474b6ddf7", + "TraceState" : "", + "SpanName" : "\/api\/trace\/attr-values", + "SpanKind" : "SPAN_KIND_SERVER", + "ServiceName" : "tsc-service-iotdev", + "ResourceAttributes" : "{'service.instance.id':'2efa4321-1f36-441a-bb1c-8eb25bc894dc','telemetry.sdk.name':'opentelemetry','telemetry.sdk.language':'dotnet','telemetry.sdk.version':'1.5.1','service.layer':'masastack','service.name':'tsc-service-iotdev','service.namespace':'Development','service.version':'1.0-Prev'}", + "ScopeName" : "OpenTelemetry.Instrumentation.AspNetCore", + "ScopeVersion" : "1.0.0.0", + "SpanAttributes" : "{'http.request_content_body':'{\"Name\": null,\"Alias\": null,\"Type\":6,\"MaxCount\":1000,\"Interval\": null,\"AllValue\": false,\"TraceId\": null,\"Service\":\"\",\"Instance\": null,\"Endpoint\": null,\"Keyword\": null,\"Start\":\"2024-01-03T18:02:12.1674699Z\",\"End\":\"2024-01-04T06:02:12.1674699Z\",\"RawQuery\": null,\"Conditions\": null,\"Sort\": null,\"Page\":1,\"PageSize\":20}','net.host.name':'localhost','http.method':'GET','http.target':'\/api\/trace\/attr-values','http.flavor':'HTTP\/1.1','user_agent.original':'','http.url':'http:\/\/localhost:18010\/api\/trace\/attr-values','http.request_content_type':'application\/json; charset=utf-8','host.name':'Lonsid-52','net.host.port':'18010','http.scheme':'http','authorization':'Bearer token','http.client_ip':'','http.request_content_length':'304','http.status_code':'200','http.response_content_type':'application\/json; charset=utf-8'}", + "Duration" : 126784100, + "StatusCode" : "STATUS_CODE_UNSET", + "StatusMessage" : "", + "Events.Timestamp" : "[]", + "Events.Name" : "[]", + "Events.Attributes" : "[]", + "Links.TraceId" : "[]", + "Links.SpanId" : "[]", + "Links.TraceState" : "[]", + "Links.Attributes" : "[]" + }, + { + "Timestamp" : "2024-01-03T22:02:12.089Z", + "TraceId" : "135eada26529589440f22dd55b6a6975", + "SpanId" : "14864123964f693f", + "ParentSpanId" : "18ead6cf4f8b0d1d", + "TraceState" : "", + "SpanName" : "\/api\/trace\/errorStatus", + "SpanKind" : "SPAN_KIND_SERVER", + "ServiceName" : "tsc-service-iotdev", + "ResourceAttributes" : "{'telemetry.sdk.name':'opentelemetry','telemetry.sdk.language':'dotnet','telemetry.sdk.version':'1.5.1','service.layer':'masastack','service.name':'tsc-service-iotdev','service.namespace':'Development','service.version':'1.0-Prev','service.instance.id':'2efa4321-1f36-441a-bb1c-8eb25bc894dc'}", + "ScopeName" : "OpenTelemetry.Instrumentation.AspNetCore", + "ScopeVersion" : "1.0.0.0", + "SpanAttributes" : "{'net.host.name':'localhost','http.target':'\/api\/trace\/errorStatus','user_agent.original':'','authorization':'Bearer token','http.client_ip':'','host.name':'Lonsid-52','http.status_code':'200','http.method':'GET','http.scheme':'http','http.url':'http:\/\/localhost:18010\/api\/trace\/errorStatus','net.host.port':'18010','http.flavor':'HTTP\/1.1','http.response_content_type':'application\/json; charset=utf-8'}", + "Duration" : 2004400, + "StatusCode" : "STATUS_CODE_UNSET", + "StatusMessage" : "", + "Events.Timestamp" : "[]", + "Events.Name" : "[]", + "Events.Attributes" : "[]", + "Links.TraceId" : "[]", + "Links.SpanId" : "[]", + "Links.TraceState" : "[]", + "Links.Attributes" : "[]" + }, + { + "Timestamp" : "2024-01-03T22:02:11.273Z", + "TraceId" : "5f0982d9ca0972fe0c90e87376477ec8", + "SpanId" : "f7f20f341b595a6c", + "ParentSpanId" : "", + "TraceState" : "", + "SpanName" : "tsc_iotdev", + "SpanKind" : "SPAN_KIND_CLIENT", + "ServiceName" : "tsc-service-iotdev", + "ResourceAttributes" : "{'service.name':'tsc-service-iotdev','service.namespace':'Development','service.version':'1.0-Prev','service.instance.id':'2efa4321-1f36-441a-bb1c-8eb25bc894dc','telemetry.sdk.name':'opentelemetry','telemetry.sdk.language':'dotnet','telemetry.sdk.version':'1.5.1','service.layer':'masastack'}", + "ScopeName" : "OpenTelemetry.Instrumentation.EntityFrameworkCore", + "ScopeVersion" : "1.0.0.7", + "SpanAttributes" : "{'db.system':'mssql','db.name':'tsc_iotdev','peer.service':'127.0.0.1,1433','db.statement_type':'Text','db.statement':'SELECT TOP(@ __p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId]\\r\\nFROM [tsc].[IntegrationEventLog] AS [i]\\r\\nWHERE [i].[State] = 0\\r\\nORDER BY [i].[CreationTime]'}", + "Duration" : 98267900, + "StatusCode" : "STATUS_CODE_UNSET", + "StatusMessage" : "", + "Events.Timestamp" : "[]", + "Events.Name" : "[]", + "Events.Attributes" : "[]", + "Links.TraceId" : "[]", + "Links.SpanId" : "[]", + "Links.TraceState" : "[]", + "Links.Attributes" : "[]" + }, + { + "Timestamp" : "2024-01-03T22:02:10.179Z", + "TraceId" : "6b0fde523c65d18862f50ed99aa6f85e", + "SpanId" : "6ad9e25caa8f4f28", + "ParentSpanId" : "", + "TraceState" : "", + "SpanName" : "tsc_iotdev", + "SpanKind" : "SPAN_KIND_CLIENT", + "ServiceName" : "tsc-service-iotdev", + "ResourceAttributes" : "{'service.namespace':'Development','service.version':'1.0-Prev','service.instance.id':'2efa4321-1f36-441a-bb1c-8eb25bc894dc','telemetry.sdk.name':'opentelemetry','telemetry.sdk.language':'dotnet','telemetry.sdk.version':'1.5.1','service.layer':'masastack','service.name':'tsc-service-iotdev'}", + "ScopeName" : "OpenTelemetry.Instrumentation.EntityFrameworkCore", + "ScopeVersion" : "1.0.0.7", + "SpanAttributes" : "{'db.name':'tsc_iotdev','peer.service':'127.0.0.1,1433','db.statement_type':'Text','db.statement':'SELECT TOP(@ __p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId]\\r\\nFROM [tsc].[IntegrationEventLog] AS [i]\\r\\nWHERE [i].[State] = 0\\r\\nORDER BY [i].[CreationTime]','db.system':'mssql'}", + "Duration" : 91859100, + "StatusCode" : "STATUS_CODE_UNSET", + "StatusMessage" : "", + "Events.Timestamp" : "[]", + "Events.Name" : "[]", + "Events.Attributes" : "[]", + "Links.TraceId" : "[]", + "Links.SpanId" : "[]", + "Links.TraceState" : "[]", + "Links.Attributes" : "[]" + }, + { + "Timestamp" : "2024-01-03T22:02:09.654Z", + "TraceId" : "87a81d697be817ff9e48f7bc80f57301", + "SpanId" : "65f6fdf66547dfad", + "ParentSpanId" : "eb3a1ae54e324251", + "TraceState" : "", + "SpanName" : "\/api\/log\/list", + "SpanKind" : "SPAN_KIND_SERVER", + "ServiceName" : "tsc-service-iotdev", + "ResourceAttributes" : "{'service.namespace':'Development','service.version':'1.0-Prev','service.instance.id':'2efa4321-1f36-441a-bb1c-8eb25bc894dc','telemetry.sdk.name':'opentelemetry','telemetry.sdk.language':'dotnet','telemetry.sdk.version':'1.5.1','service.layer':'masastack','service.name':'tsc-service-iotdev'}", + "ScopeName" : "OpenTelemetry.Instrumentation.AspNetCore", + "ScopeVersion" : "1.0.0.0", + "SpanAttributes" : "{'user_agent.original':'','net.host.port':'18010','http.target':'\/api\/log\/list','http.url':'http:\/\/localhost:18010\/api\/log\/list?Query=a=\\'test\\'&Start=2024\/1\/3%2014:47:23&End=2024\/1\/4%202:47:23&IsDesc=True&Page=1&PageSize=10','authorization':'Bearer token','host.name':'Lonsid-52','http.scheme':'http','http.method':'GET','http.client_ip':'','http.status_code':'500','net.host.name':'localhost','http.response_content_type':'text\/plain; charset=utf-8','http.flavor':'HTTP\/1.1'}", + "Duration" : 283681400, + "StatusCode" : "STATUS_CODE_ERROR", + "StatusMessage" : "", + "Events.Timestamp" : "[]", + "Events.Name" : "[]", + "Events.Attributes" : "[]", + "Links.TraceId" : "[]", + "Links.SpanId" : "[]", + "Links.TraceState" : "[]", + "Links.Attributes" : "[]" + }, + { + "Timestamp" : "2024-01-03T22:02:09.630Z", + "TraceId" : "d6ad209686047701802cbe32b8f3345e", + "SpanId" : "b243c61dabdd4676", + "ParentSpanId" : "630bb5b95c34b166", + "TraceState" : "", + "SpanName" : "\/api\/log\/list", + "SpanKind" : "SPAN_KIND_SERVER", + "ServiceName" : "tsc-service-iotdev", + "ResourceAttributes" : "{'service.version':'1.0-Prev','service.instance.id':'2efa4321-1f36-441a-bb1c-8eb25bc894dc','telemetry.sdk.name':'opentelemetry','telemetry.sdk.language':'dotnet','telemetry.sdk.version':'1.5.1','service.layer':'masastack','service.name':'tsc-service-iotdev','service.namespace':'Development'}", + "ScopeName" : "OpenTelemetry.Instrumentation.AspNetCore", + "ScopeVersion" : "1.0.0.0", + "SpanAttributes" : "{'http.response_content_type':'text\/plain; charset=utf-8','net.host.name':'localhost','http.scheme':'http','http.target':'\/api\/log\/list','http.status_code':'500','authorization':'Bearer token','host.name':'Lonsid-52','net.host.port':'18010','http.method':'GET','http.flavor':'HTTP\/1.1','http.url':'http:\/\/localhost:18010\/api\/log\/list?Query=a=\\'test\\'&Start=2024\/1\/3%2014:47:23&End=2024\/1\/4%202:47:23&IsDesc=True&Page=1&PageSize=10','user_agent.original':'','http.client_ip':''}", + "Duration" : 309668700, + "StatusCode" : "STATUS_CODE_ERROR", + "StatusMessage" : "", + "Events.Timestamp" : "[]", + "Events.Name" : "[]", + "Events.Attributes" : "[]", + "Links.TraceId" : "[]", + "Links.SpanId" : "[]", + "Links.TraceState" : "[]", + "Links.Attributes" : "[]" + } +] \ No newline at end of file diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Data/otel_trace_data.txt b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Data/otel_trace_data.txt deleted file mode 100644 index 294dce286..000000000 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Data/otel_trace_data.txt +++ /dev/null @@ -1,50 +0,0 @@ - INSERT INTO otel_traces (`Timestamp`, TraceId, SpanId, ParentSpanId, TraceState, SpanName, SpanKind, ServiceName, ResourceAttributes, ScopeName, ScopeVersion, SpanAttributes, Duration, StatusCode, StatusMessage, `Events.Timestamp`, `Events.Name`, `Events.Attributes`, `Links.TraceId`, `Links.SpanId`, `Links.TraceState`, `Links.Attributes`) -VALUES - -2024-01-04 06:02:12.982357700 ecc21857f5032d057571b1230e8eb014 1dd1edd9a64dccd3 c819bede56087c3d HTTP GET SPAN_KIND_CLIENT tsc-service-iotdev 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev Development OpenTelemetry.Instrumentation.Http.HttpClient 1.0.0.0 http GET 10.130.0.73 http://10.130.0.73:30090/api/v1/query_range?end=1704348132&query=round(sum+(increase(http_server_duration_sum%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2fsum+(increase(http_server_duration_count%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2c1)&start=1704304932&step=172s http GET 10.130.0.73 http://10.130.0.73:30090/api/v1/query_range?end=1704348132&query=round(sum+(increase(http_server_duration_sum%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2fsum+(increase(http_server_duration_count%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2c1)&start=1704304932&step=172s 10.130.0.73:30090 30090 1.1 Lonsid-52 200 http://10.130.0.73:30090/api/v1/query_range?end=1704348132&query=round(sum+(increase(http_server_duration_sum%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2fsum+(increase(http_server_duration_count%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2c1)&start=1704304932&step=172s 30090 10.130.0.73 30090 150848800 STATUS_CODE_UNSET [] [] [] [] [] [] [] -2024-01-04 06:02:12.828538400 ecc21857f5032d057571b1230e8eb014 3fe92b307ca25331 c819bede56087c3d HTTP GET SPAN_KIND_CLIENT tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev OpenTelemetry.Instrumentation.Http.HttpClient 1.0.0.0 http 10.130.0.73 30090 http://10.130.0.73:30090/api/v1/query_range?end=1704348132&query=round(sum+(increase(http_server_duration_count%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2c1)&start=1704304932&step=172s 200 http 10.130.0.73 30090 http://10.130.0.73:30090/api/v1/query_range?end=1704348132&query=round(sum+(increase(http_server_duration_count%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2c1)&start=1704304932&step=172s 200 10.130.0.73:30090 GET 1.1 Lonsid-52 200 GET http://10.130.0.73:30090/api/v1/query_range?end=1704348132&query=round(sum+(increase(http_server_duration_count%7bservice_namespace%3d%22Development%22%2c%7d%5b1m%5d))%2c1)&start=1704304932&step=172s GET 128014800 STATUS_CODE_UNSET [] [] [] [] [] [] [] -2024-01-04 06:02:12.684341700 ecc21857f5032d057571b1230e8eb014 7e2d0530e1bf8106 c819bede56087c3d HTTP GET SPAN_KIND_CLIENT tsc-service-iotdev 1.5.1 masastack tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet OpenTelemetry.Instrumentation.Http.HttpClient 1.0.0.0 http GET 30090 1.1 Lonsid-52 http GET 30090 1.1 Lonsid-52 200 10.130.0.73:30090 10.130.0.73 http://10.130.0.73:30090/api/v1/label/__name__/values?lable=__name__ Lonsid-52 200 1.1 200 105716900 STATUS_CODE_UNSET [] [] [] [] [] [] [] -2024-01-04 06:02:12.656875700 ecc21857f5032d057571b1230e8eb014 c819bede56087c3d 1ad3ed80e6cd7563 /api/metric/multi-range SPAN_KIND_SERVER tsc-service-iotdev opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc OpenTelemetry.Instrumentation.AspNetCore 1.0.0.0 /api/metric/multi-range 328 200 GET /api/metric/multi-range 328 200 GET http://localhost:18010/api/metric/multi-range HTTP/1.1 application/json; charset=utf-8 Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ Lonsid-52 application/json; charset=utf-8 18010 http GET HTTP/1.1 {"Layer":null,"Service":null,"Instance":null,"EndPoint":null,"Start":"2024-01-03T18:02:12.1674699Z","End":"2024-01-04T06:02:12.1674699Z","Step":"172s","MetricNames":["round(sum (increase(http_server_duration_count[1m])),1)","round(sum (increase(http_server_duration_sum[1m]))/sum (increase(http_server_duration_count[1m])),1)"]} 200 HTTP/1.1 490802200 STATUS_CODE_UNSET [] [] [] [] [] [] [] -2024-01-04 06:02:12.391519700 bfad12923f64935cec97f5e3dd5b9a47 f7d9f646dbfc500d tsc_iotdev SPAN_KIND_CLIENT tsc-service-iotdev dotnet 1.5.1 masastack tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry OpenTelemetry.Instrumentation.EntityFrameworkCore 1.0.0.7 10.130.0.19,2959 Text "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] -FROM [tsc].[IntegrationEventLog] AS [i] -WHERE [i].[State] = 0 -ORDER BY [i].[CreationTime]" mssql tsc_iotdev 10.130.0.19,2959 Text "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] -FROM [tsc].[IntegrationEventLog] AS [i] -WHERE [i].[State] = 0 -ORDER BY [i].[CreationTime]" mssql tsc_iotdev tsc_iotdev mssql 145378900 STATUS_CODE_UNSET [] [] [] [] [] [] [] -2024-01-04 06:02:12.372382800 bca3e932f72628858f040df98172fa09 e7591443dde17f6e 2700571f3f7589d4 /api/trace/list SPAN_KIND_SERVER tsc-service-iotdev dotnet 1.5.1 masastack tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry OpenTelemetry.Instrumentation.AspNetCore 1.0.0.0 localhost Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ 200 18010 localhost Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ 200 18010 http://localhost:18010/api/trace/list?Start=2024/1/3%2018:02:12&End=2024/1/4%206:02:12&IsDesc=True&IsError=False&Page=1&PageSize=10 Lonsid-52 http /api/trace/list GET HTTP/1.1 application/json; charset=utf-8 18010 200 243969100 STATUS_CODE_UNSET [] [] [] [] [] [] [] -2024-01-04 06:02:12.186883900 15a58db826237e3c2f9ff872c10f1e27 55fdbbb0b5755535 28acb93474b6ddf7 /api/trace/attr-values SPAN_KIND_SERVER tsc-service-iotdev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev Development 1.0-Prev OpenTelemetry.Instrumentation.AspNetCore 1.0.0.0 {"Name":null,"Alias":null,"Type":6,"MaxCount":1000,"Interval":null,"AllValue":false,"TraceId":null,"Service":"","Instance":null,"Endpoint":null,"Keyword":null,"Start":"2024-01-03T18:02:12.1674699Z","End":"2024-01-04T06:02:12.1674699Z","RawQuery":null,"Conditions":null,"Sort":null,"Page":1,"PageSize":20} localhost GET /api/trace/attr-values HTTP/1.1 {"Name":null,"Alias":null,"Type":6,"MaxCount":1000,"Interval":null,"AllValue":false,"TraceId":null,"Service":"","Instance":null,"Endpoint":null,"Keyword":null,"Start":"2024-01-03T18:02:12.1674699Z","End":"2024-01-04T06:02:12.1674699Z","RawQuery":null,"Conditions":null,"Sort":null,"Page":1,"PageSize":20} localhost GET /api/trace/attr-values HTTP/1.1 http://localhost:18010/api/trace/attr-values application/json; charset=utf-8 Lonsid-52 18010 http Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ HTTP/1.1 http://localhost:18010/api/trace/attr-values 200 /api/trace/attr-values http://localhost:18010/api/trace/attr-values 126784100 STATUS_CODE_UNSET [] [] [] [] [] [] [] -2024-01-04 06:02:12.089789900 135eada26529589440f22dd55b6a6975 14864123964f693f 18ead6cf4f8b0d1d /api/trace/errorStatus SPAN_KIND_SERVER tsc-service-iotdev opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc OpenTelemetry.Instrumentation.AspNetCore 1.0.0.0 localhost /api/trace/errorStatus Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ localhost /api/trace/errorStatus Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ Lonsid-52 200 GET http http://localhost:18010/api/trace/errorStatus 18010 HTTP/1.1 application/json; charset=utf-8 200 Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ 200 2004400 STATUS_CODE_UNSET [] [] [] [] [] [] [] -2024-01-04 06:02:11.273064100 5f0982d9ca0972fe0c90e87376477ec8 f7f20f341b595a6c tsc_iotdev SPAN_KIND_CLIENT tsc-service-iotdev tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet 1.5.1 masastack OpenTelemetry.Instrumentation.EntityFrameworkCore 1.0.0.7 mssql tsc_iotdev 10.130.0.19,2959 Text "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] -FROM [tsc].[IntegrationEventLog] AS [i] -WHERE [i].[State] = 0 -ORDER BY [i].[CreationTime]" mssql tsc_iotdev 10.130.0.19,2959 Text "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] -FROM [tsc].[IntegrationEventLog] AS [i] -WHERE [i].[State] = 0 -ORDER BY [i].[CreationTime]" "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] -FROM [tsc].[IntegrationEventLog] AS [i] -WHERE [i].[State] = 0 -ORDER BY [i].[CreationTime]" Text 98267900 STATUS_CODE_UNSET [] [] [] [] [] [] [] -2024-01-04 06:02:10.179119500 6b0fde523c65d18862f50ed99aa6f85e 6ad9e25caa8f4f28 tsc_iotdev SPAN_KIND_CLIENT tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev OpenTelemetry.Instrumentation.EntityFrameworkCore 1.0.0.7 tsc_iotdev 10.130.0.19,2959 Text "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] -FROM [tsc].[IntegrationEventLog] AS [i] -WHERE [i].[State] = 0 -ORDER BY [i].[CreationTime]" mssql tsc_iotdev 10.130.0.19,2959 Text "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] -FROM [tsc].[IntegrationEventLog] AS [i] -WHERE [i].[State] = 0 -ORDER BY [i].[CreationTime]" mssql mssql "SELECT TOP(@__p_0) [i].[Id], [i].[Content], [i].[CreationTime], [i].[EventId], [i].[EventTypeName], [i].[ExpandContent], [i].[ModificationTime], [i].[RowVersion], [i].[State], [i].[TimesSent], [i].[TransactionId] -FROM [tsc].[IntegrationEventLog] AS [i] -WHERE [i].[State] = 0 -ORDER BY [i].[CreationTime]" 91859100 STATUS_CODE_UNSET [] [] [] [] [] [] [] -2024-01-04 06:02:09.654368700 87a81d697be817ff9e48f7bc80f57301 65f6fdf66547dfad eb3a1ae54e324251 /api/log/list SPAN_KIND_SERVER tsc-service-iotdev Development 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev OpenTelemetry.Instrumentation.AspNetCore 1.0.0.0 18010 /api/log/list http://localhost:18010/api/log/list?Query=a='test'&Start=2024/1/3%2014:47:23&End=2024/1/4%202:47:23&IsDesc=True&Page=1&PageSize=10 Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ 18010 /api/log/list http://localhost:18010/api/log/list?Query=a='test'&Start=2024/1/3%2014:47:23&End=2024/1/4%202:47:23&IsDesc=True&Page=1&PageSize=10 Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ Lonsid-52 http GET 500 localhost text/plain; charset=utf-8 HTTP/1.1 Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ http http://localhost:18010/api/log/list?Query=a='test'&Start=2024/1/3%2014:47:23&End=2024/1/4%202:47:23&IsDesc=True&Page=1&PageSize=10 http 283681400 STATUS_CODE_ERROR [] [] [] [] [] [] [] -2024-01-04 06:02:09.630141400 d6ad209686047701802cbe32b8f3345e b243c61dabdd4676 630bb5b95c34b166 /api/log/list SPAN_KIND_SERVER tsc-service-iotdev 1.0-Prev 2efa4321-1f36-441a-bb1c-8eb25bc894dc opentelemetry dotnet 1.5.1 masastack tsc-service-iotdev Development OpenTelemetry.Instrumentation.AspNetCore 1.0.0.0 text/plain; charset=utf-8 localhost http /api/log/list 500 text/plain; charset=utf-8 localhost http /api/log/list 500 Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkM5NjREMUVFQTkzQ0I3Q0FFNTAzMjlCOTRCMUI1ODNGMERCRTkzREFSUzI1NiIsInR5cCI6ImF0K2p3dCIsIng1dCI6InlXVFI3cWs4dDhybEF5bTVTeHRZUHcyLWs5byJ9.eyJuYmYiOjE3MDQzMzQzMjIsImV4cCI6MTcwNDMzNzkyMiwiaXNzIjoiaHR0cHM6Ly9hdXRoLXNzby1kZXYubWFzYXN0YWNrLmNvbSIsImNsaWVudF9pZCI6InRzYy13ZWItZGV2Iiwic3ViIjoiZGZiYjRjMzEtMzg2ZS00ZGU2LWQwYWUtMDhkYmFkZDVlNGU0IiwiYXV0aF90aW1lIjoxNzAxNzYyNTk4LCJpZHAiOiJsb2NhbCIsIm5hbWUiOiJBZG1pbmlzdHJhdG9yIiwiYWNjb3VudCI6ImFkbWluIiwiZW52aXJvbm1lbnQiOiJEZXZlbG9wIiwicm9sZXMiOiJbXSIsImN1cnJlbnRfdGVhbSI6ImJjZDFhNWQxLTE5YjUtZjllNS0zNDI0LWZkNDk0OGY3ZjVhNyIsInN0YWZmX2lkIjoiMmJhYjFmNmMtOTNjMy00NzRiLTFkYjktMDhkYmFkZDVlNGVmIiwicGhvbmVfbnVtYmVyIjoiMTU4ODg4ODg4ODgiLCJlbWFpbCI6ImFkbWluQG1hc2FzdGFjay5jb20iLCJwaG9uZU51bWJlciI6IjE1ODg4ODg4ODg4IiwidXNlck5hbWUiOiJBZG1pbmlzdHJhdG9yIiwic2lkIjoiMEFEMzk2QjM5NzM4RDdGMzI1RTc2RTQyREQxOUEzMDEiLCJpYXQiOjE3MDQzMzQzMjIsInNjb3BlIjpbIm9wZW5pZCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJhbXIiOlsicHdkIl19.LmjfA-oMlu7ApwuNv1Tr8EeS8anrWMlhj18MRBirC0Ol_x3j6JjG5f0UZEBaK52JLwv0eEv3BH5i6lYGUFTaUGjFSlHlX0U5Ip74tLq6ALbMM8tIRvUmSvVE5OD27upQ0EdaVR9PriMFCzVX228k31QNqWmIOT9dLW6UQkyzawWHF-sPL80F4_7ejstBM6jYxyb6ptucIuhFXMvoR42BNg3anC2Y-d2QLyKho6n-f2gc004rqpQ-f4zJZZIV-umxAKSACAMLwh-N3dC0hJcclP5Y8HdQZ_ekkDZdPgAF2hXUSSk61ZUHKObd2HA9rxYDvlE5l2p0lFaOW1XAgVNGoQ Lonsid-52 18010 GET HTTP/1.1 http://localhost:18010/api/log/list?Query=a='test'&Start=2024/1/3%2014:47:23&End=2024/1/4%202:47:23&IsDesc=True&Page=1&PageSize=10 500 Lonsid-52 /api/log/list Lonsid-52 309668700 STATUS_CODE_ERROR [] [] [] [] [] [] [] - - - - - - - - -('2023-11-02 09:39:30.865', '2821e37c871ee22bcc10b43ddd7ca657', '402f0ed4040433e6', '1c9a707a29a9e449', '', 'HTTP POST', 'SPAN_KIND_CLIENT', 'service', '{''service.instance.id'':''c56534c5-2f62-48e8-82a5-6d8ecdbdd7de'',''telemetry.sdk.name'':''opentelemetry'',''telemetry.sdk.language'':''dotnet'',''telemetry.sdk.version'':''1.5.1'',''service.layer'':''service'',''service.name'':''service'',''service.namespace'':''Test'',''service.version'':''1.0.0''}', 'OpenTelemetry.Instrumentation.Http.HttpClient', '1.0.0.0', '{''http.flavor'':''1.1'',''host.name'':''service-device-676656994-xbnnq'',''http.request_content_body'':''{"deviceInfoId":"5EF36830-438D-4B7F-BA15-AF3A138412BA","activeTime":"2022-03-26T03:15:56.878289","active1":"","active2":"","active3":"","active4":"","active5":"","life1":"","life2":"","life3":"","life4":"","life5":"","maxLife1":"","maxLife2":"","maxLife3":"","maxLife4":"","maxLife5":"","lock":""}'',''peer.service'':''127.0.0.1:3500'',''http.scheme'':''http'',''http.method'':''POST'',''net.peer.name'':''127.0.0.1'',''net.peer.port'':''3500'',''http.url'':''http://127.0.0.1:3500/v1.0/invoke/service/method/api/filter/ActivateDeviceFilter'',''http.status_code'':''200''}', 4828300, 'STATUS_CODE_UNSET', '', [], [], [], [], [], [], []), -('2023-11-02 09:39:30.865', '2821e37c871ee22bcc10b43ddd7ca657', '402f0ed4040433e6', '1c9a707a29a9e449', '', 'HTTP POST', 'SPAN_KIND_CLIENT', 'service', '{''service.instance.id'':''c56534c5-2f62-48e8-82a5-6d8ecdbdd7de'',''telemetry.sdk.name'':''opentelemetry'',''telemetry.sdk.language'':''dotnet'',''telemetry.sdk.version'':''1.5.1'',''service.layer'':''service'',''service.name'':''service'',''service.namespace'':''Test'',''service.version'':''1.0.0''}', 'OpenTelemetry.Instrumentation.Http.HttpClient', '1.0.0.0', '{''http.flavor'':''1.1'',''host.name'':''service-device-676656994-xbnnq'',''http.request_content_body'':''{"deviceInfoId":"C829B9EE-E911-44F8-8E74-1EF991DA6803","activeTime":"2022-03-26T03:15:56.878289","active1":"","active2":"","active3":"","active4":"","active5":"","life1":"","life2":"","life3":"","life4":"","life5":"","maxLife1":"","maxLife2":"","maxLife3":"","maxLife4":"","maxLife5":"","lock":""}'',''peer.service'':''127.0.0.1:3500'',''http.scheme'':''http'',''http.method'':''POST'',''net.peer.name'':''127.0.0.1'',''net.peer.port'':''3500'',''http.url'':''http://127.0.0.1:3500/v1.0/invoke/service/method/api/filter/ActivateDeviceFilter'',''http.status_code'':''200''}', 4828300, 'STATUS_CODE_UNSET', '', [], [], [], [], [], [], []), -('2023-11-02 09:39:30.865', '2821e37c871ee22bcc10b43ddd7ca657', '402f0ed4040433e6', '1c9a707a29a9e449', '', 'HTTP POST', 'SPAN_KIND_CLIENT', 'service', '{''service.instance.id'':''c56534c5-2f62-48e8-82a5-6d8ecdbdd7de'',''telemetry.sdk.name'':''opentelemetry'',''telemetry.sdk.language'':''dotnet'',''telemetry.sdk.version'':''1.5.1'',''service.layer'':''service'',''service.name'':''service'',''service.namespace'':''Test'',''service.version'':''1.0.0''}', 'OpenTelemetry.Instrumentation.Http.HttpClient', '1.0.0.0', '{''http.flavor'':''1.1'',''host.name'':''service-device-676656994-xbnnq'',''http.request_content_body'':''{"deviceInfoId":"8995f797-0385-4b27-9452-181eb0bb4079","activeTime":"2022-03-26T03:15:56.878289","active1":"","active2":"","active3":"","active4":"","active5":"","life1":"","life2":"","life3":"","life4":"","life5":"","maxLife1":"","maxLife2":"","maxLife3":"","maxLife4":"","maxLife5":"","lock":""}'',''peer.service'':''127.0.0.1:3500'',''http.scheme'':''http'',''http.method'':''POST'',''net.peer.name'':''127.0.0.1'',''net.peer.port'':''3500'',''http.url'':''http://127.0.0.1:3500/v1.0/invoke/service/method/api/filter/ActivateDeviceFilter'',''http.status_code'':''200''}', 4828300, 'STATUS_CODE_UNSET', '', [], [], [], [], [], [], []) \ No newline at end of file diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/GlobalUsings.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/GlobalUsings.cs index 89422827d..2c5c354ed 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/GlobalUsings.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/GlobalUsings.cs @@ -1,8 +1,10 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. +global using ClickHouse.Ado; global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models; global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Request; +global using Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Logging; global using Microsoft.VisualStudio.TestTools.UnitTesting; diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj index 53b18cde4..b383df5e7 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj @@ -9,16 +9,6 @@ true - - - - - - - Always - - - @@ -28,6 +18,13 @@ + + + + + + Always + diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs index adfd4463b..0118a4bd5 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs @@ -5,6 +5,6 @@ namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests; internal class TestUtils { - //public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=localhost;Port=9000;Database=default;User=default"; - public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=10000;Host=192.168.51.234;Port=19003;User=test;Password=123456;Database=otel_test"; + public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=localhost;Port=9000;Database=default;User=default"; + //public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=10000;Host=192.168.51.234;Port=19003;User=test;Password=123456;Database=default"; } diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Common.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Common.cs index 72d0c4def..921021189 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Common.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Common.cs @@ -3,49 +3,87 @@ namespace Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests; -internal class Common +public class Common { - public static void InitTable(bool isLog) + public static void InitTable(bool isLog, IDbConnection connection) { var name = isLog ? "log" : "trace"; - using var connection = new ClickHouseConnection(Consts.ConnectionString); - connection.Open(); + if (connection.State == ConnectionState.Closed) + connection.Open(); using var cmd = connection.CreateCommand(); var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Data/otel_{name}.txt"); - using (var reader = new StreamReader(path)) - { - var sql = reader.ReadToEnd(); - cmd.CommandText = sql; - try - { - cmd.ExecuteNonQuery(); - } - catch - { - //table is exists - } - } + using var reader = new StreamReader(path); - path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Data/otel_{name}_data.txt"); - using (var dataReader = new StreamReader(path)) + var sql = reader.ReadToEnd(); + cmd.CommandText = sql; + try { - var sql = dataReader.ReadToEnd(); - cmd.CommandText = sql; cmd.ExecuteNonQuery(); } + catch + { + //table is exists + } + } - public static void InitTableData(bool isLog) + public static void InitTableData(bool isLog, string rootPath, IDbConnection connection) { var name = isLog ? "log" : "trace"; - using var connection = new ClickHouseConnection(Consts.ConnectionString); - connection.Open(); + if (connection.State == ConnectionState.Closed) + connection.Open(); using var cmd = connection.CreateCommand(); - var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Data/otel_{name}_data.txt"); + var path = Path.Combine(rootPath, $"Data/otel_{name}_data.txt"); using var dataReader = new StreamReader(path); var sql = dataReader.ReadToEnd(); cmd.CommandText = sql; cmd.ExecuteNonQuery(); + } + public static void InitTableJsonData(bool isLog, string rootPath, IDbConnection connection) + { + var name = isLog ? "log" : "trace"; + if (connection.State == ConnectionState.Closed) + connection.Open(); + using var cmd = connection.CreateCommand(); + var path = Path.Combine(rootPath, $"Data/otel_{name}_data.json"); + using var dataReader = new StreamReader(path); + var data = dataReader.ReadToEnd(); + var sql = GetInsertSql(data, $"otel_{name}s"); + cmd.CommandText = sql; + cmd.ExecuteNonQuery(); + } + + private static string? GetInsertSql(string jsonData, string table = "ttttttt") + { + var data = JsonSerializer.Deserialize(jsonData); + if (data == null || data.Length == 0) + return default; + + var header = new StringBuilder($"insert into {table}"); + header.Append('(').AppendLine(); + bool isFirst = true; + var sql = new StringBuilder(); + foreach (var jsonNode in data) + { + sql.Append('('); + foreach (var item in jsonNode.EnumerateObject()) + { + if (isFirst) + { + header.Append($"{item.Name},"); + } + if (item.Value.ValueKind == JsonValueKind.Number) + sql.Append($"{item.Value},"); + else if (item.Value.TryGetDateTime(out var time)) + sql.Append($"'{time.ToString("yyyy-MM-dd HH:mm:ss.ffff")}',"); + else + sql.Append($"'{item.Value.ToString().Replace("\\'", "''").Replace("'", "''").Replace("@","")}',"); + } + sql.Remove(sql.Length - 1, 1).AppendLine("),"); + isFirst = false; + } + header.Remove(header.Length - 1, 1).Append(')').AppendLine().Append("values").AppendLine().Append(sql.Remove(sql.Length - 1, 1)); + return header.ToString(); } } diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/LogServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/LogServiceTests.cs index 188235712..d22c1baae 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/LogServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/LogServiceTests.cs @@ -7,30 +7,31 @@ namespace Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests; public class LogServiceTests { private static ILogService logService; - private readonly DateTime startTime= DateTime.Parse("2023-11-02 09:00:00"); + private readonly DateTime startTime = DateTime.Parse("2023-11-02 09:00:00"); [ClassInitialize] public static void Initialized(TestContext testContext) { - Common.InitTable(true); - Common.InitTable(false); + var connection = new ClickHouseConnection(Consts.ConnectionString); + Common.InitTable(true, connection); + Common.InitTable(false, connection); var services = new ServiceCollection(); services.AddLogging(builder => builder.AddConsole()); services.AddMASAStackClickhouse(Consts.ConnectionString, "custom_log", "custom_trace"); - Common.InitTableData(true); + Common.InitTableData(true, AppDomain.CurrentDomain.BaseDirectory, connection); logService = services.BuildServiceProvider().GetRequiredService(); } [TestMethod] public async Task QueryListTest() - { + { var query = new BaseRequestDto { Page = 1, PageSize = 10, Start = startTime, End = startTime.AddHours(1), - Keyword="Kafka", + Keyword = "Kafka", Conditions = new List { new FieldConditionDto{ Name="Resource.service.name", @@ -67,7 +68,7 @@ public async Task MappingTest() [TestMethod] public async Task AggTest() - { + { var request = new SimpleAggregateRequestDto { Name = "Resource.service.name", diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/TraceServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/TraceServiceTests.cs index 85d1b2e53..f8276cccf 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/TraceServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/TraceServiceTests.cs @@ -13,15 +13,16 @@ public class TraceServiceTests public static void Initialized(TestContext testContext) { var services = new ServiceCollection(); - services.AddLogging(builder=>builder.AddConsole()); - services.AddMASAStackClickhouse(Consts.ConnectionString,"custom_log", "custom_trace"); - Common.InitTableData(false); + var connection = new ClickHouseConnection(Consts.ConnectionString); + services.AddLogging(builder => builder.AddConsole()); + services.AddMASAStackClickhouse(Consts.ConnectionString, "custom_log", "custom_trace"); + Common.InitTableData(false, AppDomain.CurrentDomain.BaseDirectory, connection); traceService = services.BuildServiceProvider().GetRequiredService(); } [TestMethod] public async Task QueryListTest() - { + { var query = new BaseRequestDto { Page = 1, diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/_Imports.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/_Imports.cs index 27a16da29..310c87f0a 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/_Imports.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/_Imports.cs @@ -8,3 +8,6 @@ global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Logging; global using Microsoft.VisualStudio.TestTools.UnitTesting; +global using System.Data; +global using System.Text; +global using System.Text.Json; From 5224cb8183d1ff382b7e22415bc0e129c14d0115 Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Fri, 5 Jan 2024 13:56:38 +0800 Subject: [PATCH 03/24] chore: update target frameworks --- .../Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj | 3 +-- .../Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj index 9df74467d..465eae53d 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj @@ -1,7 +1,6 @@ - - net6.0 + enable enable diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj index b383df5e7..b84fe471b 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests.csproj @@ -1,10 +1,8 @@  - - net6.0 + enable enable - false true From ce94736161b676832c30bf8d671fd558c81873b0 Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Fri, 5 Jan 2024 14:06:24 +0800 Subject: [PATCH 04/24] chore: update --- .../Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Consts.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Consts.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Consts.cs index 300063521..738be3cc6 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Consts.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Consts.cs @@ -5,6 +5,6 @@ namespace Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests; internal class Consts { - //public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=localhost;Port=9000;Database=default;User=default"; - public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=192.168.51.234;Port=19003;Database=default;User=default"; + public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=localhost;Port=9000;Database=default;User=default"; + //public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=192.168.51.234;Port=19003;Database=default;User=default"; } From 52a142c7705d56301850a436181d93c3634d4278 Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Fri, 5 Jan 2024 15:09:35 +0800 Subject: [PATCH 05/24] chore: update test timezone --- .../Cliclhouse/ClickhouseApmService.cs | 4 ++-- .../Models/Request/BaseApmRequestDto.cs | 2 +- .../Model/MASAStackClickhouseConnection.cs | 1 + .../ClickhouseApmServiceTests.cs | 6 +++++- .../{GlobalUsings.cs => _Imports.cs} | 1 + .../LogServiceTests.cs | 3 ++- .../TraceServiceTests.cs | 3 ++- 7 files changed, 14 insertions(+), 6 deletions(-) rename src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/{GlobalUsings.cs => _Imports.cs} (90%) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs index fef4e38c4..6d7008de5 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -274,7 +274,6 @@ public async Task EndpointLatencyDistributionAsy { var parameters = new List(); var where = AppendWhere(query, parameters); - var result = new EndpointLatencyDistributionDto(); var p95 = Convert.ToDouble(Scalar($"select floor(quantile(0.95)(Duration)) p95 from {Constants.TraceTableFull} where {where}", parameters)); if (p95 is not double.NaN) @@ -299,6 +298,7 @@ public async Task EndpointLatencyDistributionAsy public async Task> ErrorMessagePageAsync(ApmEndpointRequestDto query) { var parameters = new List(); + query.IsServer = default; var where = AppendWhere(query, parameters); var result = new PaginatedListBase(); @@ -347,7 +347,7 @@ private static string AppendWhere(TQuery query, List p if (query.IsServer.HasValue) { sql.AppendLine(" and SpanKind=@spanKind"); - parameters.Add(new ClickHouseParameter { ParameterName = "serviceName", Value = query.IsServer.Value ? "SPAN_KIND_SERVER" : "SPAN_KIND_CLIENT" }); + parameters.Add(new ClickHouseParameter { ParameterName = "spanKind", Value = query.IsServer.Value ? "SPAN_KIND_SERVER" : "SPAN_KIND_CLIENT" }); } if (query is ApmEndpointRequestDto traceQuery && !string.IsNullOrEmpty(traceQuery.Endpoint)) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs index 051af9739..6499b1b83 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs @@ -27,5 +27,5 @@ public class BaseApmRequestDto : RequestPageBase internal int[] ErrorStatusCodes => string.IsNullOrEmpty(StatusCodes) ? Constants.DefaultErrorStatus : StatusCodes.Split(',').Select(s => Convert.ToInt32(s)).Where(num => num != 0).ToArray(); - internal bool? IsServer { get; set; } + internal bool? IsServer { get; set; } = true; } diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs index 9813fb5b1..10481d8a5 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs @@ -1,6 +1,7 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. [assembly: InternalsVisibleTo("Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse")] +[assembly: InternalsVisibleTo("Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests")] [assembly: InternalsVisibleTo("Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests")] namespace Masa.Contrib.StackSdks.Tsc.Clickhouse; diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs index b31b2e2b7..ff2935459 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs @@ -1,13 +1,15 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. +using Masa.Contrib.StackSdks.Tsc.Clickhouse; + namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests; [TestClass] public class ClickhouseApmServiceTests { private static IAPMService _APMService; - private DateTime _start = DateTime.Parse("2024/01/03 14:00:00"); + private static DateTime _start = DateTime.Parse("2024/01/03 22:00:00"); [ClassInitialize] public static void Initialized(TestContext testContext) @@ -20,6 +22,7 @@ public static void Initialized(TestContext testContext) services.AddMASAStackApmClickhouse(TestUtils.ConnectionString, "custom_log", "custom_trace"); _APMService = services.BuildServiceProvider().GetRequiredService(); Common.InitTableJsonData(false, AppDomain.CurrentDomain.BaseDirectory, connection); + _start -= MasaStackClickhouseConnection.TimeZone.BaseUtcOffset; } [TestMethod] @@ -109,6 +112,7 @@ public async Task ErrorMessagePageAsync() }; var result = await _APMService.ErrorMessagePageAsync(query); Assert.IsNotNull(result); + Assert.IsNotNull(result.Total > 0); } [TestMethod] diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/GlobalUsings.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/_Imports.cs similarity index 90% rename from src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/GlobalUsings.cs rename to src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/_Imports.cs index 2c5c354ed..f2df84947 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/GlobalUsings.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/_Imports.cs @@ -4,6 +4,7 @@ global using ClickHouse.Ado; global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models; global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Request; +global using Masa.Contrib.StackSdks.Tsc.Clickhouse; global using Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Logging; diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/LogServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/LogServiceTests.cs index d22c1baae..463678ed2 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/LogServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/LogServiceTests.cs @@ -7,7 +7,7 @@ namespace Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests; public class LogServiceTests { private static ILogService logService; - private readonly DateTime startTime = DateTime.Parse("2023-11-02 09:00:00"); + private static DateTime startTime = DateTime.Parse("2023-11-02 09:00:00"); [ClassInitialize] public static void Initialized(TestContext testContext) @@ -20,6 +20,7 @@ public static void Initialized(TestContext testContext) services.AddMASAStackClickhouse(Consts.ConnectionString, "custom_log", "custom_trace"); Common.InitTableData(true, AppDomain.CurrentDomain.BaseDirectory, connection); logService = services.BuildServiceProvider().GetRequiredService(); + startTime -= MasaStackClickhouseConnection.TimeZone.BaseUtcOffset; } [TestMethod] diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/TraceServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/TraceServiceTests.cs index f8276cccf..9d63a7596 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/TraceServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/TraceServiceTests.cs @@ -7,7 +7,7 @@ namespace Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests; public class TraceServiceTests { private static ITraceService traceService; - private readonly DateTime startTime = DateTime.Parse("2023-11-02 09:00:00"); + private static DateTime startTime = DateTime.Parse("2023-11-02 09:00:00"); [ClassInitialize] public static void Initialized(TestContext testContext) @@ -18,6 +18,7 @@ public static void Initialized(TestContext testContext) services.AddMASAStackClickhouse(Consts.ConnectionString, "custom_log", "custom_trace"); Common.InitTableData(false, AppDomain.CurrentDomain.BaseDirectory, connection); traceService = services.BuildServiceProvider().GetRequiredService(); + startTime -= MasaStackClickhouseConnection.TimeZone.BaseUtcOffset; } [TestMethod] From ce51251afaefc411d518314ec38e982fb15f3810 Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Mon, 8 Jan 2024 11:10:01 +0800 Subject: [PATCH 06/24] refactor: recode service --- .../Cliclhouse/ClickhouseApmService.cs | 206 +++++++----------- .../TestUtils.cs | 4 +- 2 files changed, 78 insertions(+), 132 deletions(-) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs index 6d7008de5..7ff2ad930 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -1,6 +1,9 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. +using Google.Protobuf.WellKnownTypes; +using Nest; + namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Cliclhouse; internal class ClickhouseApmService : IAPMService, IDisposable @@ -20,160 +23,105 @@ public ClickhouseApmService(IDbConnection dbConnection, ITraceService traceServi public async Task> ServicePageAsync(BaseApmRequestDto query) { - var parameters = new List(); - var where = AppendWhere(query, parameters); - var result = new PaginatedListBase(); + var (where, parameters) = AppendWhere(query); var groupby = "group by ServiceName"; var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; - result.Total = Convert.ToInt64(Scalar(countSql, parameters)); + PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; var orderBy = GetOrderBy(query, "ServiceName"); - var start = (query.Page - 1) * query.PageSize; - if (result.Total - start > 0) - { - var sql = $@"select * from( + var sql = $@"select * from( select ServiceName, arrayStringConcat(groupUniqArray(`Resource.service.namespace`)) env, floor(AVG(Duration)) latency, round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) throughput, round(sum(has(['{string.Join("','", query.ErrorStatusCodes)}'],`Attributes.http.status_code`))*100.0/count(1),2) failed -from {Constants.TraceTableFull} where {where} {groupby} {orderBy} limit {start},{query.PageSize})"; - using var reader = Query(sql, parameters); - result.Result = new(); - while (reader.NextResult()) - while (reader.Read()) - { - var item = new ServiceListDto() - { - Name = reader[0].ToString()!, - Envs = reader[1]?.ToString()?.Split(',') ?? Array.Empty(), - Latency = (long)Math.Floor(Convert.ToDouble(reader[2])), - Throughput = Math.Round(Convert.ToDouble(reader[3]), 2), - Failed = Math.Round(Convert.ToDouble(reader[4]), 2), - }; - result.Result.Add(item); - } - } +from {Constants.TraceTableFull} where {where} {groupby} {orderBy} @limit)"; + SetData(sql, parameters, result, query, reader => new ServiceListDto() + { + Name = reader[0].ToString()!, + Envs = reader[1]?.ToString()?.Split(',') ?? Array.Empty(), + Latency = (long)Math.Floor(Convert.ToDouble(reader[2])), + Throughput = Math.Round(Convert.ToDouble(reader[3]), 2), + Failed = Math.Round(Convert.ToDouble(reader[4]), 2), + }); return result; } public async Task> InstancePageAsync(BaseApmRequestDto query) { - var parameters = new List(); - var where = AppendWhere(query, parameters); - - var result = new PaginatedListBase(); - var groupby = "group by ServiceName,`Attributes.http.target`,`method`"; + var (where, parameters) = AppendWhere(query); + var groupby = "group by instance"; var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; - result.Total = Convert.ToInt64(Scalar(countSql, parameters)); + PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; var orderBy = GetOrderBy(query); - var start = (query.Page - 1) * query.PageSize; - if (result.Total - start > 0) - { - var sql = $@"select * from( select `Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, + var sql = $@"select * from( select ResourceAttributesValues[indexOf(ResourceAttributesKeys,'service.instance.id')] instance`, AVG(Duration) Latency, count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)) throughput sum(has(['{string.Join("','", query.ErrorStatusCodes)}'],`Attributes.http.status_code`))/count(1) failed -from {where} {groupby} {orderBy} limit{start},{query.PageSize})"; - using var reader = Query(sql, parameters); - result.Result = new(); - while (reader.NextResult()) - while (reader.Read()) - { - var item = new EndpointListDto() - { - Name = reader[0].ToString()!, - Service = reader[1]?.ToString()!, - Method = reader[2]?.ToString()!, - Latency = (long)Math.Floor(Convert.ToDouble(reader[3])), - Throughput = Math.Round(Convert.ToDouble(reader[4]), 2), - Failed = Math.Round(Convert.ToDouble(reader[5]), 2) - }; - result.Result.Add(item); - } - } +from {where} {groupby} {orderBy} @limit)"; + SetData(sql, parameters, result, query, reader => new EndpointListDto() + { + Name = reader[0].ToString()!, + //Service = reader[1]?.ToString()!, + //Method = reader[2]?.ToString()!, + Latency = (long)Math.Floor(Convert.ToDouble(reader[1])), + Throughput = Math.Round(Convert.ToDouble(reader[2]), 2), + Failed = Math.Round(Convert.ToDouble(reader[3]), 2) + }); return result; } public async Task> DependencyPageAsync(BaseApmRequestDto query) { - var parameters = new List(); - var where = AppendWhere(query, parameters); - - var result = new PaginatedListBase(); + var (where, parameters) = AppendWhere(query); var groupby = "group by ServiceName,`Attributes.http.target`,`method`"; var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; - result.Total = Convert.ToInt64(Scalar(countSql, parameters)); + PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; var orderBy = GetOrderBy(query); - var start = (query.Page - 1) * query.PageSize; - if (result.Total - start > 0) - { - var sql = $@"select * from( select `Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, + var sql = $@"select * from( select `Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, AVG(Duration) Latency, count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)) throughput sum(has(['{string.Join(",'", query.ErrorStatusCodes)}'],`Attributes.http.status_code`))/count(1) failed -from {where} {groupby} {orderBy} limit{start},{query.PageSize})"; - using var reader = Query(sql, parameters); - result.Result = new(); - while (reader.NextResult()) - while (reader.Read()) - { - var item = new EndpointListDto() - { - Name = reader[0].ToString()!, - Service = reader[1]?.ToString()!, - Method = reader[2]?.ToString()!, - Latency = (long)Math.Floor(Convert.ToDouble(reader[3])), - Throughput = Math.Round(Convert.ToDouble(reader[4]), 2), - Failed = Math.Round(Convert.ToDouble(reader[5]), 2) - }; - result.Result.Add(item); - } - } +from {where} {groupby} {orderBy} @limit)"; + SetData(sql, parameters, result, query, reader => new EndpointListDto() + { + Name = reader[0].ToString()!, + Service = reader[1]?.ToString()!, + Method = reader[2]?.ToString()!, + Latency = (long)Math.Floor(Convert.ToDouble(reader[3])), + Throughput = Math.Round(Convert.ToDouble(reader[4]), 2), + Failed = Math.Round(Convert.ToDouble(reader[5]), 2) + }); return result; } public async Task> EndpointPageAsync(BaseApmRequestDto query) { - var parameters = new List(); - var where = AppendWhere(query, parameters); - var result = new PaginatedListBase(); + var (where, parameters) = AppendWhere(query); var groupby = "group by ServiceName,`Attributes.http.target`,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')]"; var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; - result.Total = Convert.ToInt64(Scalar(countSql, parameters)); + PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; var orderBy = GetOrderBy(query); - var start = (query.Page - 1) * query.PageSize; - if (result.Total - start > 0) - { - var sql = $@"select * from( select `Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, + var sql = $@"select * from( select `Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, floor(AVG(Duration)) Latency, round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) throughput, round(sum(has(['{string.Join("','", query.ErrorStatusCodes)}'],`Attributes.http.status_code`))*100.0/count(1),2) failed -from {Constants.TraceTableFull} where {where} and Attributes.http.target!='' {groupby} {orderBy} limit {start},{query.PageSize})"; - using var reader = Query(sql, parameters); - result.Result = new(); - while (reader.NextResult()) - while (reader.Read()) - { - var item = new EndpointListDto() - { - Name = reader[0].ToString()!, - Service = reader[1]?.ToString()!, - Method = reader[2]?.ToString()!, - Latency = (long)Math.Floor(Convert.ToDouble(reader[3])), - Throughput = Math.Round(Convert.ToDouble(reader[4]), 2), - Failed = Math.Round(Convert.ToDouble(reader[5]), 2) - }; - result.Result.Add(item); - } - } +from {Constants.TraceTableFull} where {where} and Attributes.http.target!='' {groupby} {orderBy} @limit)"; + SetData(sql, parameters, result, query, reader => new EndpointListDto() + { + Name = reader[0].ToString()!, + Service = reader[1]?.ToString()!, + Method = reader[2]?.ToString()!, + Latency = (long)Math.Floor(Convert.ToDouble(reader[3])), + Throughput = Math.Round(Convert.ToDouble(reader[4]), 2), + Failed = Math.Round(Convert.ToDouble(reader[5]), 2) + }); return result; } public async Task> ChartDataAsync(BaseApmRequestDto query) { - var parameters = new List(); - var where = AppendWhere(query, parameters); + var (where, parameters) = AppendWhere(query); var result = new List(); var groupby = "group by ServiceName ,`time` order by ServiceName ,`time`"; var sql = $@"select @@ -272,8 +220,7 @@ private static void SetChartData(List result, IDataReader reader, public async Task EndpointLatencyDistributionAsync(ApmEndpointRequestDto query) { - var parameters = new List(); - var where = AppendWhere(query, parameters); + var (where, parameters) = AppendWhere(query); var result = new EndpointLatencyDistributionDto(); var p95 = Convert.ToDouble(Scalar($"select floor(quantile(0.95)(Duration)) p95 from {Constants.TraceTableFull} where {where}", parameters)); if (p95 is not double.NaN) @@ -297,39 +244,38 @@ public async Task EndpointLatencyDistributionAsy public async Task> ErrorMessagePageAsync(ApmEndpointRequestDto query) { - var parameters = new List(); query.IsServer = default; - var where = AppendWhere(query, parameters); - - var result = new PaginatedListBase(); + var (where, parameters) = AppendWhere(query); var groupby = $"group by Type{(string.IsNullOrEmpty(query.Endpoint) ? "" : ",Endpoint")}"; var countSql = $"select count(1) from (select Attributes.exception.type as Type,max(Timestamp) time,count(1) from {Constants.ErrorTableFull} where {where} {groupby})"; - result.Total = Convert.ToInt64(Scalar(countSql, parameters)); + PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; var orderBy = GetOrderBy(query); + var sql = $@"select * from( select Attributes.exception.type as Type,max(Timestamp) time,count(1) total from {Constants.ErrorTableFull} where {where} {groupby} {orderBy} @limit)"; + SetData(sql, parameters, result, query, reader => new ErrorMessageDto() + { + Type = reader[0]?.ToString()!, + LastTime = Convert.ToDateTime(reader[1])!, + Total = Convert.ToInt32(reader[2]), + }); + return result; + } + + private void SetData(string sql, List parameters, PaginatedListBase result, BaseApmRequestDto query, Func parseFn) where TResult : class + { var start = (query.Page - 1) * query.PageSize; if (result.Total - start > 0) { - var sql = $@"select * from( select Attributes.exception.type as Type,max(Timestamp) time,count(1) total -from {Constants.ErrorTableFull} where {where} {groupby} {orderBy} limit {start},{query.PageSize})"; - using var reader = Query(sql, parameters); + using var reader = Query(sql.Replace("@limit", $"limit {start},{query.PageSize}"), parameters); result.Result = new(); while (reader.NextResult()) while (reader.Read()) - { - var item = new ErrorMessageDto() - { - Type = reader[0]?.ToString()!, - LastTime = Convert.ToDateTime(reader[1])!, - Total = Convert.ToInt32(reader[2]), - }; - result.Result.Add(item); - } + result.Result.Add(parseFn(reader)); } - return result; } - private static string AppendWhere(TQuery query, List parameters) where TQuery : BaseApmRequestDto + private static (string where, List parameters) AppendWhere(TQuery query) where TQuery : BaseApmRequestDto { + List parameters = new(); var sql = new StringBuilder(); sql.AppendLine(" Timestamp between @startTime and @endTime"); parameters.Add(new ClickHouseParameter { ParameterName = "startTime", Value = MasaStackClickhouseConnection.ToTimeZone(query.Start), DbType = DbType.DateTime }); @@ -376,7 +322,7 @@ private static string AppendWhere(TQuery query, List p } } - return sql.ToString(); + return (sql.ToString(), parameters); } public async Task> TraceLatencyDetailAsync(ApmTraceLatencyRequestDto query) diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs index 0118a4bd5..c4b8b60b5 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs @@ -5,6 +5,6 @@ namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests; internal class TestUtils { - public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=localhost;Port=9000;Database=default;User=default"; - //public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=10000;Host=192.168.51.234;Port=19003;User=test;Password=123456;Database=default"; + //public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=localhost;Port=9000;Database=default;User=default"; + public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=10000;Host=192.168.51.234;Port=19003;User=test;Password=123456;Database=default"; } From 845313c99ad635d5a2acc7ca4b091e255eac23cf Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Mon, 8 Jan 2024 11:21:50 +0800 Subject: [PATCH 07/24] chore: update test --- .../TestUtils.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs index c4b8b60b5..0118a4bd5 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs @@ -5,6 +5,6 @@ namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests; internal class TestUtils { - //public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=localhost;Port=9000;Database=default;User=default"; - public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=10000;Host=192.168.51.234;Port=19003;User=test;Password=123456;Database=default"; + public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=localhost;Port=9000;Database=default;User=default"; + //public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=10000;Host=192.168.51.234;Port=19003;User=test;Password=123456;Database=default"; } From 464a14450ee0aeda5c6bdad855133be8c9d0bc72 Mon Sep 17 00:00:00 2001 From: Qinyouzeng <102203523+Qinyouzeng@users.noreply.github.com> Date: Fri, 5 Jan 2024 15:46:51 +0800 Subject: [PATCH 08/24] Update pr_run_test_ci.yml java from 1.11 to 1.17 --- .github/workflows/pr_run_test_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_run_test_ci.yml b/.github/workflows/pr_run_test_ci.yml index d73fe8ad3..357b25aae 100644 --- a/.github/workflows/pr_run_test_ci.yml +++ b/.github/workflows/pr_run_test_ci.yml @@ -57,7 +57,7 @@ jobs: - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.11 + java-version: 1.17 - uses: actions/checkout@v2 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis From e9bab1e11f5ebb8cdaf76163816d9e5c63831ace Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Mon, 8 Jan 2024 12:05:45 +0800 Subject: [PATCH 09/24] relator: update --- .../ApmClickhouseServiceExtensions.cs | 3 +- .../Cliclhouse/ClickhouseApmService.cs | 67 +++++++------------ .../Config/Constants.cs | 4 +- .../Interfaces/IAPMService.cs | 4 +- .../Models/Request/BaseApmRequestDto.cs | 2 +- .../Models/Response/EndpointChartDto.cs | 2 - .../Models/Response/TraceDetailDto.cs | 9 --- .../ClickhouseApmServiceTests.cs | 4 +- 8 files changed, 34 insertions(+), 61 deletions(-) delete mode 100644 src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/TraceDetailDto.cs diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs index 6ac65eea3..2c7c27d38 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs @@ -11,9 +11,8 @@ public static IServiceCollection AddMASAStackApmClickhouse(this IServiceCollecti { return services.AddMASAStackClickhouse(connectionStr, logTable, traceTable, logSourceTable, traceSourceTable, con => { - var connection = (MasaStackClickhouseConnection)con; Constants.Init(MasaStackClickhouseConnection.LogTable.Split('.')[0], MasaStackClickhouseConnection.LogTable.Split('.')[1], MasaStackClickhouseConnection.TraceTable.Split('.')[1], "otel_errors"); - services.TryAddScoped(builder => new ClickhouseApmService(con, services.BuildServiceProvider().GetRequiredService())); + services.TryAddScoped(builder => new ClickhouseApmService(con, services.BuildServiceProvider().GetRequiredService())); Init(services, con); configer?.Invoke(con); diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs index 7ff2ad930..d87f99068 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -1,12 +1,11 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. -using Google.Protobuf.WellKnownTypes; using Nest; namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Cliclhouse; -internal class ClickhouseApmService : IAPMService, IDisposable +internal class ClickhouseApmService : IApmService { private readonly IDbConnection _dbConnection; private readonly IDbCommand _command; @@ -21,7 +20,7 @@ public ClickhouseApmService(IDbConnection dbConnection, ITraceService traceServi _command = dbConnection.CreateCommand(); } - public async Task> ServicePageAsync(BaseApmRequestDto query) + public Task> ServicePageAsync(BaseApmRequestDto query) { var (where, parameters) = AppendWhere(query); var groupby = "group by ServiceName"; @@ -34,7 +33,7 @@ public async Task> ServicePageAsync(BaseApmReq arrayStringConcat(groupUniqArray(`Resource.service.namespace`)) env, floor(AVG(Duration)) latency, round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) throughput, -round(sum(has(['{string.Join("','", query.ErrorStatusCodes)}'],`Attributes.http.status_code`))*100.0/count(1),2) failed +round(sum(has(['{string.Join("','", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))*100.0/count(1),2) failed from {Constants.TraceTableFull} where {where} {groupby} {orderBy} @limit)"; SetData(sql, parameters, result, query, reader => new ServiceListDto() { @@ -44,10 +43,10 @@ public async Task> ServicePageAsync(BaseApmReq Throughput = Math.Round(Convert.ToDouble(reader[3]), 2), Failed = Math.Round(Convert.ToDouble(reader[4]), 2), }); - return result; + return Task.FromResult(result); } - public async Task> InstancePageAsync(BaseApmRequestDto query) + public Task> InstancePageAsync(BaseApmRequestDto query) { var (where, parameters) = AppendWhere(query); var groupby = "group by instance"; @@ -57,31 +56,29 @@ public async Task> InstancePageAsync(BaseApmR var sql = $@"select * from( select ResourceAttributesValues[indexOf(ResourceAttributesKeys,'service.instance.id')] instance`, AVG(Duration) Latency, count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)) throughput -sum(has(['{string.Join("','", query.ErrorStatusCodes)}'],`Attributes.http.status_code`))/count(1) failed +sum(has(['{string.Join("','", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))/count(1) failed from {where} {groupby} {orderBy} @limit)"; SetData(sql, parameters, result, query, reader => new EndpointListDto() { Name = reader[0].ToString()!, - //Service = reader[1]?.ToString()!, - //Method = reader[2]?.ToString()!, Latency = (long)Math.Floor(Convert.ToDouble(reader[1])), Throughput = Math.Round(Convert.ToDouble(reader[2]), 2), Failed = Math.Round(Convert.ToDouble(reader[3]), 2) - }); - return result; + }); + return Task.FromResult(result); } - public async Task> DependencyPageAsync(BaseApmRequestDto query) + public Task> DependencyPageAsync(BaseApmRequestDto query) { var (where, parameters) = AppendWhere(query); var groupby = "group by ServiceName,`Attributes.http.target`,`method`"; var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; - PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; + PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; var orderBy = GetOrderBy(query); var sql = $@"select * from( select `Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, AVG(Duration) Latency, count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)) throughput -sum(has(['{string.Join(",'", query.ErrorStatusCodes)}'],`Attributes.http.status_code`))/count(1) failed +sum(has(['{string.Join(",'", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))/count(1) failed from {where} {groupby} {orderBy} @limit)"; SetData(sql, parameters, result, query, reader => new EndpointListDto() { @@ -92,10 +89,10 @@ public async Task> DependencyPageAsync(BaseAp Throughput = Math.Round(Convert.ToDouble(reader[4]), 2), Failed = Math.Round(Convert.ToDouble(reader[5]), 2) }); - return result; + return Task.FromResult(result); } - public async Task> EndpointPageAsync(BaseApmRequestDto query) + public Task> EndpointPageAsync(BaseApmRequestDto query) { var (where, parameters) = AppendWhere(query); var groupby = "group by ServiceName,`Attributes.http.target`,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')]"; @@ -105,7 +102,7 @@ public async Task> EndpointPageAsync(BaseApmR var sql = $@"select * from( select `Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, floor(AVG(Duration)) Latency, round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) throughput, -round(sum(has(['{string.Join("','", query.ErrorStatusCodes)}'],`Attributes.http.status_code`))*100.0/count(1),2) failed +round(sum(has(['{string.Join("','", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))*100.0/count(1),2) failed from {Constants.TraceTableFull} where {where} and Attributes.http.target!='' {groupby} {orderBy} @limit)"; SetData(sql, parameters, result, query, reader => new EndpointListDto() { @@ -115,11 +112,11 @@ public async Task> EndpointPageAsync(BaseApmR Latency = (long)Math.Floor(Convert.ToDouble(reader[3])), Throughput = Math.Round(Convert.ToDouble(reader[4]), 2), Failed = Math.Round(Convert.ToDouble(reader[5]), 2) - }); - return result; + }); + return Task.FromResult(result); } - public async Task> ChartDataAsync(BaseApmRequestDto query) + public Task> ChartDataAsync(BaseApmRequestDto query) { var (where, parameters) = AppendWhere(query); var result = new List(); @@ -130,13 +127,13 @@ public async Task> ChartDataAsync(BaseApmRequestDto qu floor(avg(Duration)) latency, floor(quantile(0.95)(Duration)) p95, floor(quantile(0.99)(Duration)) p99, -round(sum(has(['{string.Join("','", query.ErrorStatusCodes)}'],`Attributes.http.status_code`))*100.0/count(1),2) failed, +round(sum(has(['{string.Join("','", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))*100.0/count(1),2) failed, round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) throughput from {Constants.TraceTableFull} where {where} {groupby}"; using var reader = Query(sql, parameters); SetChartData(result, reader); GetPreviousChartData(query, sql, parameters, result); - return result; + return Task.FromResult(result.AsEnumerable()); } private void GetPreviousChartData(BaseApmRequestDto query, string sql, List parameters, List result) @@ -176,22 +173,10 @@ private static void SetChartData(List result, IDataReader reader, var name = reader[0].ToString()!; var time = new DateTimeOffset(Convert.ToDateTime(reader[1])).ToUnixTimeSeconds(); if (current == null || current.Name != name) - { - if (isPrevious) + { + if (isPrevious && result.Exists(item => item.Name == name)) { - if (result.Exists(item => item.Name == name)) - { - current = result.First(item => item.Name == name); - } - else - { - current = new ChartLineDto - { - Name = name, - Previous = new List() - }; - result.Add(current); - } + current = result.First(item => item.Name == name); } else { @@ -218,7 +203,7 @@ private static void SetChartData(List result, IDataReader reader, } } - public async Task EndpointLatencyDistributionAsync(ApmEndpointRequestDto query) + public Task EndpointLatencyDistributionAsync(ApmEndpointRequestDto query) { var (where, parameters) = AppendWhere(query); var result = new EndpointLatencyDistributionDto(); @@ -239,10 +224,10 @@ public async Task EndpointLatencyDistributionAsy list.Add(item); } result.Latencies = list; - return result; + return Task.FromResult(result); } - public async Task> ErrorMessagePageAsync(ApmEndpointRequestDto query) + public Task> ErrorMessagePageAsync(ApmEndpointRequestDto query) { query.IsServer = default; var (where, parameters) = AppendWhere(query); @@ -257,7 +242,7 @@ public async Task> ErrorMessagePageAsync(ApmE LastTime = Convert.ToDateTime(reader[1])!, Total = Convert.ToInt32(reader[2]), }); - return result; + return Task.FromResult(result); } private void SetData(string sql, List parameters, PaginatedListBase result, BaseApmRequestDto query, Func parseFn) where TResult : class diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Config/Constants.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Config/Constants.cs index 70c326913..e157eda06 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Config/Constants.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Config/Constants.cs @@ -19,6 +19,8 @@ internal static class Constants public static string ErrorTable { get; private set; } + public static readonly int[] DefaultErrorStatus = new int[] { 500, 501, 502, 503, 504, 505 }; + public static void Init(string database, string logTable, string traceTable, string errorTable) { Database = database; @@ -26,6 +28,4 @@ public static void Init(string database, string logTable, string traceTable, str TraceTable = traceTable; ErrorTable = errorTable; } - - public static int[] DefaultErrorStatus = new int[] { 500, 501, 502, 503, 504, 505 }; } diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IAPMService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IAPMService.cs index 176b2daa3..6bb9852e3 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IAPMService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IAPMService.cs @@ -3,7 +3,7 @@ namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse; -public interface IAPMService +public interface IApmService : IDisposable { /// /// 服务列表页，服务详情页endpoints和instance公用 @@ -45,5 +45,5 @@ public interface IAPMService /// /// /// - Task> ErrorMessagePageAsync(ApmEndpointRequestDto query); + Task> ErrorMessagePageAsync(ApmEndpointRequestDto query); } diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs index 6499b1b83..f907c039f 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs @@ -25,7 +25,7 @@ public class BaseApmRequestDto : RequestPageBase public string StatusCodes { get; set; } - internal int[] ErrorStatusCodes => string.IsNullOrEmpty(StatusCodes) ? Constants.DefaultErrorStatus : StatusCodes.Split(',').Select(s => Convert.ToInt32(s)).Where(num => num != 0).ToArray(); + internal int[] GetErrorStatusCodes() => string.IsNullOrEmpty(StatusCodes) ? Constants.DefaultErrorStatus : StatusCodes.Split(',').Select(s => Convert.ToInt32(s)).Where(num => num != 0).ToArray(); internal bool? IsServer { get; set; } = true; } diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs index 8609c5e62..c5b4fcabd 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs @@ -20,8 +20,6 @@ public class ErrorMessageDto { public string Type { get; set; } - //public string Message { get; set; } - public DateTime LastTime { get; set; } public int Total { get; set; } diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/TraceDetailDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/TraceDetailDto.cs deleted file mode 100644 index a2bdb150c..000000000 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/TraceDetailDto.cs +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) MASA Stack All rights reserved. -// Licensed under the MIT License. See LICENSE.txt in the project root for license information. - -namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Response; - -public class TraceDetailDto -{ - -} diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs index ff2935459..1012f8f55 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs @@ -8,7 +8,7 @@ namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests; [TestClass] public class ClickhouseApmServiceTests { - private static IAPMService _APMService; + private static IApmService _APMService; private static DateTime _start = DateTime.Parse("2024/01/03 22:00:00"); [ClassInitialize] @@ -20,7 +20,7 @@ public static void Initialized(TestContext testContext) var services = new ServiceCollection(); services.AddLogging(builder => builder.AddConsole()); services.AddMASAStackApmClickhouse(TestUtils.ConnectionString, "custom_log", "custom_trace"); - _APMService = services.BuildServiceProvider().GetRequiredService(); + _APMService = services.BuildServiceProvider().GetRequiredService(); Common.InitTableJsonData(false, AppDomain.CurrentDomain.BaseDirectory, connection); _start -= MasaStackClickhouseConnection.TimeZone.BaseUtcOffset; } From a826b5fc2e7a05bbed17ce90859e7a994c341a71 Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Mon, 8 Jan 2024 13:30:28 +0800 Subject: [PATCH 10/24] chore: update --- .../Cliclhouse/ClickhouseApmService.cs | 63 +++++++++---------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs index d87f99068..fb0ae6ab0 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -1,8 +1,6 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. -using Nest; - namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Cliclhouse; internal class ClickhouseApmService : IApmService @@ -166,41 +164,42 @@ private void GetPreviousChartData(BaseApmRequestDto query, string sql, List result, IDataReader reader, bool isPrevious = false) { + if (!reader.NextResult()) + return; ChartLineDto? current = null; - while (reader.NextResult()) - while (reader.Read()) + while (reader.Read()) + { + var name = reader[0].ToString()!; + var time = new DateTimeOffset(Convert.ToDateTime(reader[1])).ToUnixTimeSeconds(); + if (current == null || current.Name != name) { - var name = reader[0].ToString()!; - var time = new DateTimeOffset(Convert.ToDateTime(reader[1])).ToUnixTimeSeconds(); - if (current == null || current.Name != name) - { - if (isPrevious && result.Exists(item => item.Name == name)) - { - current = result.First(item => item.Name == name); - } - else - { - current = new ChartLineDto - { - Name = name, - Previous = new List(), - Currents = new List() - }; - result.Add(current); - } + if (isPrevious && result.Exists(item => item.Name == name)) + { + current = result.First(item => item.Name == name); } - - ((List)(isPrevious ? current.Previous : current.Currents)).Add( - new() + else + { + current = new ChartLineDto { - Latency = (long)Math.Floor(Convert.ToDouble(reader[2])), - P95 = Math.Round(Convert.ToDouble(reader[3]), 2, MidpointRounding.ToZero), - P99 = Math.Round(Convert.ToDouble(reader[4]), 2, MidpointRounding.ToZero), - Failed = Math.Round(Convert.ToDouble(reader[5]), 2, MidpointRounding.ToZero), - Throughput = Math.Round(Convert.ToDouble(reader[6]), 2, MidpointRounding.ToZero), - Time = time - }); + Name = name, + Previous = new List(), + Currents = new List() + }; + result.Add(current); + } } + + ((List)(isPrevious ? current.Previous : current.Currents)).Add( + new() + { + Latency = (long)Math.Floor(Convert.ToDouble(reader[2])), + P95 = Math.Round(Convert.ToDouble(reader[3]), 2, MidpointRounding.ToZero), + P99 = Math.Round(Convert.ToDouble(reader[4]), 2, MidpointRounding.ToZero), + Failed = Math.Round(Convert.ToDouble(reader[5]), 2, MidpointRounding.ToZero), + Throughput = Math.Round(Convert.ToDouble(reader[6]), 2, MidpointRounding.ToZero), + Time = time + }); + } } public Task EndpointLatencyDistributionAsync(ApmEndpointRequestDto query) From 1cc07c8a04e478cf73efed1c1172f745a451989a Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Fri, 8 Mar 2024 17:28:56 +0800 Subject: [PATCH 11/24] feat: update to clickhouse query for kibana --- .../Extensions/DictionaryExtenistions.cs | 3 +- .../Model/Trace/TraceDatabaseResponseDto.cs | 3 + .../Model/Trace/TraceHttpResponseDto.cs | 7 + .../ApmClickhouseServiceExtensions.cs | 6 +- .../Cliclhouse/ClickhouseApmService.cs | 375 +++++++++++++++--- .../Interfaces/IAPMService.cs | 13 + ...ontrib.StackSdks.Tsc.Apm.Clickhouse.csproj | 1 + .../Models/Request/BaseApmRequestDto.cs | 4 + .../Models/Response/EndpointChartDto.cs | 2 + .../Models/Response/ServiceListDto.cs | 16 + .../Extensions/IDbConnectionExtensitions.cs | 23 +- ...sa.Contrib.StackSdks.Tsc.Clickhouse.csproj | 1 + .../Model/MASAStackClickhouseConnection.cs | 2 + 13 files changed, 376 insertions(+), 80 deletions(-) diff --git a/src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Extensions/DictionaryExtenistions.cs b/src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Extensions/DictionaryExtenistions.cs index 95c397e58..eb0fb2d3e 100644 --- a/src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Extensions/DictionaryExtenistions.cs +++ b/src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Extensions/DictionaryExtenistions.cs @@ -7,7 +7,8 @@ public static class DictionaryExtenistions { private static readonly JsonSerializerOptions _serializerOptions = new() { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + NumberHandling = JsonNumberHandling.AllowReadingFromString }; public static Dictionary GroupByKeyPrefix(this Dictionary source, string prefix, Func? convertFunc = null) diff --git a/src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceDatabaseResponseDto.cs b/src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceDatabaseResponseDto.cs index 72c6cbe7e..a2fe169d4 100644 --- a/src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceDatabaseResponseDto.cs +++ b/src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceDatabaseResponseDto.cs @@ -21,6 +21,7 @@ public class TraceDatabaseResponseDto public virtual string PeerName { get; set; } [JsonPropertyName("net.peer.port")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] public virtual int PeerPort { get; set; } [JsonPropertyName("net.transport")] @@ -42,6 +43,7 @@ public class TraceDatabaseResponseDto public virtual string Operation { get; set; } [JsonPropertyName("db.redis.database_index")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] public virtual int RedisDatabaseIndex { get; set; } [JsonPropertyName("db.mongodb.collection")] @@ -53,6 +55,7 @@ public class TraceDatabaseResponseDto #region Cassandra [JsonPropertyName("db.cassandra.page_size")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] public virtual int CassandraPageSize { get; set; } [JsonPropertyName("db.cassandra.consistency_level")] diff --git a/src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceHttpResponseDto.cs b/src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceHttpResponseDto.cs index 8e6c7fb7f..cea96ace1 100644 --- a/src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceHttpResponseDto.cs +++ b/src/BuildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceHttpResponseDto.cs @@ -25,6 +25,7 @@ public class TraceHttpResponseDto public virtual string Scheme { get; set; } [JsonPropertyName("http.status_code")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] public virtual int StatusCode { get; set; } [JsonPropertyName("http.flavor")] @@ -34,28 +35,34 @@ public class TraceHttpResponseDto public virtual string UserAgent { get; set; } [JsonPropertyName("http.request_content_length")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] public virtual int RequestContentLength { get; set; } [JsonPropertyName("http.request_content_length_uncompressed")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] public virtual int RequestContentLengthUncompressed { get; set; } public Dictionary> RequestHeaders { get; set; } [JsonPropertyName("http.response_content_length")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] public virtual int ResponseContentLength { get; set; } [JsonPropertyName("http.response_content_length_uncompressed")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] public virtual int ResponseContentLengthUncompressed { get; set; } public Dictionary> ReponseHeaders { get; set; } [JsonPropertyName("http.retry_count")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] public virtual int RetryCount { get; set; } [JsonPropertyName("net.peer.ip")] public virtual string PeerIp { get; set; } [JsonPropertyName("net.peer.port")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] public virtual int? PeerPort { get; set; } [JsonPropertyName("net.peer.name")] diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs index 2c7c27d38..756f610a7 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs @@ -9,14 +9,14 @@ public static class ApmClickhouseServiceExtensions public static IServiceCollection AddMASAStackApmClickhouse(this IServiceCollection services, string connectionStr, string logTable, string traceTable, string? logSourceTable = null, string? traceSourceTable = null, Action? configer = null) { - return services.AddMASAStackClickhouse(connectionStr, logTable, traceTable, logSourceTable, traceSourceTable, con => + services.AddMASAStackClickhouse(connectionStr, logTable, traceTable, logSourceTable, traceSourceTable, con => { Constants.Init(MasaStackClickhouseConnection.LogTable.Split('.')[0], MasaStackClickhouseConnection.LogTable.Split('.')[1], MasaStackClickhouseConnection.TraceTable.Split('.')[1], "otel_errors"); - services.TryAddScoped(builder => new ClickhouseApmService(con, services.BuildServiceProvider().GetRequiredService())); Init(services, con); configer?.Invoke(con); - }); + services.AddScoped(); + return services; } private static void Init(IServiceCollection services, IDbConnection connection) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs index fb0ae6ab0..27624c9e1 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -1,35 +1,63 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. +using System.Text.RegularExpressions; + namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Cliclhouse; internal class ClickhouseApmService : IApmService { - private readonly IDbConnection _dbConnection; - private readonly IDbCommand _command; + private MasaStackClickhouseConnection _dbConnection; + private readonly IDbCommand command; private readonly ITraceService _traceService; + private readonly static object lockObj = new(); + private static Dictionary serviceOrders = new Dictionary() { + {nameof(ServiceListDto.Name),"ServiceName"}, + {nameof(ServiceListDto.Envs),"env"}, + {nameof(ServiceListDto.Latency),"latency"}, + {nameof(ServiceListDto.Throughput),"throughput"}, + {nameof(ServiceListDto.Failed),"failed"}, + }; + + private static Dictionary endpointOrders = new Dictionary() { + {nameof(EndpointListDto.Name),"`Attributes.http.target`"}, + {nameof(EndpointListDto.Service),"ServiceName"}, + {nameof(EndpointListDto.Method),"`method`"}, + {nameof(EndpointListDto.Latency),"latency"}, + {nameof(EndpointListDto.Throughput),"throughput"}, + {nameof(EndpointListDto.Failed),"failed"}, + }; + + private static Dictionary errorOrders = new Dictionary() { + {nameof(ErrorMessageDto.Type),"Type"}, + {nameof(ErrorMessageDto.Message),"Message"}, + {nameof(ErrorMessageDto.LastTime),"`time`"}, + {nameof(ErrorMessageDto.Total),"`total`"} + }; + const double MILLSECOND = 1e6; - public ClickhouseApmService(IDbConnection dbConnection, ITraceService traceService) + public ClickhouseApmService(MasaStackClickhouseConnection dbConnection, ITraceService traceService) { _traceService = traceService; _dbConnection = dbConnection; + command = dbConnection.CreateCommand(); if (_dbConnection.State == ConnectionState.Closed) _dbConnection.Open(); - _command = dbConnection.CreateCommand(); } public Task> ServicePageAsync(BaseApmRequestDto query) { + query.IsServer = true; var (where, parameters) = AppendWhere(query); var groupby = "group by ServiceName"; var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; - var orderBy = GetOrderBy(query, "ServiceName"); + var orderBy = GetOrderBy(query, serviceOrders, defaultSort: "ServiceName"); var sql = $@"select * from( select ServiceName, arrayStringConcat(groupUniqArray(`Resource.service.namespace`)) env, -floor(AVG(Duration)) latency, +floor(AVG(Duration/{MILLSECOND})) latency, round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) throughput, round(sum(has(['{string.Join("','", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))*100.0/count(1),2) failed from {Constants.TraceTableFull} where {where} {groupby} {orderBy} @limit)"; @@ -50,9 +78,9 @@ public Task> InstancePageAsync(BaseApmRequest var groupby = "group by instance"; var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; - var orderBy = GetOrderBy(query); + var orderBy = GetOrderBy(query, new()); var sql = $@"select * from( select ResourceAttributesValues[indexOf(ResourceAttributesKeys,'service.instance.id')] instance`, -AVG(Duration) Latency, +AVG(Duration/{MILLSECOND}) Latency, count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)) throughput sum(has(['{string.Join("','", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))/count(1) failed from {where} {groupby} {orderBy} @limit)"; @@ -72,9 +100,9 @@ public Task> DependencyPageAsync(BaseApmReque var groupby = "group by ServiceName,`Attributes.http.target`,`method`"; var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; - var orderBy = GetOrderBy(query); + var orderBy = GetOrderBy(query, new()); var sql = $@"select * from( select `Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, -AVG(Duration) Latency, +AVG(Duration{MILLSECOND}) Latency, count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)) throughput sum(has(['{string.Join(",'", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))/count(1) failed from {where} {groupby} {orderBy} @limit)"; @@ -92,13 +120,14 @@ public Task> DependencyPageAsync(BaseApmReque public Task> EndpointPageAsync(BaseApmRequestDto query) { + query.IsServer = true; var (where, parameters) = AppendWhere(query); var groupby = "group by ServiceName,`Attributes.http.target`,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')]"; var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; - var orderBy = GetOrderBy(query); + var orderBy = GetOrderBy(query, endpointOrders); var sql = $@"select * from( select `Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, -floor(AVG(Duration)) Latency, +floor(AVG(Duration/{MILLSECOND})) latency, round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) throughput, round(sum(has(['{string.Join("','", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))*100.0/count(1),2) failed from {Constants.TraceTableFull} where {where} and Attributes.http.target!='' {groupby} {orderBy} @limit)"; @@ -116,20 +145,24 @@ public Task> EndpointPageAsync(BaseApmRequest public Task> ChartDataAsync(BaseApmRequestDto query) { + query.IsServer = true; var (where, parameters) = AppendWhere(query); var result = new List(); var groupby = "group by ServiceName ,`time` order by ServiceName ,`time`"; var sql = $@"select ServiceName, -toStartOfInterval(`Timestamp` , INTERVAL 1 minute ) as `time`, -floor(avg(Duration)) latency, -floor(quantile(0.95)(Duration)) p95, -floor(quantile(0.99)(Duration)) p99, -round(sum(has(['{string.Join("','", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))*100.0/count(1),2) failed, -round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) throughput +toStartOfInterval(`Timestamp` , INTERVAL {GetPeriod(query)} ) as `time`, +floor(avg(Duration/{MILLSECOND})) `latency`, +floor(quantile(0.95)(Duration/{MILLSECOND})) `p95`, +floor(quantile(0.99)(Duration/{MILLSECOND})) `p99`, +round(sum(has(['{string.Join("','", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))*100.0/count(1),2) `failed`, +round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) `throughput` from {Constants.TraceTableFull} where {where} {groupby}"; - using var reader = Query(sql, parameters); - SetChartData(result, reader); + lock (lockObj) + { + using var reader = Query(sql, parameters); + SetChartData(result, reader); + } GetPreviousChartData(query, sql, parameters, result); return Task.FromResult(result.AsEnumerable()); } @@ -158,8 +191,11 @@ private void GetPreviousChartData(BaseApmRequestDto query, string sql, List p.ParameterName == "endTime"); paramEndTime.Value = ((DateTime)paramEndTime.Value!).AddDays(day); - using var readerPrevious = Query(sql, parameters); - SetChartData(result, readerPrevious, isPrevious: true); + lock (lockObj) + { + using var readerPrevious = Query(sql, parameters); + SetChartData(result, readerPrevious, isPrevious: true); + } } private static void SetChartData(List result, IDataReader reader, bool isPrevious = false) @@ -206,22 +242,25 @@ public Task EndpointLatencyDistributionAsync(Apm { var (where, parameters) = AppendWhere(query); var result = new EndpointLatencyDistributionDto(); - var p95 = Convert.ToDouble(Scalar($"select floor(quantile(0.95)(Duration)) p95 from {Constants.TraceTableFull} where {where}", parameters)); + var p95 = Convert.ToDouble(Scalar($"select floor(quantile(0.95)(Duration/{MILLSECOND})) p95 from {Constants.TraceTableFull} where {where}", parameters)); if (p95 is not double.NaN) result.P95 = (long)Math.Floor(p95); - var sql = $@"select Duration,count(1) total from {Constants.TraceTableFull} where {where} group by Duration order by Duration"; - using var reader = Query(sql, parameters); + var sql = $@"select Duration/{MILLSECOND},count(1) total from {Constants.TraceTableFull} where {where} group by Duration order by Duration"; var list = new List(); - while (reader.NextResult()) - while (reader.Read()) - { - var item = new ChartPointDto() + lock (lockObj) + { + using var reader = Query(sql, parameters); + while (reader.NextResult()) + while (reader.Read()) { - X = reader[0].ToString()!, - Y = reader[1]?.ToString()! - }; - list.Add(item); - } + var item = new ChartPointDto() + { + X = reader[0].ToString()!, + Y = reader[1]?.ToString()! + }; + list.Add(item); + } + } result.Latencies = list; return Task.FromResult(result); } @@ -230,16 +269,17 @@ public Task> ErrorMessagePageAsync(ApmEndpoin { query.IsServer = default; var (where, parameters) = AppendWhere(query); - var groupby = $"group by Type{(string.IsNullOrEmpty(query.Endpoint) ? "" : ",Endpoint")}"; - var countSql = $"select count(1) from (select Attributes.exception.type as Type,max(Timestamp) time,count(1) from {Constants.ErrorTableFull} where {where} {groupby})"; + var groupby = $"group by Type,Message{(string.IsNullOrEmpty(query.Endpoint) ? "" : ",Endpoint")}"; + var countSql = $"select count(1) from (select Attributes.exception.type as Type,Attributes.exception.message as Message,max(Timestamp) time,count(1) from {Constants.ErrorTableFull} where {where} {groupby})"; PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; - var orderBy = GetOrderBy(query); - var sql = $@"select * from( select Attributes.exception.type as Type,max(Timestamp) time,count(1) total from {Constants.ErrorTableFull} where {where} {groupby} {orderBy} @limit)"; + var orderBy = GetOrderBy(query, errorOrders); + var sql = $@"select * from( select Attributes.exception.type as Type,Attributes.exception.message as Message,max(Timestamp) time,count(1) total from {Constants.ErrorTableFull} where {where} {groupby} {orderBy} @limit)"; SetData(sql, parameters, result, query, reader => new ErrorMessageDto() { Type = reader[0]?.ToString()!, - LastTime = Convert.ToDateTime(reader[1])!, - Total = Convert.ToInt32(reader[2]), + Message = reader[1]?.ToString()!, + LastTime = Convert.ToDateTime(reader[2])!, + Total = Convert.ToInt32(reader[3]), }); return Task.FromResult(result); } @@ -249,11 +289,14 @@ private void SetData(string sql, List parameters, Pag var start = (query.Page - 1) * query.PageSize; if (result.Total - start > 0) { - using var reader = Query(sql.Replace("@limit", $"limit {start},{query.PageSize}"), parameters); - result.Result = new(); - while (reader.NextResult()) - while (reader.Read()) - result.Result.Add(parseFn(reader)); + lock (lockObj) + { + using var reader = Query(sql.Replace("@limit", $"limit {start},{query.PageSize}"), parameters); + result.Result = new(); + while (reader.NextResult()) + while (reader.Read()) + result.Result.Add(parseFn(reader)); + } } } @@ -282,8 +325,16 @@ private static (string where, List parameters) AppendWhere=0 and LogAttributesValues[indexOf(LogAttributesKeys,'RequestPath')] LIKE @endpoint"); + parameters.Add(new ClickHouseParameter { ParameterName = "endpoint", Value = $"{traceQuery.Endpoint}%" }); + } + else + { + sql.AppendLine(" and Attributes.http.target=@endpoint"); + parameters.Add(new ClickHouseParameter { ParameterName = "endpoint", Value = traceQuery.Endpoint }); + } } if (query is ApmTraceLatencyRequestDto durationQuery) @@ -306,6 +357,13 @@ private static (string where, List parameters) AppendWhere 0) + { + if (!query.Queries.Trim().StartsWith("and ", StringComparison.CurrentCultureIgnoreCase)) + sql.Append(" and "); + sql.AppendLine(query.Queries); + } + return (sql.ToString(), parameters); } @@ -355,38 +413,42 @@ public async Task> TraceLatencyDetailAsync(A private IDataReader Query(string sql, IEnumerable parameters) { - _command.CommandText = sql; + command.CommandText = sql; SetParameters(parameters); - return _command.ExecuteReader(); + return command.ExecuteReader(); } private object Scalar(string sql, IEnumerable parameters) { - _command.CommandText = sql; - SetParameters(parameters); - return _command.ExecuteScalar()!; + lock (lockObj) + { + command.CommandText = sql; + SetParameters(parameters); + return command.ExecuteScalar()!; + } } private void SetParameters(IEnumerable parameters) { - if (_command.Parameters.Count > 0) - _command.Parameters.Clear(); + if (command.Parameters.Count > 0) + command.Parameters.Clear(); if (parameters != null && parameters.Any()) foreach (var param in parameters) - _command.Parameters.Add(param); + command.Parameters.Add(param); } - private static string? GetOrderBy(BaseApmRequestDto query, string? defaultSort = null, bool isDesc = false) + private static string? GetOrderBy(BaseApmRequestDto query, Dictionary sortFields, string? defaultSort = null, bool isDesc = false) { - if (string.IsNullOrEmpty(query.OrderField)) + if (!string.IsNullOrEmpty(query.OrderField) && sortFields.ContainsKey(query.OrderField)) { - if (string.IsNullOrEmpty(defaultSort)) - return null; - return $"order by {defaultSort}{(isDesc ? " desc" : "")}"; + if (!query.IsDesc.HasValue) + return $"order by {sortFields[query.OrderField]}"; + return $"order by {sortFields[query.OrderField]}{(query.IsDesc.Value ? "" : " desc")}"; } - if (!query.IsDesc.HasValue) - return $"order by {query.OrderField}"; - return $"order by {query.OrderField}{(query.IsDesc.Value ? "" : " desc")}"; + + if (string.IsNullOrEmpty(defaultSort)) + return null; + return $"order by {defaultSort}{(isDesc ? " desc" : "")}"; } public void Dispose() @@ -394,4 +456,191 @@ public void Dispose() _dbConnection.Close(); _dbConnection.Dispose(); } + + public Task> GetErrorChartAsync(ApmEndpointRequestDto query) + { + query.IsServer = default; + query.IsLog = true; + var (where, parameters) = AppendWhere(query); + var groupby = "group by `time` order by `time`"; + var sql = $@"select +toStartOfInterval(`Timestamp` , INTERVAL {GetPeriod(query)} ) as `time`, +count(1) `total` +from {Constants.LogTableFull} where {where} and SeverityText='Error' {groupby}"; + + return Task.FromResult(getChartCountData(sql, parameters, query.ComparisonType)); + } + + private IEnumerable getChartCountData(string sql, IEnumerable parameters, ComparisonTypes? comparisonTypes = null) + { + var result = new List(); + lock (lockObj) + { + using var currentReader = Query(sql, parameters); + SetChartCountData(result, currentReader); + } + + if (comparisonTypes.HasValue && (comparisonTypes.Value == ComparisonTypes.DayBefore || comparisonTypes.Value == ComparisonTypes.WeekBefore)) + { + var day = comparisonTypes.Value == ComparisonTypes.DayBefore ? -1 : -7; + var paramStartTime = parameters.First(p => p.ParameterName == "startTime"); + paramStartTime.Value = ((DateTime)paramStartTime.Value!).AddDays(day); + + var paramEndTime = parameters.First(p => p.ParameterName == "endTime"); + paramEndTime.Value = ((DateTime)paramEndTime.Value!).AddDays(day); + + lock (lockObj) + { + using var previousReader = Query(sql, parameters); + SetChartCountData(result, previousReader, true); + } + } + + return result; + } + + private void SetChartCountData(List result, IDataReader reader, bool isPrevious = false) + { + if (!reader.NextResult()) + return; + ChartLineCountDto? current = null; + while (reader.Read()) + { + var name = reader[0].ToString()!; + var time = new DateTimeOffset(Convert.ToDateTime(reader[0])).ToUnixTimeSeconds(); + if (current == null || current.Name != name) + { + if (isPrevious && result.Exists(item => item.Name == name)) + { + current = result.First(item => item.Name == name); + } + else + { + current = new ChartLineCountDto + { + Name = name, + Previous = new List(), + Currents = new List() + }; + result.Add(current); + } + } + + ((List)(isPrevious ? current.Previous : current.Currents)).Add( + new() + { + Value = reader[1], + Time = time + }); + } + } + + public Task> GetEndpointChartAsync(ApmEndpointRequestDto query) + { + query.IsServer = false; + var (where, parameters) = AppendWhere(query); + var groupby = "group by `time` order by `time`"; + var sql = $@"select +toStartOfInterval(`Timestamp` , INTERVAL {GetPeriod(query)} ) as `time`, +count(1) `total` +from {Constants.TraceTable} where {where} {groupby}"; + + return Task.FromResult(getChartCountData(sql, parameters, query.ComparisonType)); + } + + public Task> GetLogChartAsync(ApmEndpointRequestDto query) + { + query.IsServer = default; + query.IsLog = true; + var (where, parameters) = AppendWhere(query); + var groupby = "group by `time` order by `time`"; + var sql = $@"select +toStartOfInterval(`Timestamp` , INTERVAL {GetPeriod(query)} ) as `time`, +count(1) `total` +from {Constants.LogTableFull} where {where} {groupby}"; + return Task.FromResult(getChartCountData(sql, parameters, query.ComparisonType)); + } + + private string GetPeriod(BaseApmRequestDto query) + { + var reg = new Regex(@"/d+"); + if (string.IsNullOrEmpty(query.Period) || !reg.IsMatch(query.Period)) + { + return GetDefaultPeriod(query.End - query.Start); + } + var unit = reg.Replace(query.Period, "").Trim().ToLower(); + var units = new string[] { "year", "month", "week", "day", "hour", "minute", "second" }; + var find = units.FirstOrDefault(s => s.StartsWith(unit)); + if (string.IsNullOrEmpty(find)) + find = "minute"; + return $"{reg.Match(query.Period).Result} {unit}"; + } + + private string GetDefaultPeriod(TimeSpan timeSpan) + { + if ((int)timeSpan.TotalMinutes < 1) + { + return "5 second"; + } + + if ((int)timeSpan.TotalHours < 1) + { + return "1 minute"; + } + + var days = (int)timeSpan.TotalDays; + if (days <= 0) + { + if ((int)timeSpan.TotalHours - 12 <= 0) + { + return "1 minute"; + } + return "30 minute"; + } + + if (days - 7 <= 0) + { + return "1 hour"; + } + + if (days - 30 <= 0) + { + return "1 day"; + } + + if (days - 365 <= 0) + { + return "1 week"; + } + + return "1 month"; + } + + public Task> GetTraceErrorsAsync(ApmEndpointRequestDto query) + { + query.IsServer = default; + query.IsLog = true; + var (where, parameters) = AppendWhere(query); + var groupby = "group by `SpanId` order by `SpanId`"; + var sql = $@"select +SpanId, +count(1) `total` +from {Constants.ErrorTableFull} where {where} {groupby}"; + var list = new List(); + lock (lockObj) + { + using var reader = Query(sql, parameters); + while (reader.NextResult()) + while (reader.Read()) + { + var item = new ChartPointDto() + { + X = reader[0].ToString()!, + Y = reader[1]?.ToString()! + }; + list.Add(item); + } + } + return Task.FromResult(list.AsEnumerable()); + } } diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IAPMService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IAPMService.cs index 6bb9852e3..b85f3373a 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IAPMService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IAPMService.cs @@ -46,4 +46,17 @@ public interface IApmService : IDisposable /// /// Task> ErrorMessagePageAsync(ApmEndpointRequestDto query); + + /// + /// 获取trace下的错误信息统计，按照spanId + /// + /// + /// + Task> GetTraceErrorsAsync(ApmEndpointRequestDto query); + + Task> GetErrorChartAsync(ApmEndpointRequestDto query); + + Task> GetEndpointChartAsync(ApmEndpointRequestDto query); + + Task> GetLogChartAsync(ApmEndpointRequestDto query); } diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj index 465eae53d..2e96e1e79 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj @@ -3,6 +3,7 @@ enable enable + 1.0-local diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs index f907c039f..7f39231e0 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs @@ -28,4 +28,8 @@ public class BaseApmRequestDto : RequestPageBase internal int[] GetErrorStatusCodes() => string.IsNullOrEmpty(StatusCodes) ? Constants.DefaultErrorStatus : StatusCodes.Split(',').Select(s => Convert.ToInt32(s)).Where(num => num != 0).ToArray(); internal bool? IsServer { get; set; } = true; + + internal bool? IsTrace { get; set; } + + internal bool? IsLog { get; set; } } diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs index c5b4fcabd..9d32bb8de 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs @@ -20,6 +20,8 @@ public class ErrorMessageDto { public string Type { get; set; } + public string Message { get; set; } + public DateTime LastTime { get; set; } public int Total { get; set; } diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/ServiceListDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/ServiceListDto.cs index 6789b0fec..fffcabaa4 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/ServiceListDto.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/ServiceListDto.cs @@ -27,6 +27,22 @@ public class ChartLineDto public IEnumerable Previous { get; set; } } +public class ChartLineCountDto +{ + public string Name { get; set; } + + public IEnumerable Currents { get; set; } + + public IEnumerable Previous { get; set; } +} + +public class ChartLineCountItemDto +{ + public long Time { get; set; } + + public object Value { get; set; } +} + public class ChartLineItemDto { public long Time { get; set; } diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs index c0eb79738..a6e7383ca 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs @@ -76,8 +76,14 @@ public static List GetTraceByTraceId(this IDbConnection connec public static string AppendOrderBy(BaseRequestDto query, bool isLog) { - var str = query.Sort?.IsDesc ?? true ? " desc" : ""; - return $" order by Timestamp{str}"; + string field = "Timestamp"; + var isDesc = query.Sort?.IsDesc ?? true; + if (isLog && query.Sort != null && !string.IsNullOrEmpty(query.Sort.Name)) + { + field = GetName(query.Sort.Name, isLog); + isDesc = query.Sort?.IsDesc ?? false; + } + return $" order by {field}{(isDesc ? " desc" : "")}"; } public static (string where, List @parameters, List ors) AppendWhere(BaseRequestDto query, bool isTrace = true) @@ -270,16 +276,8 @@ private static void ParseWhere(StringBuilder sql, object value, List Query(this IDbConnection dbConnection, string sql, IDataParameter[]? @parameters, Func parse) @@ -301,7 +299,6 @@ public static List Query(this IDbConnection dbConnection, string sql, IDat { list.Add(parse.Invoke(reader)); } - return list; } catch (Exception ex) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Masa.Contrib.StackSdks.Tsc.Clickhouse.csproj b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Masa.Contrib.StackSdks.Tsc.Clickhouse.csproj index 4ebb962de..2ca7d8fef 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Masa.Contrib.StackSdks.Tsc.Clickhouse.csproj +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Masa.Contrib.StackSdks.Tsc.Clickhouse.csproj @@ -3,6 +3,7 @@ enable enable + 1.0-local diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs index 10481d8a5..cbdc8dc59 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs @@ -24,6 +24,8 @@ public static DateTime ToTimeZone(DateTime utcTime) return utcTime + TimeZone.BaseUtcOffset; } + public object LockObj { get; init; } = new(); + public MasaStackClickhouseConnection(string connection, string logTable, string traceTable, string? logSourceTable = null, string? traceSourceTable = null) { ArgumentNullException.ThrowIfNull(connection); From a67227b4e402a692a545cb8e59bd027dbea21571 Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Mon, 11 Mar 2024 09:53:16 +0800 Subject: [PATCH 12/24] chore: update --- .../Cliclhouse/ClickhouseApmService.cs | 41 +++++++++++-------- .../_Imports.cs | 3 +- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs index 27624c9e1..558b583d6 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -1,8 +1,6 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. -using System.Text.RegularExpressions; - namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Cliclhouse; internal class ClickhouseApmService : IApmService @@ -106,15 +104,7 @@ public Task> DependencyPageAsync(BaseApmReque count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)) throughput sum(has(['{string.Join(",'", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))/count(1) failed from {where} {groupby} {orderBy} @limit)"; - SetData(sql, parameters, result, query, reader => new EndpointListDto() - { - Name = reader[0].ToString()!, - Service = reader[1]?.ToString()!, - Method = reader[2]?.ToString()!, - Latency = (long)Math.Floor(Convert.ToDouble(reader[3])), - Throughput = Math.Round(Convert.ToDouble(reader[4]), 2), - Failed = Math.Round(Convert.ToDouble(reader[5]), 2) - }); + SetData(sql, parameters, result, query, ConvertEndpointDto); return Task.FromResult(result); } @@ -131,7 +121,13 @@ public Task> EndpointPageAsync(BaseApmRequest round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) throughput, round(sum(has(['{string.Join("','", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))*100.0/count(1),2) failed from {Constants.TraceTableFull} where {where} and Attributes.http.target!='' {groupby} {orderBy} @limit)"; - SetData(sql, parameters, result, query, reader => new EndpointListDto() + SetData(sql, parameters, result, query, ConvertEndpointDto); + return Task.FromResult(result); + } + + private EndpointListDto ConvertEndpointDto(IDataReader reader) + { + return new EndpointListDto() { Name = reader[0].ToString()!, Service = reader[1]?.ToString()!, @@ -139,8 +135,7 @@ public Task> EndpointPageAsync(BaseApmRequest Latency = (long)Math.Floor(Convert.ToDouble(reader[3])), Throughput = Math.Round(Convert.ToDouble(reader[4]), 2), Failed = Math.Round(Convert.ToDouble(reader[5]), 2) - }); - return Task.FromResult(result); + }; } public Task> ChartDataAsync(BaseApmRequestDto query) @@ -451,10 +446,20 @@ private void SetParameters(IEnumerable parameters) return $"order by {defaultSort}{(isDesc ? " desc" : "")}"; } - public void Dispose() + void IDisposable.Dispose() { - _dbConnection.Close(); - _dbConnection.Dispose(); + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool isDisposing) + { + if (isDisposing) + { + _dbConnection.Close(); + _dbConnection.Dispose(); + GC.SuppressFinalize(this); + } } public Task> GetErrorChartAsync(ApmEndpointRequestDto query) @@ -642,5 +647,5 @@ public Task> GetTraceErrorsAsync(ApmEndpointRequestDt } } return Task.FromResult(list.AsEnumerable()); - } + } } diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs index 0640ffc04..ff1b5ffc4 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs @@ -13,9 +13,8 @@ global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Response; global using Masa.Contrib.StackSdks.Tsc.Clickhouse; global using Masa.Utils.Models; -global using Microsoft.Extensions.DependencyInjection.Extensions; global using Microsoft.Extensions.Logging; global using System.Data; global using System.Data.Common; -global using System.Runtime.CompilerServices; global using System.Text; +global using System.Text.RegularExpressions; From 6a111bc680eef11a2f64953a5d1ef53b31628780 Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Mon, 11 Mar 2024 11:30:44 +0800 Subject: [PATCH 13/24] fix: restore --- .../MasaDbContextBuilderExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Contrib/Data/Contracts/Masa.Contrib.Data.Contracts/MasaDbContextBuilderExtensions.cs b/src/Contrib/Data/Contracts/Masa.Contrib.Data.Contracts/MasaDbContextBuilderExtensions.cs index 42917a4b1..4dd83c555 100644 --- a/src/Contrib/Data/Contracts/Masa.Contrib.Data.Contracts/MasaDbContextBuilderExtensions.cs +++ b/src/Contrib/Data/Contracts/Masa.Contrib.Data.Contracts/MasaDbContextBuilderExtensions.cs @@ -3,7 +3,7 @@ // ReSharper disable once CheckNamespace -namespace Masa.BuildingBlocks.Data; +namespace Microsoft.EntityFrameworkCore; public static class MasaDbContextBuilderExtensions { From bc4b4aadacd4373bad6e90444012519f8cd302dd Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Mon, 11 Mar 2024 15:23:27 +0800 Subject: [PATCH 14/24] fix: fix clickhouse query init time --- .../ClickhouseApmServiceTests.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs index 1012f8f55..8b5324e3b 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs @@ -1,15 +1,13 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. -using Masa.Contrib.StackSdks.Tsc.Clickhouse; - namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests; [TestClass] public class ClickhouseApmServiceTests { private static IApmService _APMService; - private static DateTime _start = DateTime.Parse("2024/01/03 22:00:00"); + private static DateTime _start = DateTime.Parse("2024-01-03T22:00:00.000Z"); [ClassInitialize] public static void Initialized(TestContext testContext) @@ -23,6 +21,7 @@ public static void Initialized(TestContext testContext) _APMService = services.BuildServiceProvider().GetRequiredService(); Common.InitTableJsonData(false, AppDomain.CurrentDomain.BaseDirectory, connection); _start -= MasaStackClickhouseConnection.TimeZone.BaseUtcOffset; + _start -= TimeZoneInfo.Local.BaseUtcOffset; } [TestMethod] From 04475e06befbd19caf219271f3a45f3460be4f96 Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Tue, 12 Mar 2024 10:16:24 +0800 Subject: [PATCH 15/24] chore: update version sort --- .github/workflows/Codecov.yml | 4 ++-- .github/workflows/ci_test_reports_to_oss.yml | 4 ++-- .github/workflows/package_push_nuget.org.yml | 4 ++-- .github/workflows/pr_run_test_ci.yml | 5 +++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Codecov.yml b/.github/workflows/Codecov.yml index 0b2e696a8..d9c8f9d3c 100644 --- a/.github/workflows/Codecov.yml +++ b/.github/workflows/Codecov.yml @@ -94,9 +94,9 @@ jobs: uses: actions/setup-dotnet@v1 with: dotnet-version: | - 6.0.x - 7.0.x 8.0.x + 7.0.x + 6.0.x include-prerelease: true diff --git a/.github/workflows/ci_test_reports_to_oss.yml b/.github/workflows/ci_test_reports_to_oss.yml index 76c208e18..4eb76a75d 100644 --- a/.github/workflows/ci_test_reports_to_oss.yml +++ b/.github/workflows/ci_test_reports_to_oss.yml @@ -27,9 +27,9 @@ jobs: uses: actions/setup-dotnet@v1 with: dotnet-version: | - 6.0.x - 7.0.x 8.0.x + 7.0.x + 6.0.x include-prerelease: true # - name: dependencies diff --git a/.github/workflows/package_push_nuget.org.yml b/.github/workflows/package_push_nuget.org.yml index 0eec0a826..65fdf63ec 100644 --- a/.github/workflows/package_push_nuget.org.yml +++ b/.github/workflows/package_push_nuget.org.yml @@ -19,9 +19,9 @@ jobs: uses: actions/setup-dotnet@v1 with: dotnet-version: | - 6.0.x - 7.0.x 8.0.x + 7.0.x + 6.0.x include-prerelease: true - name: restore diff --git a/.github/workflows/pr_run_test_ci.yml b/.github/workflows/pr_run_test_ci.yml index 357b25aae..dcdc174e7 100644 --- a/.github/workflows/pr_run_test_ci.yml +++ b/.github/workflows/pr_run_test_ci.yml @@ -38,9 +38,10 @@ jobs: uses: actions/setup-dotnet@v1 with: dotnet-version: | - 6.0.x - 7.0.x 8.0.x + 7.0.x + 6.0.x + include-prerelease: true # - name: Start Redis From e35ca416e9cf735d76d9ec765dacb77bb558a564 Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Tue, 12 Mar 2024 10:33:59 +0800 Subject: [PATCH 16/24] chore: update --- .../ClickhouseApmServiceTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs index 8b5324e3b..21d096e28 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs @@ -38,8 +38,8 @@ public async Task ServicePageAsync() }; var result = await _APMService.ServicePageAsync(query); Assert.IsNotNull(result); - Assert.IsTrue(result.Total > 0); - Assert.IsNotNull(result.Result); + //Assert.IsTrue(result.Total > 0); + //Assert.IsNotNull(result.Result); } [TestMethod] @@ -57,8 +57,8 @@ public async Task EndpointPageAsync() }; var result = await _APMService.EndpointPageAsync(query); Assert.IsNotNull(result); - Assert.IsTrue(result.Total > 0); - Assert.IsNotNull(result.Result); + //Assert.IsTrue(result.Total > 0); + //Assert.IsNotNull(result.Result); } [TestMethod] From 638132f23776677868e76085104809dfb6df89c7 Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Tue, 12 Mar 2024 13:22:44 +0800 Subject: [PATCH 17/24] chore: update --- .../Cliclhouse/ClickhouseApmService.cs | 77 ++++++++++++++----- .../Extensions/IDbConnectionExtensitions.cs | 76 +++++++++--------- .../MasaTscCliclhouseExtensitions.cs | 4 +- .../ClickhouseApmServiceTests.cs | 8 +- 4 files changed, 104 insertions(+), 61 deletions(-) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs index 558b583d6..f27f0a315 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -5,8 +5,8 @@ namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Cliclhouse; internal class ClickhouseApmService : IApmService { - private MasaStackClickhouseConnection _dbConnection; - private readonly IDbCommand command; + private readonly MasaStackClickhouseConnection _dbConnection; + private readonly ClickHouseCommand command; private readonly ITraceService _traceService; private readonly static object lockObj = new(); private static Dictionary serviceOrders = new Dictionary() { @@ -162,7 +162,7 @@ public Task> ChartDataAsync(BaseApmRequestDto query) return Task.FromResult(result.AsEnumerable()); } - private void GetPreviousChartData(BaseApmRequestDto query, string sql, List parameters, List result) + private void GetPreviousChartData(BaseApmRequestDto query, string sql, List parameters, List result) { if (!query.ComparisonType.HasValue) return; @@ -279,7 +279,7 @@ public Task> ErrorMessagePageAsync(ApmEndpoin return Task.FromResult(result); } - private void SetData(string sql, List parameters, PaginatedListBase result, BaseApmRequestDto query, Func parseFn) where TResult : class + private void SetData(string sql, List parameters, PaginatedListBase result, BaseApmRequestDto query, Func parseFn) where TResult : class { var start = (query.Page - 1) * query.PageSize; if (result.Total - start > 0) @@ -295,9 +295,9 @@ private void SetData(string sql, List parameters, Pag } } - private static (string where, List parameters) AppendWhere(TQuery query) where TQuery : BaseApmRequestDto + private static (string where, List parameters) AppendWhere(TQuery query) where TQuery : BaseApmRequestDto { - List parameters = new(); + List parameters = new(); var sql = new StringBuilder(); sql.AppendLine(" Timestamp between @startTime and @endTime"); parameters.Add(new ClickHouseParameter { ParameterName = "startTime", Value = MasaStackClickhouseConnection.ToTimeZone(query.Start), DbType = DbType.DateTime }); @@ -317,6 +317,8 @@ private static (string where, List parameters) AppendWhere parameters) AppendWhere parameters) + { + if (traceQuery == null || string.IsNullOrEmpty(traceQuery.Endpoint)) + return; + + if (traceQuery.IsLog.HasValue && traceQuery.IsLog.Value) + { + sql.AppendLine(" and indexOf(LogAttributesKeys,'RequestPath')>=0 and LogAttributesValues[indexOf(LogAttributesKeys,'RequestPath')] LIKE @endpoint"); + parameters.Add(new ClickHouseParameter { ParameterName = "endpoint", Value = $"{traceQuery.Endpoint}%" }); + } + else + { + sql.AppendLine(" and Attributes.http.target=@endpoint"); + parameters.Add(new ClickHouseParameter { ParameterName = "endpoint", Value = traceQuery.Endpoint }); + } + } + + private static void AppendDuration(ApmTraceLatencyRequestDto? query, StringBuilder sql, List parameters) + { + if (query == null) return; + if (query.LatMin > 0 && query.LatMax > 0) + { + sql.AppendLine(" and Duration between @minDuration and @maxDuration"); + parameters.Add(new ClickHouseParameter { ParameterName = "minDuration", Value = query.LatMin }); + parameters.Add(new ClickHouseParameter { ParameterName = "maxDuration", Value = query.LatMax }); + } + else if (query.LatMin > 0) + { + sql.AppendLine(" and Duration >=@minDuration"); + parameters.Add(new ClickHouseParameter { ParameterName = "minDuration", Value = query.LatMin }); + } + else if (query.LatMax > 0) + { + sql.AppendLine(" and Duration <=@maxDuration"); + parameters.Add(new ClickHouseParameter { ParameterName = "maxDuration", Value = query.LatMax }); + } + } + public async Task> TraceLatencyDetailAsync(ApmTraceLatencyRequestDto query) { var queryDto = new BaseRequestDto @@ -400,20 +440,20 @@ public async Task> TraceLatencyDetailAsync(A Type = ConditionTypes.LessEqual, Value = query.LatMax.Value, }); - if (conditions.Any()) + if (conditions.Count > 0) queryDto.Conditions = conditions; return await _traceService.ListAsync(queryDto); } - private IDataReader Query(string sql, IEnumerable parameters) + private IDataReader Query(string sql, IEnumerable parameters) { command.CommandText = sql; SetParameters(parameters); return command.ExecuteReader(); } - private object Scalar(string sql, IEnumerable parameters) + private object Scalar(string sql, IEnumerable parameters) { lock (lockObj) { @@ -423,7 +463,7 @@ private object Scalar(string sql, IEnumerable parameters) } } - private void SetParameters(IEnumerable parameters) + private void SetParameters(IEnumerable parameters) { if (command.Parameters.Count > 0) command.Parameters.Clear(); @@ -458,7 +498,6 @@ protected virtual void Dispose(bool isDisposing) { _dbConnection.Close(); _dbConnection.Dispose(); - GC.SuppressFinalize(this); } } @@ -473,10 +512,10 @@ public Task> GetErrorChartAsync(ApmEndpointReques count(1) `total` from {Constants.LogTableFull} where {where} and SeverityText='Error' {groupby}"; - return Task.FromResult(getChartCountData(sql, parameters, query.ComparisonType)); + return Task.FromResult(getChartCountData(sql, parameters, query.ComparisonType).AsEnumerable()); } - private IEnumerable getChartCountData(string sql, IEnumerable parameters, ComparisonTypes? comparisonTypes = null) + private List getChartCountData(string sql, IEnumerable parameters, ComparisonTypes? comparisonTypes = null) { var result = new List(); lock (lockObj) @@ -504,7 +543,7 @@ private IEnumerable getChartCountData(string sql, IEnumerable return result; } - private void SetChartCountData(List result, IDataReader reader, bool isPrevious = false) + private static void SetChartCountData(List result, IDataReader reader, bool isPrevious = false) { if (!reader.NextResult()) return; @@ -550,7 +589,7 @@ public Task> GetEndpointChartAsync(ApmEndpointReq count(1) `total` from {Constants.TraceTable} where {where} {groupby}"; - return Task.FromResult(getChartCountData(sql, parameters, query.ComparisonType)); + return Task.FromResult(getChartCountData(sql, parameters, query.ComparisonType).AsEnumerable()); } public Task> GetLogChartAsync(ApmEndpointRequestDto query) @@ -563,12 +602,12 @@ public Task> GetLogChartAsync(ApmEndpointRequestD toStartOfInterval(`Timestamp` , INTERVAL {GetPeriod(query)} ) as `time`, count(1) `total` from {Constants.LogTableFull} where {where} {groupby}"; - return Task.FromResult(getChartCountData(sql, parameters, query.ComparisonType)); + return Task.FromResult(getChartCountData(sql, parameters, query.ComparisonType).AsEnumerable()); } private string GetPeriod(BaseApmRequestDto query) { - var reg = new Regex(@"/d+"); + var reg = new Regex(@"/d+", default, TimeSpan.FromSeconds(5)); if (string.IsNullOrEmpty(query.Period) || !reg.IsMatch(query.Period)) { return GetDefaultPeriod(query.End - query.Start); @@ -578,10 +617,10 @@ private string GetPeriod(BaseApmRequestDto query) var find = units.FirstOrDefault(s => s.StartsWith(unit)); if (string.IsNullOrEmpty(find)) find = "minute"; - return $"{reg.Match(query.Period).Result} {unit}"; + return $"{reg.Match(query.Period).Result} {find}"; } - private string GetDefaultPeriod(TimeSpan timeSpan) + private static string GetDefaultPeriod(TimeSpan timeSpan) { if ((int)timeSpan.TotalMinutes < 1) { diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs index a6e7383ca..72b35023f 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs @@ -5,6 +5,10 @@ namespace System.Data.Common; internal static class IDbConnectionExtensitions { + const string ATTRIBUTE_KEY = "Attributes."; + const string RESOURCE_KEY = "Resource."; + const string TIMSTAMP_KEY = "Timestamp"; + public static PaginatedListBase QueryTrace(this IDbConnection connection, BaseRequestDto query) { var (where, parameters, ors) = AppendWhere(query); @@ -15,7 +19,7 @@ public static PaginatedListBase QueryTrace(this IDbConnection var result = new PaginatedListBase() { Total = total, Result = new() }; if (total > 0 && start - total < 0) { - var querySql = CombineOrs($"select ServiceName,Timestamp,TraceId,SpanId,ParentSpanId,TraceState,SpanKind,Duration,SpanName,Spans,Resources from {MasaStackClickhouseConnection.TraceTable} where {where}", ors, orderBy); + var querySql = CombineOrs($"select ServiceName,{TIMSTAMP_KEY},TraceId,SpanId,ParentSpanId,TraceState,SpanKind,Duration,SpanName,Spans,Resources from {MasaStackClickhouseConnection.TraceTable} where {where}", ors, orderBy); result.Result = Query(connection, $"select * from {querySql} as t limit {start},{query.PageSize}", parameters?.ToArray(), ConvertTraceDto); } return result; @@ -33,7 +37,7 @@ public static PaginatedListBase QueryLog(this IDbConnection conn if (total > 0 && start - total < 0) { - var querySql = CombineOrs($"select Timestamp,TraceId,SpanId,TraceFlags,SeverityText,SeverityNumber,ServiceName,Body,Resources,Logs from {MasaStackClickhouseConnection.LogTable} where {where}", ors, orderBy); + var querySql = CombineOrs($"select {TIMSTAMP_KEY},TraceId,SpanId,TraceFlags,SeverityText,SeverityNumber,ServiceName,Body,Resources,Logs from {MasaStackClickhouseConnection.LogTable} where {where}", ors, orderBy); result.Result = Query(connection, $"select * from {querySql} as t limit {start},{query.PageSize}", parameters?.ToArray(), ConvertLogDto); } return result; @@ -57,13 +61,13 @@ public static List GetMapping(this IDbConnection dbConnectio { var type = isLog ? "log" : "trace"; var result = dbConnection.Query($"select DISTINCT Name from otel_mapping Array join Name where `Type`='{type}_basic' order by Name", default, ConvertToMapping); - if (result == null || !result.Any()) + if (result == null || result.Count == 0) return default!; - var attributes = dbConnection.Query($"select DISTINCT concat('Attributes.',Name) from otel_mapping Array join Name where `Type`='{type}_attributes' order by Name", default, ConvertToMapping); - var resources = dbConnection.Query("select DISTINCT concat('Resource.',Name) from otel_mapping Array join Name where `Type`='resource' order by Name", default, ConvertToMapping); - if (attributes != null && attributes.Any()) result.AddRange(attributes); - if (resources != null && resources.Any()) result.AddRange(resources); + var attributes = dbConnection.Query($"select DISTINCT concat('{ATTRIBUTE_KEY}',Name) from otel_mapping Array join Name where `Type`='{type}_attributes' order by Name", default, ConvertToMapping); + var resources = dbConnection.Query($"select DISTINCT concat('{RESOURCE_KEY}',Name) from otel_mapping Array join Name where `Type`='resource' order by Name", default, ConvertToMapping); + if (attributes != null && attributes.Count > 0) result.AddRange(attributes); + if (resources != null && resources.Count > 0) result.AddRange(resources); return result; } @@ -71,12 +75,12 @@ public static List GetMapping(this IDbConnection dbConnectio public static List GetTraceByTraceId(this IDbConnection connection, string traceId) { string where = $"TraceId=@TraceId"; - return Query(connection, $"select * from (select Timestamp,TraceId,SpanId,ParentSpanId,TraceState,SpanKind,Duration,SpanName,Spans,Resources from {MasaStackClickhouseConnection.TraceTable} where {where}) as t limit 1000", new IDataParameter[] { new ClickHouseParameter { ParameterName = "TraceId", Value = traceId } }, ConvertTraceDto); + return Query(connection, $"select * from (select {TIMSTAMP_KEY},TraceId,SpanId,ParentSpanId,TraceState,SpanKind,Duration,SpanName,Spans,Resources from {MasaStackClickhouseConnection.TraceTable} where {where}) as t limit 1000", new IDataParameter[] { new ClickHouseParameter { ParameterName = "TraceId", Value = traceId } }, ConvertTraceDto); } public static string AppendOrderBy(BaseRequestDto query, bool isLog) { - string field = "Timestamp"; + var field = TIMSTAMP_KEY; var isDesc = query.Sort?.IsDesc ?? true; if (isLog && query.Sort != null && !string.IsNullOrEmpty(query.Sort.Name)) { @@ -95,7 +99,7 @@ public static (string where, List @parameters, List ors) && query.End > DateTime.MinValue && query.End < DateTime.MaxValue && query.End > query.Start) { - sql.Append($" and Timestamp BETWEEN @Start and @End"); + sql.Append($" and {TIMSTAMP_KEY} BETWEEN @Start and @End"); @paramerters.Add(new ClickHouseParameter() { ParameterName = "Start", Value = MasaStackClickhouseConnection.ToTimeZone(query.Start), DbType = DbType.DateTime2 }); @paramerters.Add(new ClickHouseParameter() { ParameterName = "End", Value = MasaStackClickhouseConnection.ToTimeZone(query.End), DbType = DbType.DateTime2 }); } @@ -106,12 +110,12 @@ public static (string where, List @parameters, List ors) } if (!string.IsNullOrEmpty(query.Instance)) { - sql.Append(" and `Resource.service.instance.id`=@ServiceInstanceId"); + sql.Append($" and `{RESOURCE_KEY}service.instance.id`=@ServiceInstanceId"); @paramerters.Add(new ClickHouseParameter() { ParameterName = "ServiceInstanceId", Value = query.Instance }); } if (isTrace && !string.IsNullOrEmpty(query.Endpoint)) { - sql.Append(" and `Attributes.http.target`=@HttpTarget"); + sql.Append($" and `{ATTRIBUTE_KEY}http.target`=@HttpTarget"); @paramerters.Add(new ClickHouseParameter() { ParameterName = "HttpTarget", Value = query.Endpoint }); } var ors = AppendKeyword(query.Keyword, paramerters, isTrace); @@ -134,8 +138,8 @@ private static List AppendKeyword(string keyword, List @ //status_code if (int.TryParse(keyword, out var num) && num != 0 && num - 1000 < 0 && isTrace) { - sqls.Add(" and `Attributes.http.status_code`=@HttpStatusCode"); - sqls.Add(" and `Attributes.http.request_content_body` like @Keyword"); + sqls.Add($" and `{ATTRIBUTE_KEY}http.status_code`=@HttpStatusCode"); + sqls.Add($" and `{ATTRIBUTE_KEY}http.request_content_body` like @Keyword"); paramerters.Add(new ClickHouseParameter() { ParameterName = "HttpStatusCode", Value = num }); paramerters.Add(new ClickHouseParameter() { ParameterName = "Keyword", Value = $"%{keyword}%" }); return sqls; @@ -143,16 +147,16 @@ private static List AppendKeyword(string keyword, List @ if (isTrace) { - sqls.Add(" and `Attributes.http.request_content_body` like @Keyword"); - sqls.Add(" and `Attributes.http.response_content_body` like @Keyword"); - sqls.Add(" and `Attributes.exception.message` like @Keyword"); + sqls.Add($" and `{ATTRIBUTE_KEY}http.request_content_body` like @Keyword"); + sqls.Add($" and `{ATTRIBUTE_KEY}http.response_content_body` like @Keyword"); + sqls.Add($" and `{ATTRIBUTE_KEY}exception.message` like @Keyword"); } else { if (keyword.Equals("error", StringComparison.CurrentCultureIgnoreCase)) sqls.Add(" and SeverityText='Error'"); sqls.Add(" and Body like @Keyword"); - sqls.Add(" and `Attributes.exception.message` like @Keyword"); + sqls.Add($" and `{ATTRIBUTE_KEY}exception.message` like @Keyword"); } paramerters.Add(new ClickHouseParameter() { ParameterName = "Keyword", Value = $"%{keyword}%" }); return sqls; @@ -171,9 +175,9 @@ private static void AppendConditions(IEnumerable? conditions, { item.Value = MasaStackClickhouseConnection.ToTimeZone(time); } - if (item.Name.StartsWith("resource.", StringComparison.CurrentCultureIgnoreCase)) + if (item.Name.StartsWith(RESOURCE_KEY, StringComparison.CurrentCultureIgnoreCase)) { - var filed = item.Name["resource.".Length..]; + var filed = item.Name[RESOURCE_KEY.Length..]; if (string.Equals(filed, "service.name")) { AppendField(item, @paramerters, sql, name, "ServiceName"); @@ -187,9 +191,9 @@ private static void AppendConditions(IEnumerable? conditions, AppendField(item, @paramerters, sql, name, "ServiceNameSpace"); } } - else if (item.Name.StartsWith("attributes.", StringComparison.CurrentCultureIgnoreCase)) + else if (item.Name.StartsWith(ATTRIBUTE_KEY, StringComparison.CurrentCultureIgnoreCase)) { - var filed = item.Name["attributes.".Length..]; + var filed = item.Name[ATTRIBUTE_KEY.Length..]; AppendField(item, @paramerters, sql, name, filed.Replace('.', '_')); } else @@ -259,7 +263,7 @@ private static void ParseWhere(StringBuilder sql, object value, List 0) foreach (var p in @parameters) cmd.Parameters.Add(p); OpenConnection(dbConnection); @@ -269,7 +273,7 @@ private static void ParseWhere(StringBuilder sql, object value, List Query(this IDbConnection dbConnection, string sql, IDat { using var cmd = dbConnection.CreateCommand(); cmd.CommandText = sql; - if (@parameters != null && @parameters.Any()) + if (@parameters != null && @parameters.Length > 0) foreach (var p in @parameters) cmd.Parameters.Add(p); OpenConnection(dbConnection); @@ -303,7 +307,7 @@ public static List Query(this IDbConnection dbConnection, string sql, IDat } catch (Exception ex) { - MasaTscCliclhouseExtensitions.Logger?.LogError(ex, "query sql error:{rawSql}, paramters:{parameters}", sql, parameters); + MasaTscCliclhouseExtensitions.Logger?.LogError(ex, "query sql error:{RawSql}, paramters:{Parameters}", sql, parameters); throw; } } @@ -319,7 +323,7 @@ public static MappingResponseDto ConvertToMapping(IDataReader reader) public static TraceResponseDto ConvertTraceDto(IDataReader reader) { - var startTime = Convert.ToDateTime(reader["Timestamp"]); + var startTime = Convert.ToDateTime(reader[TIMSTAMP_KEY]); long ns = Convert.ToInt64(reader["Duration"]); string resource = reader["Resources"].ToString()!, spans = reader["Spans"].ToString()!; var result = new TraceResponseDto @@ -350,7 +354,7 @@ public static LogResponseDto ConvertLogDto(IDataReader reader) SeverityText = reader["SeverityText"].ToString()!, TraceFlags = Convert.ToInt32(reader["TraceFlags"]), SpanId = reader["SpanId"].ToString()!, - Timestamp = Convert.ToDateTime(reader["Timestamp"]), + Timestamp = Convert.ToDateTime(reader[TIMSTAMP_KEY]), }; if (!string.IsNullOrEmpty(resource)) result.Resource = JsonSerializer.Deserialize>(resource)!; @@ -445,7 +449,7 @@ private static void AppendAggtype(SimpleAggregateRequestDto requestDto, StringBu private static string GetName(string name, bool isLog) { if (name.Equals("@timestamp", StringComparison.CurrentCultureIgnoreCase)) - return "Timestamp"; + return TIMSTAMP_KEY; if (!isLog && name.Equals("duration", StringComparison.CurrentCultureIgnoreCase)) return "Duration"; @@ -453,10 +457,10 @@ private static string GetName(string name, bool isLog) if (!isLog && name.Equals("kind", StringComparison.InvariantCultureIgnoreCase)) return "SpanKind"; - if (name.StartsWith("resource.", StringComparison.CurrentCultureIgnoreCase)) + if (name.StartsWith(RESOURCE_KEY, StringComparison.CurrentCultureIgnoreCase)) return GetResourceName(name); - if (name.StartsWith("attributes.", StringComparison.CurrentCultureIgnoreCase)) + if (name.StartsWith(ATTRIBUTE_KEY, StringComparison.CurrentCultureIgnoreCase)) return GetAttributeName(name, isLog); return name; @@ -464,12 +468,12 @@ private static string GetName(string name, bool isLog) private static string GetResourceName(string name) { - var field = name[("resource.".Length)..]; + var field = name[(RESOURCE_KEY.Length)..]; if (field.Equals("service.name", StringComparison.CurrentCultureIgnoreCase)) return "ServiceName"; if (field.Equals("service.namespace", StringComparison.CurrentCultureIgnoreCase) || field.Equals("service.instance.id", StringComparison.CurrentCultureIgnoreCase)) - return $"Resource.{field}"; + return $"{RESOURCE_KEY}{field}"; return $"ResourceAttributesValues[indexOf(ResourceAttributesKeys,'{field}')]"; } @@ -477,9 +481,9 @@ private static string GetResourceName(string name) private static string GetAttributeName(string name, bool isLog) { var pre = isLog ? "Log" : "Span"; - var field = name[("attributes.".Length)..]; + var field = name[(ATTRIBUTE_KEY.Length)..]; if (isLog && (field.Equals("exception.message", StringComparison.CurrentCultureIgnoreCase))) - return $"Attributes.{field}"; + return $"{ATTRIBUTE_KEY}{field}"; if (!isLog && (field.Equals("http.status_code", StringComparison.CurrentCultureIgnoreCase) || field.Equals("http.request_content_body", StringComparison.CurrentCultureIgnoreCase) @@ -487,7 +491,7 @@ private static string GetAttributeName(string name, bool isLog) || field.Equals("exception.message", StringComparison.CurrentCultureIgnoreCase) || field.Equals("http.target", StringComparison.CurrentCultureIgnoreCase)) ) - return $"Attributes.{field}"; + return $"{ATTRIBUTE_KEY}{field}"; return $"{pre}AttributesValues[indexOf({pre}AttributesKeys,'{field}')]"; } diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/MasaTscCliclhouseExtensitions.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/MasaTscCliclhouseExtensitions.cs index c8c88b97e..9c7f28cdc 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/MasaTscCliclhouseExtensitions.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/MasaTscCliclhouseExtensitions.cs @@ -250,7 +250,7 @@ internal static void ExecuteSql(this IDbConnection connection, string sql) } catch (Exception ex) { - Logger?.LogError(ex, "ExecuteSql {rawSql} error", sql); + Logger?.LogError(ex, "ExecuteSql {RawSql} error", sql); } } @@ -267,7 +267,7 @@ private static string GetTimezone(MasaStackClickhouseConnection connection) } catch (Exception ex) { - Logger?.LogError(ex, "ExecuteSql {rawSql} error", sql); + Logger?.LogError(ex, "ExecuteSql {RawSql} error", sql); throw; } } diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs index 21d096e28..8b5324e3b 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs @@ -38,8 +38,8 @@ public async Task ServicePageAsync() }; var result = await _APMService.ServicePageAsync(query); Assert.IsNotNull(result); - //Assert.IsTrue(result.Total > 0); - //Assert.IsNotNull(result.Result); + Assert.IsTrue(result.Total > 0); + Assert.IsNotNull(result.Result); } [TestMethod] @@ -57,8 +57,8 @@ public async Task EndpointPageAsync() }; var result = await _APMService.EndpointPageAsync(query); Assert.IsNotNull(result); - //Assert.IsTrue(result.Total > 0); - //Assert.IsNotNull(result.Result); + Assert.IsTrue(result.Total > 0); + Assert.IsNotNull(result.Result); } [TestMethod] From c72a57e04faa218df8944979dfd83cbb52581b88 Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Tue, 12 Mar 2024 15:14:37 +0800 Subject: [PATCH 18/24] chore: update bad code and update tests --- .../Cliclhouse/ClickhouseApmService.cs | 12 ++++++------ .../Models/Request/ApmTraceLatencyRequestDto.cs | 6 ++++++ .../ClickhouseApmServiceTests.cs | 11 +++++++++-- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs index f27f0a315..44faa1f49 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -387,18 +387,18 @@ private static void AppendDuration(ApmTraceLatencyRequestDto? query, StringBuild if (query.LatMin > 0 && query.LatMax > 0) { sql.AppendLine(" and Duration between @minDuration and @maxDuration"); - parameters.Add(new ClickHouseParameter { ParameterName = "minDuration", Value = query.LatMin }); - parameters.Add(new ClickHouseParameter { ParameterName = "maxDuration", Value = query.LatMax }); + parameters.Add(new ClickHouseParameter { ParameterName = "minDuration", Value = (long)(query.LatMin * MILLSECOND) }); + parameters.Add(new ClickHouseParameter { ParameterName = "maxDuration", Value = (long)(query.LatMax * MILLSECOND) }); } else if (query.LatMin > 0) { sql.AppendLine(" and Duration >=@minDuration"); - parameters.Add(new ClickHouseParameter { ParameterName = "minDuration", Value = query.LatMin }); + parameters.Add(new ClickHouseParameter { ParameterName = "minDuration", Value = (long)(query.LatMin * MILLSECOND) }); } else if (query.LatMax > 0) { sql.AppendLine(" and Duration <=@maxDuration"); - parameters.Add(new ClickHouseParameter { ParameterName = "maxDuration", Value = query.LatMax }); + parameters.Add(new ClickHouseParameter { ParameterName = "maxDuration", Value = (long)(query.LatMax * MILLSECOND) }); } } @@ -427,7 +427,7 @@ public async Task> TraceLatencyDetailAsync(A { Name = "Duration", Type = ConditionTypes.GreatEqual, - Value = query.LatMin.Value, + Value = (long)(query.LatMin.Value * MILLSECOND), }); } @@ -438,7 +438,7 @@ public async Task> TraceLatencyDetailAsync(A { Name = "Duration", Type = ConditionTypes.LessEqual, - Value = query.LatMax.Value, + Value = (long)(query.LatMax.Value * MILLSECOND), }); if (conditions.Count > 0) queryDto.Conditions = conditions; diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmTraceLatencyRequestDto.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmTraceLatencyRequestDto.cs index 14b8264bf..eef88f80b 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmTraceLatencyRequestDto.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmTraceLatencyRequestDto.cs @@ -5,8 +5,14 @@ namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Request; public class ApmTraceLatencyRequestDto : ApmEndpointRequestDto { + /// + /// unit ms + /// public long? LatMin { get; set; } + /// + /// unit ms + /// public long? LatMax { get; set; } public new int PageSize { get; } = 1; diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs index 8b5324e3b..e03cff62d 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs @@ -19,6 +19,8 @@ public static void Initialized(TestContext testContext) services.AddLogging(builder => builder.AddConsole()); services.AddMASAStackApmClickhouse(TestUtils.ConnectionString, "custom_log", "custom_trace"); _APMService = services.BuildServiceProvider().GetRequiredService(); + //首次测试数据可能还没写入，连续写入两次就能保证数据一定写入成功 + Common.InitTableJsonData(false, AppDomain.CurrentDomain.BaseDirectory, connection); Common.InitTableJsonData(false, AppDomain.CurrentDomain.BaseDirectory, connection); _start -= MasaStackClickhouseConnection.TimeZone.BaseUtcOffset; _start -= TimeZoneInfo.Local.BaseUtcOffset; @@ -76,6 +78,7 @@ public async Task ChartDataAsync() }; var result = await _APMService.ChartDataAsync(query); Assert.IsNotNull(result); + Assert.IsTrue(result.Count() > 0); } [TestMethod] @@ -94,6 +97,7 @@ public async Task EndpointLatencyDistributionAsync() }; var result = await _APMService.EndpointLatencyDistributionAsync(query); Assert.IsNotNull(result); + Assert.IsNotNull(result.Latencies); } [TestMethod] @@ -111,7 +115,6 @@ public async Task ErrorMessagePageAsync() }; var result = await _APMService.ErrorMessagePageAsync(query); Assert.IsNotNull(result); - Assert.IsNotNull(result.Total > 0); } [TestMethod] @@ -129,9 +132,13 @@ public async Task TraceLatencyDetailAsync() }; var result = await _APMService.TraceLatencyDetailAsync(query); Assert.IsNotNull(result); + Assert.IsTrue(result.Total > 0); + Assert.IsNotNull(result.Result); query.Env = "Development"; - query.LatMax = 1000000; + query.LatMax = 10_000;//10s result = await _APMService.TraceLatencyDetailAsync(query); Assert.IsNotNull(result); + Assert.IsTrue(result.Total > 0); + Assert.IsNotNull(result.Result); } } From 3804fa5e56741c32c4f0de560f54c2bd1d8cafa9 Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Tue, 12 Mar 2024 15:38:49 +0800 Subject: [PATCH 19/24] chore: update clickhouse tests --- .../TestUtils.cs | 2 +- .../Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Common.cs | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs index 0118a4bd5..636292590 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/TestUtils.cs @@ -5,6 +5,6 @@ namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests; internal class TestUtils { - public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=5000;Host=localhost;Port=9000;Database=default;User=default"; + public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=10000;Host=localhost;Port=9000;Database=default;User=default"; //public const string ConnectionString = "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=10000;Host=192.168.51.234;Port=19003;User=test;Password=123456;Database=default"; } diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Common.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Common.cs index 921021189..41197fa56 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Common.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests/Common.cs @@ -46,6 +46,10 @@ public static void InitTableJsonData(bool isLog, string rootPath, IDbConnection if (connection.State == ConnectionState.Closed) connection.Open(); using var cmd = connection.CreateCommand(); + cmd.CommandText = $"select count() from otel_{name}s"; + var count = Convert.ToInt32(cmd.ExecuteScalar()); + if (count > 0) + return; var path = Path.Combine(rootPath, $"Data/otel_{name}_data.json"); using var dataReader = new StreamReader(path); var data = dataReader.ReadToEnd(); @@ -78,7 +82,7 @@ public static void InitTableJsonData(bool isLog, string rootPath, IDbConnection else if (item.Value.TryGetDateTime(out var time)) sql.Append($"'{time.ToString("yyyy-MM-dd HH:mm:ss.ffff")}',"); else - sql.Append($"'{item.Value.ToString().Replace("\\'", "''").Replace("'", "''").Replace("@","")}',"); + sql.Append($"'{item.Value.ToString().Replace("\\'", "''").Replace("'", "''").Replace("@", "")}',"); } sql.Remove(sql.Length - 1, 1).AppendLine("),"); isFirst = false; From e9a2cc9a921de4d9f2bd04c19679f6fa88dc272e Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Tue, 12 Mar 2024 16:13:50 +0800 Subject: [PATCH 20/24] chore: update --- .../Cliclhouse/ClickhouseApmService.cs | 90 +++++-------------- .../ClickhouseApmServiceTests.cs | 2 +- 2 files changed, 22 insertions(+), 70 deletions(-) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs index 44faa1f49..8ae51b627 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -3,7 +3,7 @@ namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Cliclhouse; -internal class ClickhouseApmService : IApmService +internal class ClickhouseApmService : IApmService, IDisposable { private readonly MasaStackClickhouseConnection _dbConnection; private readonly ClickHouseCommand command; @@ -320,40 +320,6 @@ private static (string where, List parameters) AppendWhere< AppendEndpoint(query as ApmEndpointRequestDto, sql, parameters); AppendDuration(query as ApmTraceLatencyRequestDto, sql, parameters); - if (query is ApmEndpointRequestDto traceQuery && !string.IsNullOrEmpty(traceQuery.Endpoint)) - { - if (query.IsLog.HasValue && query.IsLog.Value) - { - sql.AppendLine(" and indexOf(LogAttributesKeys,'RequestPath')>=0 and LogAttributesValues[indexOf(LogAttributesKeys,'RequestPath')] LIKE @endpoint"); - parameters.Add(new ClickHouseParameter { ParameterName = "endpoint", Value = $"{traceQuery.Endpoint}%" }); - } - else - { - sql.AppendLine(" and Attributes.http.target=@endpoint"); - parameters.Add(new ClickHouseParameter { ParameterName = "endpoint", Value = traceQuery.Endpoint }); - } - } - - if (query is ApmTraceLatencyRequestDto durationQuery) - { - if (durationQuery.LatMin > 0 && durationQuery.LatMax > 0) - { - sql.AppendLine(" and Duration between @minDuration and @maxDuration"); - parameters.Add(new ClickHouseParameter { ParameterName = "minDuration", Value = durationQuery.LatMin }); - parameters.Add(new ClickHouseParameter { ParameterName = "maxDuration", Value = durationQuery.LatMax }); - } - else if (durationQuery.LatMin > 0) - { - sql.AppendLine(" and Duration >=@minDuration"); - parameters.Add(new ClickHouseParameter { ParameterName = "minDuration", Value = durationQuery.LatMin }); - } - else if (durationQuery.LatMax > 0) - { - sql.AppendLine(" and Duration <=@maxDuration"); - parameters.Add(new ClickHouseParameter { ParameterName = "maxDuration", Value = durationQuery.LatMax }); - } - } - if (!string.IsNullOrEmpty(query.Queries) && query.Queries.Trim().Length > 0) { if (!query.Queries.Trim().StartsWith("and ", StringComparison.CurrentCultureIgnoreCase)) @@ -368,34 +334,28 @@ private static void AppendEndpoint(ApmEndpointRequestDto? traceQuery, StringBuil { if (traceQuery == null || string.IsNullOrEmpty(traceQuery.Endpoint)) return; - + var name = "endpoint"; if (traceQuery.IsLog.HasValue && traceQuery.IsLog.Value) { - sql.AppendLine(" and indexOf(LogAttributesKeys,'RequestPath')>=0 and LogAttributesValues[indexOf(LogAttributesKeys,'RequestPath')] LIKE @endpoint"); - parameters.Add(new ClickHouseParameter { ParameterName = "endpoint", Value = $"{traceQuery.Endpoint}%" }); + sql.AppendLine($" and indexOf(LogAttributesKeys,'RequestPath')>=0 and LogAttributesValues[indexOf(LogAttributesKeys,'RequestPath')] LIKE @{name}"); + parameters.Add(new ClickHouseParameter { ParameterName = name, Value = $"{traceQuery.Endpoint}%" }); } else { - sql.AppendLine(" and Attributes.http.target=@endpoint"); - parameters.Add(new ClickHouseParameter { ParameterName = "endpoint", Value = traceQuery.Endpoint }); + sql.AppendLine($" and Attributes.http.target=@{name}"); + parameters.Add(new ClickHouseParameter { ParameterName = name, Value = traceQuery.Endpoint }); } } private static void AppendDuration(ApmTraceLatencyRequestDto? query, StringBuilder sql, List parameters) { - if (query == null) return; - if (query.LatMin > 0 && query.LatMax > 0) - { - sql.AppendLine(" and Duration between @minDuration and @maxDuration"); - parameters.Add(new ClickHouseParameter { ParameterName = "minDuration", Value = (long)(query.LatMin * MILLSECOND) }); - parameters.Add(new ClickHouseParameter { ParameterName = "maxDuration", Value = (long)(query.LatMax * MILLSECOND) }); - } - else if (query.LatMin > 0) + if (query == null || !query.LatMin.HasValue && !query.LatMax.HasValue) return; + if (query.LatMin.HasValue && query.LatMin > 0) { sql.AppendLine(" and Duration >=@minDuration"); parameters.Add(new ClickHouseParameter { ParameterName = "minDuration", Value = (long)(query.LatMin * MILLSECOND) }); } - else if (query.LatMax > 0) + if (query.LatMax.HasValue && query.LatMax > 0) { sql.AppendLine(" and Duration <=@maxDuration"); parameters.Add(new ClickHouseParameter { ParameterName = "maxDuration", Value = (long)(query.LatMax * MILLSECOND) }); @@ -421,11 +381,12 @@ public async Task> TraceLatencyDetailAsync(A Value = query.Env }); } + var name = "Duration"; if (query.LatMin.HasValue && query.LatMin.Value >= 0) { conditions.Add(new FieldConditionDto { - Name = "Duration", + Name = name, Type = ConditionTypes.GreatEqual, Value = (long)(query.LatMin.Value * MILLSECOND), }); @@ -436,7 +397,7 @@ public async Task> TraceLatencyDetailAsync(A || query.LatMin.HasValue && query.LatMax - query.LatMin.Value > 0)) conditions.Add(new FieldConditionDto { - Name = "Duration", + Name = name, Type = ConditionTypes.LessEqual, Value = (long)(query.LatMax.Value * MILLSECOND), }); @@ -474,11 +435,11 @@ private void SetParameters(IEnumerable parameters) private static string? GetOrderBy(BaseApmRequestDto query, Dictionary sortFields, string? defaultSort = null, bool isDesc = false) { - if (!string.IsNullOrEmpty(query.OrderField) && sortFields.ContainsKey(query.OrderField)) + if (!string.IsNullOrEmpty(query.OrderField) && sortFields.TryGetValue(query.OrderField, out var field)) { if (!query.IsDesc.HasValue) - return $"order by {sortFields[query.OrderField]}"; - return $"order by {sortFields[query.OrderField]}{(query.IsDesc.Value ? "" : " desc")}"; + return $"order by {field}"; + return $"order by {field}{(query.IsDesc.Value ? "" : " desc")}"; } if (string.IsNullOrEmpty(defaultSort)) @@ -486,19 +447,10 @@ private void SetParameters(IEnumerable parameters) return $"order by {defaultSort}{(isDesc ? " desc" : "")}"; } - void IDisposable.Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - protected virtual void Dispose(bool isDisposing) + public void Dispose() { - if (isDisposing) - { - _dbConnection.Close(); - _dbConnection.Dispose(); - } + _dbConnection.Close(); + _dbConnection.Dispose(); } public Task> GetErrorChartAsync(ApmEndpointRequestDto query) @@ -605,7 +557,7 @@ public Task> GetLogChartAsync(ApmEndpointRequestD return Task.FromResult(getChartCountData(sql, parameters, query.ComparisonType).AsEnumerable()); } - private string GetPeriod(BaseApmRequestDto query) + private static string GetPeriod(BaseApmRequestDto query) { var reg = new Regex(@"/d+", default, TimeSpan.FromSeconds(5)); if (string.IsNullOrEmpty(query.Period) || !reg.IsMatch(query.Period)) @@ -613,8 +565,8 @@ private string GetPeriod(BaseApmRequestDto query) return GetDefaultPeriod(query.End - query.Start); } var unit = reg.Replace(query.Period, "").Trim().ToLower(); - var units = new string[] { "year", "month", "week", "day", "hour", "minute", "second" }; - var find = units.FirstOrDefault(s => s.StartsWith(unit)); + var units = new List { "year", "month", "week", "day", "hour", "minute", "second" }; + var find = units.Find(s => s.StartsWith(unit)); if (string.IsNullOrEmpty(find)) find = "minute"; return $"{reg.Match(query.Period).Result} {find}"; diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs index e03cff62d..c972bd41b 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs @@ -78,7 +78,7 @@ public async Task ChartDataAsync() }; var result = await _APMService.ChartDataAsync(query); Assert.IsNotNull(result); - Assert.IsTrue(result.Count() > 0); + Assert.IsTrue(result.Any()); } [TestMethod] From b66aba3cbdfc0623a1873a8a6e75f13ddbf94a6b Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Tue, 12 Mar 2024 17:16:14 +0800 Subject: [PATCH 21/24] chore: update --- .../Cliclhouse/ClickhouseApmService.cs | 70 ++++++++++--------- .../ClickhouseApmServiceTests.cs | 4 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs index 8ae51b627..57f9c0dd0 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -3,13 +3,13 @@ namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Cliclhouse; -internal class ClickhouseApmService : IApmService, IDisposable +internal class ClickhouseApmService : IApmService { private readonly MasaStackClickhouseConnection _dbConnection; private readonly ClickHouseCommand command; private readonly ITraceService _traceService; private readonly static object lockObj = new(); - private static Dictionary serviceOrders = new Dictionary() { + private static Dictionary serviceOrders = new() { {nameof(ServiceListDto.Name),"ServiceName"}, {nameof(ServiceListDto.Envs),"env"}, {nameof(ServiceListDto.Latency),"latency"}, @@ -17,7 +17,7 @@ internal class ClickhouseApmService : IApmService, IDisposable {nameof(ServiceListDto.Failed),"failed"}, }; - private static Dictionary endpointOrders = new Dictionary() { + private static Dictionary endpointOrders = new() { {nameof(EndpointListDto.Name),"`Attributes.http.target`"}, {nameof(EndpointListDto.Service),"ServiceName"}, {nameof(EndpointListDto.Method),"`method`"}, @@ -26,7 +26,7 @@ internal class ClickhouseApmService : IApmService, IDisposable {nameof(EndpointListDto.Failed),"failed"}, }; - private static Dictionary errorOrders = new Dictionary() { + private static Dictionary errorOrders = new() { {nameof(ErrorMessageDto.Type),"Type"}, {nameof(ErrorMessageDto.Message),"Message"}, {nameof(ErrorMessageDto.LastTime),"`time`"}, @@ -72,57 +72,50 @@ public Task> ServicePageAsync(BaseApmRequestDt public Task> InstancePageAsync(BaseApmRequestDto query) { - var (where, parameters) = AppendWhere(query); - var groupby = "group by instance"; - var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; - PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; - var orderBy = GetOrderBy(query, new()); - var sql = $@"select * from( select ResourceAttributesValues[indexOf(ResourceAttributesKeys,'service.instance.id')] instance`, + var groupBy = "group by instance"; + var selectField = @"ResourceAttributesValues[indexOf(ResourceAttributesKeys,'service.instance.id')] instance`, AVG(Duration/{MILLSECOND}) Latency, count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)) throughput -sum(has(['{string.Join("','", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))/count(1) failed -from {where} {groupby} {orderBy} @limit)"; - SetData(sql, parameters, result, query, reader => new EndpointListDto() +sum(has(['{string.Join(""','"", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))/count(1) failed"; + return GetEndpointAsync(query, groupBy, selectField, reader => new EndpointListDto() { Name = reader[0].ToString()!, Latency = (long)Math.Floor(Convert.ToDouble(reader[1])), Throughput = Math.Round(Convert.ToDouble(reader[2]), 2), Failed = Math.Round(Convert.ToDouble(reader[3]), 2) }); - return Task.FromResult(result); } public Task> DependencyPageAsync(BaseApmRequestDto query) + { + var groupBy = "group by ServiceName,`Attributes.http.target`,`method`"; + var selectField = @"`Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, +AVG(Duration{MILLSECOND}) Latency, +count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)) throughput +sum(has(['{string.Join("",'"", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))/count(1) failed"; + return GetEndpointAsync(query, groupBy, selectField, ConvertEndpointDto); + } + + private Task> GetEndpointAsync(BaseApmRequestDto query, string groupBy, string selectField, Func parseFn) { var (where, parameters) = AppendWhere(query); - var groupby = "group by ServiceName,`Attributes.http.target`,`method`"; - var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; + var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupBy})"; PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; var orderBy = GetOrderBy(query, new()); - var sql = $@"select * from( select `Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, -AVG(Duration{MILLSECOND}) Latency, -count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)) throughput -sum(has(['{string.Join(",'", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))/count(1) failed -from {where} {groupby} {orderBy} @limit)"; - SetData(sql, parameters, result, query, ConvertEndpointDto); + var sql = $@"select * from( select {selectField} from {where} {groupBy} {orderBy} @limit)"; + SetData(sql, parameters, result, query, parseFn); return Task.FromResult(result); } public Task> EndpointPageAsync(BaseApmRequestDto query) { query.IsServer = true; - var (where, parameters) = AppendWhere(query); - var groupby = "group by ServiceName,`Attributes.http.target`,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')]"; - var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupby})"; - PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; - var orderBy = GetOrderBy(query, endpointOrders); - var sql = $@"select * from( select `Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, + var groupBy = "group by ServiceName,`Attributes.http.target`,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')]"; + var selectField = @"`Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, floor(AVG(Duration/{MILLSECOND})) latency, round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) throughput, -round(sum(has(['{string.Join("','", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))*100.0/count(1),2) failed -from {Constants.TraceTableFull} where {where} and Attributes.http.target!='' {groupby} {orderBy} @limit)"; - SetData(sql, parameters, result, query, ConvertEndpointDto); - return Task.FromResult(result); +round(sum(has(['{string.Join(""','"", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))*100.0/count(1),2) failed"; + return GetEndpointAsync(query, groupBy, selectField, ConvertEndpointDto); } private EndpointListDto ConvertEndpointDto(IDataReader reader) @@ -449,8 +442,17 @@ private void SetParameters(IEnumerable parameters) public void Dispose() { - _dbConnection.Close(); - _dbConnection.Dispose(); + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + _dbConnection.Close(); + _dbConnection.Dispose(); + } } public Task> GetErrorChartAsync(ApmEndpointRequestDto query) diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs index c972bd41b..fcac84221 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs @@ -19,9 +19,7 @@ public static void Initialized(TestContext testContext) services.AddLogging(builder => builder.AddConsole()); services.AddMASAStackApmClickhouse(TestUtils.ConnectionString, "custom_log", "custom_trace"); _APMService = services.BuildServiceProvider().GetRequiredService(); - //首次测试数据可能还没写入，连续写入两次就能保证数据一定写入成功 - Common.InitTableJsonData(false, AppDomain.CurrentDomain.BaseDirectory, connection); - Common.InitTableJsonData(false, AppDomain.CurrentDomain.BaseDirectory, connection); + Common.InitTableJsonData(false, AppDomain.CurrentDomain.BaseDirectory, connection); _start -= MasaStackClickhouseConnection.TimeZone.BaseUtcOffset; _start -= TimeZoneInfo.Local.BaseUtcOffset; } From 78985ddb14c665739dceadf43dd20a0f72be3f5e Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Tue, 12 Mar 2024 17:33:33 +0800 Subject: [PATCH 22/24] chore: update --- .../Cliclhouse/ClickhouseApmService.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs index 57f9c0dd0..4276924cf 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -73,10 +73,10 @@ public Task> ServicePageAsync(BaseApmRequestDt public Task> InstancePageAsync(BaseApmRequestDto query) { var groupBy = "group by instance"; - var selectField = @"ResourceAttributesValues[indexOf(ResourceAttributesKeys,'service.instance.id')] instance`, + var selectField = $@"ResourceAttributesValues[indexOf(ResourceAttributesKeys,'service.instance.id')] instance`, AVG(Duration/{MILLSECOND}) Latency, count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)) throughput -sum(has(['{string.Join(""','"", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))/count(1) failed"; +sum(has(['{string.Join(',', query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))/count(1) failed"; return GetEndpointAsync(query, groupBy, selectField, reader => new EndpointListDto() { Name = reader[0].ToString()!, @@ -89,10 +89,10 @@ public Task> InstancePageAsync(BaseApmRequest public Task> DependencyPageAsync(BaseApmRequestDto query) { var groupBy = "group by ServiceName,`Attributes.http.target`,`method`"; - var selectField = @"`Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, + var selectField = $@"`Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, AVG(Duration{MILLSECOND}) Latency, count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)) throughput -sum(has(['{string.Join("",'"", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))/count(1) failed"; +sum(has(['{string.Join(',', query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))/count(1) failed"; return GetEndpointAsync(query, groupBy, selectField, ConvertEndpointDto); } @@ -102,7 +102,7 @@ private Task> GetEndpointAsync(BaseApmRequest var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupBy})"; PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; var orderBy = GetOrderBy(query, new()); - var sql = $@"select * from( select {selectField} from {where} {groupBy} {orderBy} @limit)"; + var sql = $@"select * from( select {selectField} from {Constants.TraceTableFull} where {where} {groupBy} {orderBy} @limit)"; SetData(sql, parameters, result, query, parseFn); return Task.FromResult(result); } @@ -111,10 +111,10 @@ public Task> EndpointPageAsync(BaseApmRequest { query.IsServer = true; var groupBy = "group by ServiceName,`Attributes.http.target`,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')]"; - var selectField = @"`Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, + var selectField = $@"`Attributes.http.target`,ServiceName,SpanAttributesValues[indexOf(SpanAttributesKeys,'http.method')] `method`, floor(AVG(Duration/{MILLSECOND})) latency, round(count(1)*1.0/DATEDIFF(MINUTE ,toDateTime(@startTime),toDateTime (@endTime)),2) throughput, -round(sum(has(['{string.Join(""','"", query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))*100.0/count(1),2) failed"; +round(sum(has(['{string.Join(',', query.GetErrorStatusCodes())}'],`Attributes.http.status_code`))*100.0/count(1),2) failed"; return GetEndpointAsync(query, groupBy, selectField, ConvertEndpointDto); } From 76b5aecff7e7d3e2e4ec341fe2897b6f807cc2fa Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Tue, 12 Mar 2024 17:51:53 +0800 Subject: [PATCH 23/24] chore: update tests --- .../ClickhouseApmServiceTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs index fcac84221..95cfbfa22 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests/ClickhouseApmServiceTests.cs @@ -19,7 +19,8 @@ public static void Initialized(TestContext testContext) services.AddLogging(builder => builder.AddConsole()); services.AddMASAStackApmClickhouse(TestUtils.ConnectionString, "custom_log", "custom_trace"); _APMService = services.BuildServiceProvider().GetRequiredService(); - Common.InitTableJsonData(false, AppDomain.CurrentDomain.BaseDirectory, connection); + Common.InitTableJsonData(false, AppDomain.CurrentDomain.BaseDirectory, connection); + Common.InitTableJsonData(false, AppDomain.CurrentDomain.BaseDirectory, connection); _start -= MasaStackClickhouseConnection.TimeZone.BaseUtcOffset; _start -= TimeZoneInfo.Local.BaseUtcOffset; } From 1aa3f437fd4b55aa4ed38c4df6cf6be213368b56 Mon Sep 17 00:00:00 2001 From: qinyouzeng Date: Wed, 13 Mar 2024 09:12:04 +0800 Subject: [PATCH 24/24] chore: update --- .../Cliclhouse/ClickhouseApmService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs index 4276924cf..bf3e28eb6 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -101,7 +101,7 @@ private Task> GetEndpointAsync(BaseApmRequest var (where, parameters) = AppendWhere(query); var countSql = $"select count(1) from(select count(1) from {Constants.TraceTableFull} where {where} {groupBy})"; PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; - var orderBy = GetOrderBy(query, new()); + var orderBy = GetOrderBy(query, endpointOrders); var sql = $@"select * from( select {selectField} from {Constants.TraceTableFull} where {where} {groupBy} {orderBy} @limit)"; SetData(sql, parameters, result, query, parseFn); return Task.FromResult(result);