From 4a0040ed9d29be400ea998f0fa1e0cb1a08d4fee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BA=D0=BF=D0=B0=D0=B5=D0=B2=20=D0=95=D0=B2=D0=B3?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B0?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 18 May 2021 12:16:48 +0300 Subject: [PATCH] develop branch has been created --- .../ClickHouse/ClickHouseStorage.cs | 162 ++++++------ .../ElasticSearch/AuthenticationType.cs | 2 +- .../ElasticSearch/ElasticSearchHost.cs | 2 +- .../ElasticSearch/ElasticSearchStorage.cs | 239 +++++++++--------- .../ElasticSearchStorageSettings.cs | 6 +- .../EventLogExporter.cs | 102 ++++---- .../EventLogPosition.cs | 12 +- .../FileLogger.cs | 26 +- .../FileLoggerProvider.cs | 11 +- .../IEventLogStorage.cs | 6 +- .../ILoggerBuilderExtensions.cs | 4 +- .../OneSTools.EventLog.Exporter.Core.csproj | 3 +- .../StorageType.cs | 10 +- .../ClstEventArgs.cs | 10 +- .../ClstWatcher.cs | 45 ++-- .../ExportersManager.cs | 101 +++++--- ...OneSTools.EventLog.Exporter.Manager.csproj | 2 +- .../Program.cs | 19 +- .../Properties/launchSettings.json | 2 +- .../appsettings.Development.json | 2 +- .../appsettings.json | 2 +- OneSTools.EventLog.Exporter.Manager/log.txt | 22 -- .../EventLogExporterService.cs | 19 +- .../OneSTools.EventLog.Exporter.csproj | 10 +- OneSTools.EventLog.Exporter/Program.cs | 14 +- .../appsettings.Development.json | 8 +- OneSTools.EventLog.Exporter/appsettings.json | 2 +- OneSTools.EventLog/DateTimeZoneExtensions.cs | 12 +- OneSTools.EventLog/EventLogItem.cs | 6 +- OneSTools.EventLog/EventLogReader.cs | 78 +++--- OneSTools.EventLog/EventLogReaderSettings.cs | 6 +- .../EventLogReaderTimeoutException.cs | 4 +- OneSTools.EventLog/LgfReader.cs | 76 +++--- OneSTools.EventLog/LgpReader.cs | 167 ++++++------ OneSTools.EventLog/ObjectType.cs | 2 +- OneSTools.EventLog/OneSTools.EventLog.csproj | 6 +- OneSTools.EventLog/StreamReaderExtensions.cs | 21 +- 37 files changed, 612 insertions(+), 609 deletions(-) delete mode 100644 OneSTools.EventLog.Exporter.Manager/log.txt diff --git a/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs b/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs index 44c9b3b..236d813 100644 --- a/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs +++ b/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs @@ -1,14 +1,13 @@ -using ClickHouse.Client.ADO; -using ClickHouse.Client.Copy; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using System; +using System; using System.Collections.Generic; -using System.Data; using System.Linq; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using ClickHouse.Client.ADO; +using ClickHouse.Client.Copy; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; namespace OneSTools.EventLog.Exporter.Core.ClickHouse { @@ -16,9 +15,9 @@ public class ClickHouseStorage : IEventLogStorage { private const string TableName = "EventLogItems"; private readonly ILogger _logger; + private ClickHouseConnection _connection; private string _connectionString; private string _databaseName; - private ClickHouseConnection _connection; public ClickHouseStorage(string connectionsString, ILogger logger = null) { @@ -36,6 +35,81 @@ public ClickHouseStorage(ILogger logger, IConfiguration confi Init(); } + public async Task ReadEventLogPositionAsync(CancellationToken cancellationToken = default) + { + await CreateConnectionAsync(cancellationToken); + + var commandText = + $"SELECT TOP 1 FileName, EndPosition, LgfEndPosition, Id FROM {TableName} ORDER BY Id DESC"; + + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = commandText; + + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + + if (await reader.ReadAsync(cancellationToken)) + return new EventLogPosition(reader.GetString(0), reader.GetInt64(1), reader.GetInt64(2), + reader.GetInt64(3)); + return null; + } + + public async Task WriteEventLogDataAsync(List entities, + CancellationToken cancellationToken = default) + { + await CreateConnectionAsync(cancellationToken); + + using var copy = new ClickHouseBulkCopy(_connection) + { + DestinationTableName = TableName, + BatchSize = entities.Count + }; + + var data = entities.Select(item => new object[] + { + item.FileName ?? "", + item.EndPosition, + item.LgfEndPosition, + item.Id, + item.DateTime, + item.TransactionStatus ?? "", + item.TransactionDateTime == DateTime.MinValue ? new DateTime(1970, 1, 1) : item.TransactionDateTime, + item.TransactionNumber, + item.UserUuid ?? "", + item.User ?? "", + item.Computer ?? "", + item.Application ?? "", + item.Connection, + item.Event ?? "", + item.Severity ?? "", + item.Comment ?? "", + item.MetadataUuid ?? "", + item.Metadata ?? "", + item.Data ?? "", + item.DataPresentation ?? "", + item.Server ?? "", + item.MainPort, + item.AddPort, + item.Session + }).AsEnumerable(); + + try + { + await copy.WriteToServerAsync(data, cancellationToken); + } + catch (Exception ex) + { + _logger?.LogError(ex, $"Failed to write data to {_databaseName}"); + throw; + } + + _logger?.LogDebug($"{entities.Count} items were being written to {_databaseName}"); + } + + public void Dispose() + { + _connection?.Dispose(); + } + private void Init() { if (_connectionString == string.Empty) @@ -106,77 +180,5 @@ ORDER BY (DateTime, EndPosition) cmd.CommandText = commandText; await cmd.ExecuteNonQueryAsync(cancellationToken); } - - public async Task ReadEventLogPositionAsync(CancellationToken cancellationToken = default) - { - await CreateConnectionAsync(cancellationToken); - - var commandText = $"SELECT TOP 1 FileName, EndPosition, LgfEndPosition, Id FROM {TableName} ORDER BY Id DESC"; - - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = commandText; - - await using var reader = await cmd.ExecuteReaderAsync(cancellationToken); - - if (await reader.ReadAsync(cancellationToken)) - return new EventLogPosition(reader.GetString(0), reader.GetInt64(1), reader.GetInt64(2), reader.GetInt64(3)); - else - return null; - } - - public async Task WriteEventLogDataAsync(List entities, CancellationToken cancellationToken = default) - { - await CreateConnectionAsync(cancellationToken); - - using var copy = new ClickHouseBulkCopy(_connection) - { - DestinationTableName = TableName, - BatchSize = entities.Count - }; - - var data = entities.Select(item => new object[] { - item.FileName ?? "", - item.EndPosition, - item.LgfEndPosition, - item.Id, - item.DateTime, - item.TransactionStatus ?? "", - item.TransactionDateTime == DateTime.MinValue ? new DateTime(1970, 1, 1) : item.TransactionDateTime, - item.TransactionNumber, - item.UserUuid ?? "", - item.User ?? "", - item.Computer ?? "", - item.Application ?? "", - item.Connection, - item.Event ?? "", - item.Severity ?? "", - item.Comment ?? "", - item.MetadataUuid ?? "", - item.Metadata ?? "", - item.Data ?? "", - item.DataPresentation ?? "", - item.Server ?? "", - item.MainPort, - item.AddPort, - item.Session - }).AsEnumerable(); - - try - { - await copy.WriteToServerAsync(data, cancellationToken); - } - catch (Exception ex) - { - _logger?.LogError(ex, $"Failed to write data to {_databaseName}"); - throw; - } - - _logger?.LogDebug($"{entities.Count} items were being written to {_databaseName}"); - } - - public void Dispose() - { - _connection?.Dispose(); - } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Core/ElasticSearch/AuthenticationType.cs b/OneSTools.EventLog.Exporter.Core/ElasticSearch/AuthenticationType.cs index ec39aee..45124a8 100644 --- a/OneSTools.EventLog.Exporter.Core/ElasticSearch/AuthenticationType.cs +++ b/OneSTools.EventLog.Exporter.Core/ElasticSearch/AuthenticationType.cs @@ -6,4 +6,4 @@ public enum AuthenticationType Basic = 1, ApiKey = 2 } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Core/ElasticSearch/ElasticSearchHost.cs b/OneSTools.EventLog.Exporter.Core/ElasticSearch/ElasticSearchHost.cs index 71112eb..aa545cd 100644 --- a/OneSTools.EventLog.Exporter.Core/ElasticSearch/ElasticSearchHost.cs +++ b/OneSTools.EventLog.Exporter.Core/ElasticSearch/ElasticSearchHost.cs @@ -33,4 +33,4 @@ public override int GetHashCode() return !(left == right); } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Core/ElasticSearch/ElasticSearchStorage.cs b/OneSTools.EventLog.Exporter.Core/ElasticSearch/ElasticSearchStorage.cs index e413284..4d96ff9 100644 --- a/OneSTools.EventLog.Exporter.Core/ElasticSearch/ElasticSearchStorage.cs +++ b/OneSTools.EventLog.Exporter.Core/ElasticSearch/ElasticSearchStorage.cs @@ -1,17 +1,12 @@ using System; using System.Collections.Generic; -using System.Text; +using System.Linq; using System.Threading; using System.Threading.Tasks; -using OneSTools.EventLog.Exporter.Core; -using System.Linq; -using System.Data; using Elasticsearch.Net; -using Nest; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using System.Globalization; +using Nest; namespace OneSTools.EventLog.Exporter.Core.ElasticSearch { @@ -19,15 +14,15 @@ public class ElasticSearchStorage : IEventLogStorage { public static int DefaultMaximumRetries = 2; public static int DefaultMaxRetryTimeoutSec = 30; + private readonly string _eventLogItemsIndex; private readonly ILogger _logger; - private readonly List _nodes = new List(); - private ElasticSearchNode _currentNode; - private readonly string _eventLogItemsIndex; private readonly int _maximumRetries; private readonly TimeSpan _maxRetryTimeout; + private readonly List _nodes = new List(); private readonly string _separation; private ElasticClient _client; + private ElasticSearchNode _currentNode; public ElasticSearchStorage(ElasticSearchStorageSettings settings, ILogger logger = null) { @@ -50,11 +45,112 @@ public ElasticSearchStorage(ILogger logger, IConfiguration _eventLogItemsIndex = configuration.GetValue("ElasticSearch:Index", ""); _separation = configuration.GetValue("ElasticSearch:Separation", "H"); _maximumRetries = configuration.GetValue("ElasticSearch:MaximumRetries", DefaultMaximumRetries); - _maxRetryTimeout = TimeSpan.FromSeconds(configuration.GetValue("ElasticSearch:MaxRetryTimeout", DefaultMaxRetryTimeoutSec)); + _maxRetryTimeout = + TimeSpan.FromSeconds(configuration.GetValue("ElasticSearch:MaxRetryTimeout", + DefaultMaxRetryTimeoutSec)); CheckSettings(); } + public async Task ReadEventLogPositionAsync(CancellationToken cancellationToken = default) + { + if (_client is null) + await ConnectAsync(cancellationToken); + + while (true) + { + var response = await _client.SearchAsync(sd => sd + .Index($"{_eventLogItemsIndex}-*") + .Sort(ss => + ss.Descending(c => c.Id)) + .Size(1) + , cancellationToken); + + if (response.IsValid) + { + var item = response.Documents.FirstOrDefault(); + + if (item is null) + return null; + return new EventLogPosition(item.FileName, item.EndPosition, item.LgfEndPosition, item.Id); + } + + if (response.OriginalException is TaskCanceledException) + throw response.OriginalException; + + _logger?.LogError( + $"Failed to get last file's position ({_eventLogItemsIndex}): {response.OriginalException.Message}"); + + var currentNodeHost = _currentNode.Host; + + await ConnectAsync(cancellationToken); + + // If it's the same node then wait while MaxRetryTimeout occurs, otherwise it'll be a too often request's loop + if (_currentNode.Host.Equals(currentNodeHost)) + await Task.Delay(_maxRetryTimeout, cancellationToken); + } + } + + public async Task WriteEventLogDataAsync(List entities, + CancellationToken cancellationToken = default) + { + if (_client is null) + await ConnectAsync(cancellationToken); + + var data = GetGroupedData(entities); + + for (var i = 0; i < data.Count; i++) + { + if (cancellationToken.IsCancellationRequested) + return; + + var item = data[i]; + + var responseItems = await _client.IndexManyAsync(item.Entities, item.IndexName, cancellationToken); + + if (!responseItems.ApiCall.Success) + { + if (responseItems.OriginalException is TaskCanceledException) + throw responseItems.OriginalException; + + if (responseItems.Errors) + { + foreach (var itemWithError in responseItems.ItemsWithErrors) + _logger?.LogError( + $"Failed to index document {itemWithError.Id} in {item.IndexName}: {itemWithError.Error}"); + + throw new Exception( + $"Failed to write items to {item.IndexName}: {responseItems.OriginalException.Message}"); + } + + _logger?.LogError( + $"Failed to write items to {item.IndexName}: {responseItems.OriginalException.Message}"); + + await ConnectAsync(cancellationToken); + + i--; + } + else + { + if (responseItems.Errors) + { + foreach (var itemWithError in responseItems.ItemsWithErrors) + _logger?.LogError( + $"Failed to index document {itemWithError.Id} in {item.IndexName}: {itemWithError.Error}"); + + throw new Exception( + $"Failed to write items to {item.IndexName}: {responseItems.OriginalException.Message}"); + } + + _logger?.LogDebug($"{item.Entities.Count} items were being written to {item.IndexName}"); + } + } + } + + public void Dispose() + { + } + private void CheckSettings() { if (_nodes.Count == 0) @@ -83,12 +179,13 @@ private async Task CreateIndexTemplateAsync(CancellationToken cancellationToken { var indexTemplateName = "oneslogs"; - var getItResponse = await _client.LowLevel.DoRequestAsync(HttpMethod.GET, $"_index_template/{indexTemplateName}", cancellationToken); + var getItResponse = await _client.LowLevel.DoRequestAsync(HttpMethod.GET, + $"_index_template/{indexTemplateName}", cancellationToken); // if it exists then skip creating if (!getItResponse.Success) throw getItResponse.OriginalException; - else if (getItResponse.HttpStatusCode != 404) + if (getItResponse.HttpStatusCode != 404) return; var cmd = @@ -129,7 +226,8 @@ private async Task CreateIndexTemplateAsync(CancellationToken cancellationToken } }"; - var response = await _client.LowLevel.DoRequestAsync(HttpMethod.PUT, $"_index_template/{indexTemplateName}", cancellationToken, PostData.String(cmd)); + var response = await _client.LowLevel.DoRequestAsync(HttpMethod.PUT, + $"_index_template/{indexTemplateName}", cancellationToken, PostData.String(cmd)); if (!response.Success) throw response.OriginalException; @@ -138,7 +236,9 @@ private async Task CreateIndexTemplateAsync(CancellationToken cancellationToken private async Task SwitchToNextNodeAsync(CancellationToken cancellationToken = default) { if (_currentNode == null) + { _currentNode = _nodes[0]; + } else { var currentIndex = _nodes.IndexOf(_currentNode); @@ -161,8 +261,6 @@ private async Task SwitchToNextNodeAsync(CancellationToken cancellationTok case AuthenticationType.ApiKey: settings.ApiKeyAuthentication(_currentNode.Id, _currentNode.ApiKey); break; - default: - break; } _client = new ElasticClient(settings); @@ -174,7 +272,8 @@ private async Task SwitchToNextNodeAsync(CancellationToken cancellationTok if (!(response.OriginalException is TaskCanceledException)) { if (!response.IsValid) - _logger?.LogWarning($"Failed to connect to {uri} ({_eventLogItemsIndex}): {response.OriginalException.Message}"); + _logger?.LogWarning( + $"Failed to connect to {uri} ({_eventLogItemsIndex}): {response.OriginalException.Message}"); else _logger?.LogInformation($"Successfully connected to {uri} ({_eventLogItemsIndex})"); } @@ -182,47 +281,6 @@ private async Task SwitchToNextNodeAsync(CancellationToken cancellationTok return response.IsValid; } - public async Task ReadEventLogPositionAsync(CancellationToken cancellationToken = default) - { - if (_client is null) - await ConnectAsync(cancellationToken); - - while (true) - { - var response = await _client.SearchAsync(sd => sd - .Index($"{_eventLogItemsIndex}-*") - .Sort(ss => - ss.Descending(c => c.Id)) - .Size(1) - , cancellationToken); - - if (response.IsValid) - { - var item = response.Documents.FirstOrDefault(); - - if (item is null) - return null; - else - return new EventLogPosition(item.FileName, item.EndPosition, item.LgfEndPosition, item.Id); - } - else - { - if (response.OriginalException is TaskCanceledException) - throw response.OriginalException; - - _logger?.LogError($"Failed to get last file's position ({_eventLogItemsIndex}): {response.OriginalException.Message}"); - - var currentNodeHost = _currentNode.Host; - - await ConnectAsync(cancellationToken); - - // If it's the same node then wait while MaxRetryTimeout occurs, otherwise it'll be a too often request's loop - if (_currentNode.Host.Equals(currentNodeHost)) - await Task.Delay(_maxRetryTimeout, cancellationToken); - } - } - } - private List<(string IndexName, List Entities)> GetGroupedData(List entities) { var data = new List<(string IndexName, List Entities)>(); @@ -231,17 +289,17 @@ public async Task ReadEventLogPositionAsync(CancellationToken { case "H": var groups = entities.GroupBy(c => c.DateTime.ToString("yyyyMMddhh")).OrderBy(c => c.Key); - foreach (IGrouping item in groups) + foreach (var item in groups) data.Add(($"{_eventLogItemsIndex}-{item.Key}", item.ToList())); break; case "D": groups = entities.GroupBy(c => c.DateTime.ToString("yyyyMMdd")).OrderBy(c => c.Key); - foreach (IGrouping item in groups) + foreach (var item in groups) data.Add(($"{_eventLogItemsIndex}-{item.Key}", item.ToList())); break; case "M": groups = entities.GroupBy(c => c.DateTime.ToString("yyyyMM")).OrderBy(c => c.Key); - foreach (IGrouping item in groups) + foreach (var item in groups) data.Add(($"{_eventLogItemsIndex}-{item.Key}", item.ToList())); break; default: @@ -251,62 +309,5 @@ public async Task ReadEventLogPositionAsync(CancellationToken return data; } - - public async Task WriteEventLogDataAsync(List entities, CancellationToken cancellationToken = default) - { - if (_client is null) - await ConnectAsync(cancellationToken); - - var data = GetGroupedData(entities); - - for (int i = 0; i < data.Count; i++) - { - if (cancellationToken.IsCancellationRequested) - return; - - var item = data[i]; - - var responseItems = await _client.IndexManyAsync(item.Entities, item.IndexName, cancellationToken); - - if (!responseItems.ApiCall.Success) - { - if (responseItems.OriginalException is TaskCanceledException) - throw responseItems.OriginalException; - - if (responseItems.Errors) - { - foreach (var itemWithError in responseItems.ItemsWithErrors) - _logger?.LogError($"Failed to index document {itemWithError.Id} in {item.IndexName}: {itemWithError.Error}"); - - throw new Exception($"Failed to write items to {item.IndexName}: {responseItems.OriginalException.Message}"); - } - else - { - _logger?.LogError($"Failed to write items to {item.IndexName}: {responseItems.OriginalException.Message}"); - - await ConnectAsync(cancellationToken); - - i--; - } - } - else - { - if (responseItems.Errors) - { - foreach (var itemWithError in responseItems.ItemsWithErrors) - _logger?.LogError($"Failed to index document {itemWithError.Id} in {item.IndexName}: {itemWithError.Error}"); - - throw new Exception($"Failed to write items to {item.IndexName}: {responseItems.OriginalException.Message}"); - } - else - _logger?.LogDebug($"{item.Entities.Count} items were being written to {item.IndexName}"); - } - } - } - - public void Dispose() - { - - } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Core/ElasticSearch/ElasticSearchStorageSettings.cs b/OneSTools.EventLog.Exporter.Core/ElasticSearch/ElasticSearchStorageSettings.cs index d4db04f..695e78a 100644 --- a/OneSTools.EventLog.Exporter.Core/ElasticSearch/ElasticSearchStorageSettings.cs +++ b/OneSTools.EventLog.Exporter.Core/ElasticSearch/ElasticSearchStorageSettings.cs @@ -9,6 +9,8 @@ public class ElasticSearchStorageSettings public string Index { get; set; } = ""; public string Separation { get; set; } = ""; public int MaximumRetries { get; set; } = ElasticSearchStorage.DefaultMaximumRetries; - public TimeSpan MaxRetryTimeout { get; set; } = TimeSpan.FromSeconds(ElasticSearchStorage.DefaultMaxRetryTimeoutSec); + + public TimeSpan MaxRetryTimeout { get; set; } = + TimeSpan.FromSeconds(ElasticSearchStorage.DefaultMaxRetryTimeoutSec); } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Core/EventLogExporter.cs b/OneSTools.EventLog.Exporter.Core/EventLogExporter.cs index 3d1e06b..3f31706 100644 --- a/OneSTools.EventLog.Exporter.Core/EventLogExporter.cs +++ b/OneSTools.EventLog.Exporter.Core/EventLogExporter.cs @@ -1,5 +1,4 @@ using System; -using System.Diagnostics; using System.IO; using System.Linq; using System.Threading; @@ -11,41 +10,31 @@ namespace OneSTools.EventLog.Exporter.Core { - public class EventLogExporterSettings - { - public string LogFolder { get; set; } = ""; - public int Portion { get; set; } = 10000; - public DateTimeZone TimeZone { get; set; } = DateTimeZoneProviders.Tzdb.GetSystemDefault(); - public int WritingMaxDop { get; set; } = 1; - public int CollectedFactor { get; set; } = 2; - public int ReadingTimeout { get; set; } = 1; - public bool LoadArchive { get; set; } = false; - } - public class EventLogExporter { - private readonly ILogger _logger; - private readonly IEventLogStorage _storage; + private readonly int _collectedFactor; + private readonly bool _loadArchive; // Exporter settings private readonly string _logFolder; + private readonly ILogger _logger; private readonly int _portion; + private readonly int _readingTimeout; + private readonly IEventLogStorage _storage; private readonly DateTimeZone _timeZone = DateTimeZoneProviders.Tzdb.GetSystemDefault(); private readonly int _writingMaxDop; - private readonly int _collectedFactor; - private readonly bool _loadArchive; - private readonly int _readingTimeout; + private BatchBlock _batchBlock; private string _currentLgpFile; - // Dataflow blocks + private bool _disposedValue; + + // DataFlow blocks private EventLogReader _eventLogReader; private ActionBlock _writeBlock; - private BatchBlock _batchBlock; - private bool _disposedValue; - - public EventLogExporter(EventLogExporterSettings settings, IEventLogStorage storage, ILogger logger = null) + public EventLogExporter(EventLogExporterSettings settings, IEventLogStorage storage, + ILogger logger = null) { _logger = logger; _storage = storage; @@ -61,7 +50,8 @@ public EventLogExporter(EventLogExporterSettings settings, IEventLogStorage stor CheckSettings(); } - public EventLogExporter(ILogger logger, IConfiguration configuration, IEventLogStorage storage) + public EventLogExporter(ILogger logger, IConfiguration configuration, + IEventLogStorage storage) { _logger = logger; _storage = storage; @@ -94,10 +84,10 @@ private void CheckSettings() throw new Exception($"Event log folder ({_logFolder}) doesn't exist"); if (_writingMaxDop <= 0) - throw new Exception($"WritingMaxDegreeOfParallelism cannot be equal to or less than 0"); + throw new Exception("WritingMaxDegreeOfParallelism cannot be equal to or less than 0"); if (_collectedFactor <= 0) - throw new Exception($"CollectedFactor cannot be equal to or less than 0"); + throw new Exception("CollectedFactor cannot be equal to or less than 0"); } public async Task StartAsync(CancellationToken cancellationToken = default) @@ -105,7 +95,7 @@ public async Task StartAsync(CancellationToken cancellationToken = default) _logger?.LogInformation($"Log folder: {_logFolder}"); if (_loadArchive) - _logger?.LogWarning($"\"Load archive\" mode enabled"); + _logger?.LogWarning("\"Load archive\" mode enabled"); _logger?.LogInformation($"Portion per request: {_portion}"); @@ -118,7 +108,7 @@ public async Task StartAsync(CancellationToken cancellationToken = default) while (!cancellationToken.IsCancellationRequested) { - bool forceSending = false; + var forceSending = false; EventLogItem item = null; @@ -140,7 +130,8 @@ public async Task StartAsync(CancellationToken cancellationToken = default) { await SendAsync(_batchBlock, item, cancellationToken); - if (!string.IsNullOrEmpty(_eventLogReader.LgpFileName) && _currentLgpFile != _eventLogReader.LgpFileName) + if (!string.IsNullOrEmpty(_eventLogReader.LgpFileName) && + _currentLgpFile != _eventLogReader.LgpFileName) { _logger?.LogInformation($"Reader started reading {_eventLogReader.LgpFileName}"); @@ -148,34 +139,40 @@ public async Task StartAsync(CancellationToken cancellationToken = default) } } else if (!settings.LiveMode) + { forceSending = true; + } if (forceSending) _batchBlock.TriggerBatch(); } } - catch (TaskCanceledException) { } + catch (TaskCanceledException) + { + } } private void InitializeDataflow(CancellationToken cancellationToken = default) { - var writeBlockSettings = new ExecutionDataflowBlockOptions() + var writeBlockSettings = new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, BoundedCapacity = _collectedFactor, MaxDegreeOfParallelism = _writingMaxDop }; - var batchBlockSettings = new GroupingDataflowBlockOptions() + var batchBlockSettings = new GroupingDataflowBlockOptions { CancellationToken = cancellationToken, BoundedCapacity = _portion * _collectedFactor }; - _writeBlock = new ActionBlock(c => _storage.WriteEventLogDataAsync(c.ToList(), cancellationToken), writeBlockSettings); + _writeBlock = + new ActionBlock(c => _storage.WriteEventLogDataAsync(c.ToList(), cancellationToken), + writeBlockSettings); _batchBlock = new BatchBlock(_portion, batchBlockSettings); - _batchBlock.LinkTo(_writeBlock, new DataflowLinkOptions() { PropagateCompletion = true }); + _batchBlock.LinkTo(_writeBlock, new DataflowLinkOptions {PropagateCompletion = true}); } private async Task GetReaderSettingsAsync(CancellationToken cancellationToken = default) @@ -197,7 +194,10 @@ private async Task GetReaderSettingsAsync(CancellationTo var lgpFilePath = Path.Combine(_logFolder, position.FileName); if (!File.Exists(lgpFilePath)) - _logger?.LogWarning($"Lgp file ({lgpFilePath}) doesn't exist. The reading will be started from the first found file"); + { + _logger?.LogWarning( + $"Lgp file ({lgpFilePath}) doesn't exist. The reading will be started from the first found file"); + } else { eventLogReaderSettings.LgpFileName = position.FileName; @@ -205,15 +205,19 @@ private async Task GetReaderSettingsAsync(CancellationTo eventLogReaderSettings.LgfStartPosition = position.LgfEndPosition; eventLogReaderSettings.ItemId = position.Id; - _logger?.LogInformation($"File {position.FileName} will be read from {position.EndPosition} position, LGF file will be read from {position.LgfEndPosition} position"); + _logger?.LogInformation( + $"File {position.FileName} will be read from {position.EndPosition} position, LGF file will be read from {position.LgfEndPosition} position"); } } else - _logger?.LogInformation($"There're no log items in the database, first found log file will be read from 0 position"); + { + _logger?.LogInformation( + "There're no log items in the database, first found log file will be read from 0 position"); + } } else { - _logger?.LogWarning($"LoadArchive parameter is true. Live mode will not be used"); + _logger?.LogWarning("LoadArchive parameter is true. Live mode will not be used"); eventLogReaderSettings.LiveMode = false; } @@ -221,38 +225,34 @@ private async Task GetReaderSettingsAsync(CancellationTo return eventLogReaderSettings; } - private async Task SendAsync(ITargetBlock nextBlock, EventLogItem item, CancellationToken stoppingToken = default) + private static async Task SendAsync(ITargetBlock nextBlock, EventLogItem item, + CancellationToken stoppingToken = default) { while (!stoppingToken.IsCancellationRequested && !nextBlock.Completion.IsFaulted) - { if (await nextBlock.SendAsync(item, stoppingToken)) break; - } } protected virtual void Dispose(bool disposing) { - if (!_disposedValue) - { - if (disposing) - { - _storage?.Dispose(); - } + if (_disposedValue) + return; - _eventLogReader?.Dispose(); + if (disposing) _storage?.Dispose(); - _disposedValue = true; - } + _eventLogReader?.Dispose(); + + _disposedValue = true; } ~EventLogExporter() { - Dispose(disposing: false); + Dispose(false); } public void Dispose() { - Dispose(disposing: true); + Dispose(true); GC.SuppressFinalize(this); } } diff --git a/OneSTools.EventLog.Exporter.Core/EventLogPosition.cs b/OneSTools.EventLog.Exporter.Core/EventLogPosition.cs index d5153c7..6f7eba0 100644 --- a/OneSTools.EventLog.Exporter.Core/EventLogPosition.cs +++ b/OneSTools.EventLog.Exporter.Core/EventLogPosition.cs @@ -2,11 +2,6 @@ { public class EventLogPosition { - public string FileName { get; } - public long EndPosition { get; } - public long LgfEndPosition { get; } - public long Id { get; } - public EventLogPosition(string fileName, long endPosition, long lgfEndPosition, long id) { FileName = fileName; @@ -14,5 +9,10 @@ public EventLogPosition(string fileName, long endPosition, long lgfEndPosition, LgfEndPosition = lgfEndPosition; Id = id; } + + public string FileName { get; } + public long EndPosition { get; } + public long LgfEndPosition { get; } + public long Id { get; } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Core/FileLogger.cs b/OneSTools.EventLog.Exporter.Core/FileLogger.cs index 513d56e..28c0e83 100644 --- a/OneSTools.EventLog.Exporter.Core/FileLogger.cs +++ b/OneSTools.EventLog.Exporter.Core/FileLogger.cs @@ -1,16 +1,14 @@ -using Microsoft.Extensions.Logging; -using System; -using System.Collections.Generic; +using System; using System.IO; -using System.Text; +using Microsoft.Extensions.Logging; namespace OneSTools.EventLog.Exporter.Core { public class FileLogger : ILogger { private static readonly object Locker = new object(); - private readonly string _path; private readonly string _categoryName; + private readonly string _path; public FileLogger(string path, string categoryName) { @@ -19,19 +17,27 @@ public FileLogger(string path, string categoryName) } public IDisposable BeginScope(TState state) - => null; + { + return null; + } public bool IsEnabled(LogLevel logLevel) - => true; + { + return true; + } - public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, + Func formatter) { var levelName = Enum.GetName(typeof(LogLevel), logLevel); - var message = $"{DateTime.Now:yyyy-MM-hh HH:mm:ss.fff} | {levelName} | {_categoryName}[{eventId.Id}]\n\t{ formatter(state, exception)}"; + var message = + $"{DateTime.Now:yyyy-MM-hh HH:mm:ss.fff} | {levelName} | {_categoryName}[{eventId.Id}]\n\t{formatter(state, exception)}"; lock (Locker) + { File.AppendAllText(_path, message + Environment.NewLine); + } } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Core/FileLoggerProvider.cs b/OneSTools.EventLog.Exporter.Core/FileLoggerProvider.cs index f54137b..a8b7143 100644 --- a/OneSTools.EventLog.Exporter.Core/FileLoggerProvider.cs +++ b/OneSTools.EventLog.Exporter.Core/FileLoggerProvider.cs @@ -7,14 +7,17 @@ public class FileLoggerProvider : ILoggerProvider private readonly string _path; public FileLoggerProvider(string path) - => _path = path; + { + _path = path; + } public ILogger CreateLogger(string categoryName) - => new FileLogger(_path, categoryName); + { + return new FileLogger(_path, categoryName); + } public void Dispose() { - } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Core/IEventLogStorage.cs b/OneSTools.EventLog.Exporter.Core/IEventLogStorage.cs index 511a322..6c1a423 100644 --- a/OneSTools.EventLog.Exporter.Core/IEventLogStorage.cs +++ b/OneSTools.EventLog.Exporter.Core/IEventLogStorage.cs @@ -1,7 +1,5 @@ -using OneSTools.EventLog; -using System; +using System; using System.Collections.Generic; -using System.Text; using System.Threading; using System.Threading.Tasks; @@ -12,4 +10,4 @@ public interface IEventLogStorage : IDisposable Task ReadEventLogPositionAsync(CancellationToken cancellationToken = default); Task WriteEventLogDataAsync(List entities, CancellationToken cancellationToken = default); } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Core/ILoggerBuilderExtensions.cs b/OneSTools.EventLog.Exporter.Core/ILoggerBuilderExtensions.cs index a1181f4..88794d1 100644 --- a/OneSTools.EventLog.Exporter.Core/ILoggerBuilderExtensions.cs +++ b/OneSTools.EventLog.Exporter.Core/ILoggerBuilderExtensions.cs @@ -7,8 +7,8 @@ public static class LoggerBuilderExtensions public static ILoggingBuilder AddFile(this ILoggingBuilder loggingBuilder, string path) { var provider = new FileLoggerProvider(path); - + return loggingBuilder.AddProvider(provider); } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Core/OneSTools.EventLog.Exporter.Core.csproj b/OneSTools.EventLog.Exporter.Core/OneSTools.EventLog.Exporter.Core.csproj index 016c194..dde28c2 100644 --- a/OneSTools.EventLog.Exporter.Core/OneSTools.EventLog.Exporter.Core.csproj +++ b/OneSTools.EventLog.Exporter.Core/OneSTools.EventLog.Exporter.Core.csproj @@ -7,7 +7,7 @@ Akpaev Evgeny Базовый пакет экспортеров ЖР Akpaev Evgeny - 1.1.0 + 1.1.1 @@ -32,7 +32,6 @@ - diff --git a/OneSTools.EventLog.Exporter.Core/StorageType.cs b/OneSTools.EventLog.Exporter.Core/StorageType.cs index 16c6cea..439eedc 100644 --- a/OneSTools.EventLog.Exporter.Core/StorageType.cs +++ b/OneSTools.EventLog.Exporter.Core/StorageType.cs @@ -1,10 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace OneSTools.EventLog.Exporter.Core +namespace OneSTools.EventLog.Exporter.Core { public enum StorageType { @@ -12,4 +6,4 @@ public enum StorageType ClickHouse, ElasticSearch } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Manager/ClstEventArgs.cs b/OneSTools.EventLog.Exporter.Manager/ClstEventArgs.cs index ed7de5a..6455661 100644 --- a/OneSTools.EventLog.Exporter.Manager/ClstEventArgs.cs +++ b/OneSTools.EventLog.Exporter.Manager/ClstEventArgs.cs @@ -2,15 +2,15 @@ { public class ClstEventArgs { - public string Path { get; } - public string Name { get; } - public string DataBaseName { get; } - internal ClstEventArgs(string path, string name, string databaseName) { Path = path; Name = name; DataBaseName = databaseName; } + + public string Path { get; } + public string Name { get; } + public string DataBaseName { get; } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Manager/ClstWatcher.cs b/OneSTools.EventLog.Exporter.Manager/ClstWatcher.cs index 044d2b4..8bd6deb 100644 --- a/OneSTools.EventLog.Exporter.Manager/ClstWatcher.cs +++ b/OneSTools.EventLog.Exporter.Manager/ClstWatcher.cs @@ -1,28 +1,24 @@ -using OneSTools.BracketsFile; -using System; +using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.IO; using System.Linq; using System.Text.RegularExpressions; +using OneSTools.BracketsFile; namespace OneSTools.EventLog.Exporter.Manager { public class ClstWatcher : IDisposable { - private readonly string _folder; - private readonly string _path; - private readonly List _templates; - private Dictionary _infoBases; - private FileSystemWatcher _clstWatcher; - public delegate void InfoBaseAddedHandler(object sender, ClstEventArgs args); - public event InfoBaseAddedHandler InfoBasesAdded; public delegate void InfoBaseDeletedHandler(object sender, ClstEventArgs args); - public event InfoBaseDeletedHandler InfoBasesDeleted; - public ReadOnlyDictionary InfoBases => new(_infoBases); + private readonly string _folder; + private readonly string _path; + private readonly List _templates; + private FileSystemWatcher _clstWatcher; + private Dictionary _infoBases; public ClstWatcher(string folder, List templates) { @@ -36,6 +32,17 @@ public ClstWatcher(string folder, List templates) InitializeWatcher(); } + public ReadOnlyDictionary InfoBases => new(_infoBases); + + public void Dispose() + { + _clstWatcher?.Dispose(); + GC.SuppressFinalize(this); + } + + public event InfoBaseAddedHandler InfoBasesAdded; + public event InfoBaseDeletedHandler InfoBasesDeleted; + private Dictionary ReadInfoBases() { var items = new Dictionary(); @@ -47,16 +54,14 @@ public ClstWatcher(string folder, List templates) int count = infoBasesNode[0]; if (count > 0) - { for (var i = 1; i <= count; i++) { var infoBaseNode = infoBasesNode[i]; - string elPath = Path.Combine(_folder, infoBaseNode[0]); + var elPath = Path.Combine(_folder, infoBaseNode[0]); string name = infoBaseNode[5]; foreach (var template in _templates) - { if (Regex.IsMatch(name, template.Mask)) { var dataBaseName = template.Template.Replace("[IBNAME]", name); @@ -64,9 +69,7 @@ public ClstWatcher(string folder, List templates) break; } - } } - } return items; } @@ -99,13 +102,9 @@ private void InitializeWatcher() private void ClstWatcher_Changed(object sender, FileSystemEventArgs e) { lock (InfoBases) + { ReadInfoBasesAndRaiseEvents(); - } - - public void Dispose() - { - _clstWatcher?.Dispose(); - GC.SuppressFinalize(this); + } } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Manager/ExportersManager.cs b/OneSTools.EventLog.Exporter.Manager/ExportersManager.cs index f7004a8..d927757 100644 --- a/OneSTools.EventLog.Exporter.Manager/ExportersManager.cs +++ b/OneSTools.EventLog.Exporter.Manager/ExportersManager.cs @@ -1,43 +1,50 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using NodaTime; -using OneSTools.EventLog.Exporter.Core.ClickHouse; -using OneSTools.EventLog.Exporter.Core; -using OneSTools.EventLog.Exporter.Core.ElasticSearch; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using NodaTime; +using OneSTools.EventLog.Exporter.Core; +using OneSTools.EventLog.Exporter.Core.ClickHouse; +using OneSTools.EventLog.Exporter.Core.ElasticSearch; namespace OneSTools.EventLog.Exporter.Manager { public class ExportersManager : BackgroundService { - private readonly ILogger _logger; - private readonly IServiceProvider _serviceProvider; - private readonly Dictionary _runExporters = new(); - private readonly List _clstWatchers = new(); - // Common settings - private readonly StorageType _storageType; private readonly List _clstFolders; - private readonly int _portion; - private readonly DateTimeZone _timeZone = DateTimeZoneProviders.Tzdb.GetSystemDefault(); - private readonly int _writingMaxDop; + private readonly List _clstWatchers = new(); + private readonly int _collectedFactor; - private readonly bool _loadArchive; - private readonly int _readingTimeout; + // ClickHouse private readonly string _connectionString; + private readonly bool _loadArchive; + private readonly ILogger _logger; + private readonly int _maximumRetries; + + private readonly TimeSpan _maxRetryTimeout; + // ElasticSearch private readonly List _nodes; + private readonly int _portion; + private readonly int _readingTimeout; + private readonly Dictionary _runExporters = new(); private readonly string _separation; - private readonly int _maximumRetries; - private readonly TimeSpan _maxRetryTimeout; - public ExportersManager(ILogger logger, IServiceProvider serviceProvider, IConfiguration configuration) + private readonly IServiceProvider _serviceProvider; + + // Common settings + private readonly StorageType _storageType; + private readonly DateTimeZone _timeZone = DateTimeZoneProviders.Tzdb.GetSystemDefault(); + private readonly int _writingMaxDop; + + public ExportersManager(ILogger logger, IServiceProvider serviceProvider, + IConfiguration configuration) { _logger = logger; _serviceProvider = serviceProvider; @@ -53,7 +60,8 @@ public ExportersManager(ILogger logger, IServiceProvider servi var timeZone = configuration.GetValue("Exporter:TimeZone", ""); if (!string.IsNullOrWhiteSpace(timeZone)) - _timeZone = DateTimeZoneProviders.Tzdb.GetZoneOrNull(timeZone) ?? throw new Exception($"\"{timeZone}\" is unknown time zone"); + _timeZone = DateTimeZoneProviders.Tzdb.GetZoneOrNull(timeZone) ?? + throw new Exception($"\"{timeZone}\" is unknown time zone"); CheckSettings(); @@ -73,8 +81,10 @@ public ExportersManager(ILogger logger, IServiceProvider servi throw new Exception("ElasticSearch nodes are not specified"); _separation = configuration.GetValue("ElasticSearch:Separation", "H"); - _maximumRetries = configuration.GetValue("ElasticSearch:MaximumRetries", ElasticSearchStorage.DefaultMaximumRetries); - _maxRetryTimeout = TimeSpan.FromSeconds(configuration.GetValue("ElasticSearch:MaxRetryTimeout", ElasticSearchStorage.DefaultMaxRetryTimeoutSec)); + _maximumRetries = configuration.GetValue("ElasticSearch:MaximumRetries", + ElasticSearchStorage.DefaultMaximumRetries); + _maxRetryTimeout = TimeSpan.FromSeconds(configuration.GetValue("ElasticSearch:MaxRetryTimeout", + ElasticSearchStorage.DefaultMaxRetryTimeoutSec)); break; } } @@ -86,9 +96,7 @@ private void CheckSettings() throw new Exception("\"ClstFolders\" is not specified"); foreach (var clstFolder in _clstFolders.Where(clstFolder => !Directory.Exists(clstFolder.Folder))) - { throw new Exception($"Clst folder ({clstFolder.Folder}) doesn't exist"); - } if (_writingMaxDop <= 0) throw new Exception("WritingMaxDegreeOfParallelism cannot be equal to or less than 0"); @@ -125,10 +133,14 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } private void ClstWatcher_InfoBasesDeleted(object sender, ClstEventArgs args) - => StartExporter(args.Path, args.Name, args.DataBaseName); + { + StartExporter(args.Path, args.Name, args.DataBaseName); + } private void ClstWatcher_InfoBasesAdded(object sender, ClstEventArgs args) - => StopExporter(args.Path, args.Name); + { + StopExporter(args.Path, args.Name); + } private void StartExporter(string path, string name, string dataBaseName) { @@ -142,10 +154,12 @@ private void StartExporter(string path, string name, string dataBaseName) if (needStart) { lock (_runExporters) + { if (!_runExporters.ContainsKey(path)) { var cts = new CancellationTokenSource(); - var logger = (ILogger)_serviceProvider.GetService(typeof(ILogger)); + var logger = + (ILogger) _serviceProvider.GetService(typeof(ILogger)); var storage = GetStorage(dataBaseName); var settings = new EventLogExporterSettings @@ -161,12 +175,15 @@ private void StartExporter(string path, string name, string dataBaseName) var exporter = new EventLogExporter(settings, storage, logger); - Task.Factory.StartNew(async () => { + Task.Factory.StartNew(async () => + { try { await exporter.StartAsync(cts.Token); } - catch (TaskCanceledException) { } + catch (TaskCanceledException) + { + } catch (Exception ex) { _logger?.LogCritical(ex, "Failed to execute EventLogExporter"); @@ -175,21 +192,28 @@ private void StartExporter(string path, string name, string dataBaseName) _runExporters.Add(path, cts); - _logger?.LogInformation($"Event log exporter for \"{name}\" information base to \"{dataBaseName}\" is started"); + _logger?.LogInformation( + $"Event log exporter for \"{name}\" information base to \"{dataBaseName}\" is started"); } + } } else - _logger?.LogInformation($"Event log of \"{name}\" information base is in \"new\" format, it won't be handled"); + { + _logger?.LogInformation( + $"Event log of \"{name}\" information base is in \"new\" format, it won't be handled"); + } } private void StopExporter(string id, string name) { - lock(_runExporters) + lock (_runExporters) + { if (_runExporters.TryGetValue(id, out var cts)) { cts.Cancel(); _logger?.LogInformation($"Event log exporter for \"{name}\" information base is stopped"); } + } } private IEventLogStorage GetStorage(string dataBaseName) @@ -198,14 +222,17 @@ private IEventLogStorage GetStorage(string dataBaseName) { case StorageType.ClickHouse: { - var logger = (ILogger)_serviceProvider.GetService(typeof(ILogger)); + var logger = + (ILogger) _serviceProvider.GetService(typeof(ILogger)); var connectionString = $"{_connectionString}Database={dataBaseName};"; return new ClickHouseStorage(connectionString, logger); } case StorageType.ElasticSearch: { - var logger = (ILogger)_serviceProvider.GetService(typeof(ILogger)); + var logger = + (ILogger) _serviceProvider.GetService( + typeof(ILogger)); var settings = new ElasticSearchStorageSettings { @@ -235,4 +262,4 @@ public override void Dispose() GC.SuppressFinalize(this); } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Manager/OneSTools.EventLog.Exporter.Manager.csproj b/OneSTools.EventLog.Exporter.Manager/OneSTools.EventLog.Exporter.Manager.csproj index 59732aa..b6edc72 100644 --- a/OneSTools.EventLog.Exporter.Manager/OneSTools.EventLog.Exporter.Manager.csproj +++ b/OneSTools.EventLog.Exporter.Manager/OneSTools.EventLog.Exporter.Manager.csproj @@ -4,7 +4,7 @@ net5.0 dotnet-OneSTools.EventLog.Exporter.Manager-436AD68C-DBEB-42E3-8010-E66614FE89B6 EventLogExportersManager - 0.0.4 + 0.0.5 Akpaev Evgeny Akpaev Evgeny Менеджер служб экспорта журналов регистрации diff --git a/OneSTools.EventLog.Exporter.Manager/Program.cs b/OneSTools.EventLog.Exporter.Manager/Program.cs index b8ba0dd..3149fcf 100644 --- a/OneSTools.EventLog.Exporter.Manager/Program.cs +++ b/OneSTools.EventLog.Exporter.Manager/Program.cs @@ -1,12 +1,8 @@ -using Microsoft.Extensions.Configuration; +using System.IO; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using OneSTools.EventLog.Exporter.Core.ClickHouse; using OneSTools.EventLog.Exporter.Core; -using OneSTools.EventLog.Exporter.Core.ElasticSearch; -using System; -using System.IO; namespace OneSTools.EventLog.Exporter.Manager { @@ -17,8 +13,9 @@ public static void Main(string[] args) CreateHostBuilder(args).Build().Run(); } - public static IHostBuilder CreateHostBuilder(string[] args) => - Host.CreateDefaultBuilder(args) + public static IHostBuilder CreateHostBuilder(string[] args) + { + return Host.CreateDefaultBuilder(args) .UseWindowsService() .UseSystemd() .ConfigureLogging((hostingContext, logging) => @@ -27,9 +24,7 @@ public static IHostBuilder CreateHostBuilder(string[] args) => logging.AddFile(logPath); logging.AddConfiguration(hostingContext.Configuration.GetSection("Logging")); }) - .ConfigureServices((_, services) => - { - services.AddHostedService(); - }); + .ConfigureServices((_, services) => { services.AddHostedService(); }); + } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Manager/Properties/launchSettings.json b/OneSTools.EventLog.Exporter.Manager/Properties/launchSettings.json index 8aeb2fc..3adb118 100644 --- a/OneSTools.EventLog.Exporter.Manager/Properties/launchSettings.json +++ b/OneSTools.EventLog.Exporter.Manager/Properties/launchSettings.json @@ -7,4 +7,4 @@ } } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Manager/appsettings.Development.json b/OneSTools.EventLog.Exporter.Manager/appsettings.Development.json index 4f30a00..a73ce31 100644 --- a/OneSTools.EventLog.Exporter.Manager/appsettings.Development.json +++ b/OneSTools.EventLog.Exporter.Manager/appsettings.Development.json @@ -6,4 +6,4 @@ "Microsoft.Hosting.Lifetime": "Information" } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Manager/appsettings.json b/OneSTools.EventLog.Exporter.Manager/appsettings.json index c320ec8..e61ce86 100644 --- a/OneSTools.EventLog.Exporter.Manager/appsettings.json +++ b/OneSTools.EventLog.Exporter.Manager/appsettings.json @@ -42,4 +42,4 @@ "MaximumRetries": 2, "MaxRetryTimeout": 30 } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter.Manager/log.txt b/OneSTools.EventLog.Exporter.Manager/log.txt deleted file mode 100644 index 879b50f..0000000 --- a/OneSTools.EventLog.Exporter.Manager/log.txt +++ /dev/null @@ -1,22 +0,0 @@ -2021-01-01 01:02:09.061 | Information | OneSTools.EventLog.Exporter.Manager.ExportersManager[0] - Event log exporter for "upp_main" information base to "upp_main_el" is started -2021-01-01 01:02:09.192 | Information | Microsoft.Hosting.Lifetime[0] - Application started. Press Ctrl+C to shut down. -2021-01-01 01:02:09.218 | Information | OneSTools.EventLog.Exporter.Core.EventLogExporter[0] - Log folder: \\s01\c$\Program Files\1cv8\srvinfo\reg_1541\ac44b33e-112a-49ec-b6e1-856ee8c2e8f0\1Cv8Log -2021-01-01 01:02:09.223 | Information | Microsoft.Hosting.Lifetime[0] - Hosting environment: Development -2021-01-01 01:02:09.238 | Information | OneSTools.EventLog.Exporter.Core.EventLogExporter[0] - Portion per request: 10000 -2021-01-01 01:02:09.255 | Information | Microsoft.Hosting.Lifetime[0] - Content root path: C:\Users\akpaev.e.ENTERPRISE\source\repos\OneSTools.EventLog\OneSTools.EventLog.Exporter.Manager -2021-01-01 01:03:21.146 | Information | OneSTools.EventLog.Exporter.Core.EventLogExporter[0] - There're no log items in the database, first found log file will be read from 0 position -2021-01-01 01:03:21.635 | Information | OneSTools.EventLog.Exporter.Core.EventLogExporter[0] - Reader started reading 20210111000000.lgp -2021-01-01 01:03:22.938 | Debug | OneSTools.EventLog.Exporter.Core.ClickHouse.ClickHouseStorage[0] - 10000 items were being written to upp_main_el -2021-01-01 01:03:23.230 | Debug | OneSTools.EventLog.Exporter.Core.ClickHouse.ClickHouseStorage[0] - 10000 items were being written to upp_main_el -2021-01-01 01:03:24.460 | Debug | OneSTools.EventLog.Exporter.Core.ClickHouse.ClickHouseStorage[0] - 1242 items were being written to upp_main_el diff --git a/OneSTools.EventLog.Exporter/EventLogExporterService.cs b/OneSTools.EventLog.Exporter/EventLogExporterService.cs index 742226c..97f7d79 100644 --- a/OneSTools.EventLog.Exporter/EventLogExporterService.cs +++ b/OneSTools.EventLog.Exporter/EventLogExporterService.cs @@ -1,16 +1,16 @@ -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using OneSTools.EventLog.Exporter.Core; using System; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using OneSTools.EventLog.Exporter.Core; namespace OneSTools.EventLog.Exporter { public class EventLogExporterService : BackgroundService { - private readonly ILogger _logger; private readonly EventLogExporter _exporter; + private readonly ILogger _logger; private bool _disposedValue; public EventLogExporterService(ILogger logger, EventLogExporter exporter) @@ -25,7 +25,9 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await _exporter.StartAsync(stoppingToken); } - catch (TaskCanceledException) { } + catch (TaskCanceledException) + { + } catch (Exception ex) { _logger.LogCritical(ex, "Failed to execute EventLogExporter"); @@ -38,7 +40,6 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - } _exporter?.Dispose(); @@ -49,13 +50,13 @@ protected virtual void Dispose(bool disposing) ~EventLogExporterService() { - Dispose(disposing: false); + Dispose(false); } public override void Dispose() { - Dispose(disposing: true); + Dispose(true); GC.SuppressFinalize(this); } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter/OneSTools.EventLog.Exporter.csproj b/OneSTools.EventLog.Exporter/OneSTools.EventLog.Exporter.csproj index 9057da7..945a023 100644 --- a/OneSTools.EventLog.Exporter/OneSTools.EventLog.Exporter.csproj +++ b/OneSTools.EventLog.Exporter/OneSTools.EventLog.Exporter.csproj @@ -7,7 +7,7 @@ Akpaev Evgeny Служба для экспорта журнала регистрации 1С в ClickHouse и ElasticSearch - 1.0.4 + 1.0.6 EventLogExporter OneSTools.EventLog.Exporter @@ -34,5 +34,9 @@ - - + + + + + + \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter/Program.cs b/OneSTools.EventLog.Exporter/Program.cs index fa2a202..5e23683 100644 --- a/OneSTools.EventLog.Exporter/Program.cs +++ b/OneSTools.EventLog.Exporter/Program.cs @@ -1,3 +1,5 @@ +using System; +using System.IO; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -5,8 +7,6 @@ using OneSTools.EventLog.Exporter.Core; using OneSTools.EventLog.Exporter.Core.ClickHouse; using OneSTools.EventLog.Exporter.Core.ElasticSearch; -using System; -using System.IO; namespace OneSTools.EventLog.Exporter { @@ -17,8 +17,9 @@ public static void Main(string[] args) CreateHostBuilder(args).Build().Run(); } - public static IHostBuilder CreateHostBuilder(string[] args) => - Host.CreateDefaultBuilder(args) + public static IHostBuilder CreateHostBuilder(string[] args) + { + return Host.CreateDefaultBuilder(args) //.ConfigureAppConfiguration(c => { // c.SetBasePath(Path.GetDirectoryName(Assembly.GetEntryAssembly().Location)); // c.AddJsonFile("appsettings.json"); @@ -44,7 +45,7 @@ public static IHostBuilder CreateHostBuilder(string[] args) => services.AddSingleton(); break; case StorageType.None: - throw new Exception($"You must set StorageType parameter before starting the exporter"); + throw new Exception("You must set StorageType parameter before starting the exporter"); default: throw new Exception($"{storageType} is not available value of StorageType enum"); } @@ -52,5 +53,6 @@ public static IHostBuilder CreateHostBuilder(string[] args) => services.AddSingleton(); services.AddHostedService(); }); + } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter/appsettings.Development.json b/OneSTools.EventLog.Exporter/appsettings.Development.json index 3d4597c..352c5bb 100644 --- a/OneSTools.EventLog.Exporter/appsettings.Development.json +++ b/OneSTools.EventLog.Exporter/appsettings.Development.json @@ -8,7 +8,7 @@ }, "Exporter": { "StorageType": 2, - "LogFolder": "C:\\Users\\akpaev.e.ENTERPRISE\\Desktop\\1Cv8Log", + "LogFolder": "\\\\rds\\c$\\Program Files\\1cv8\\srvinfo\\reg_1541\\985bc677-ba89-4910-ad73-c4794a523b3c\\1Cv8Log", "Portion": 10000, "TimeZone": "Europe/Moscow", "WritingMaxDegreeOfParallelism": 8, @@ -17,7 +17,7 @@ "LoadArchive": false }, "ClickHouse": { - "ConnectionString": "Host=192.168.0.93;Port=8123;Database=upp_main_el;Username=default;password=;" + "ConnectionString": "Host=192.168.0.93;Port=8123;Database=zup3_united_el;Username=default;password=;" }, "ElasticSearch": { "Nodes": [ @@ -26,9 +26,9 @@ "AuthenticationType": "0" // 0 - None, 1 - Basic, 2 - ApiKey } ], - "Index": "upp-main-el", + "Index": "zup3-united-el", "Separation": "M", "MaximumRetries": 2, "MaxRetryTimeout": 30 } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog.Exporter/appsettings.json b/OneSTools.EventLog.Exporter/appsettings.json index 2db9659..1e2fea1 100644 --- a/OneSTools.EventLog.Exporter/appsettings.json +++ b/OneSTools.EventLog.Exporter/appsettings.json @@ -31,4 +31,4 @@ "MaximumRetries": 2, "MaxRetryTimeout": 30 } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog/DateTimeZoneExtensions.cs b/OneSTools.EventLog/DateTimeZoneExtensions.cs index c4a6539..6c80f36 100644 --- a/OneSTools.EventLog/DateTimeZoneExtensions.cs +++ b/OneSTools.EventLog/DateTimeZoneExtensions.cs @@ -1,13 +1,13 @@ -using NodaTime; -using System; -using System.Collections.Generic; -using System.Text; +using System; +using NodaTime; namespace OneSTools.EventLog { internal static class DateTimeZoneExtensions { public static DateTime ToUtc(this DateTimeZone dateTimeZone, DateTime dateTime) - => LocalDateTime.FromDateTime(dateTime).InZoneStrictly(dateTimeZone).ToDateTimeUtc(); + { + return LocalDateTime.FromDateTime(dateTime).InZoneStrictly(dateTimeZone).ToDateTimeUtc(); + } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog/EventLogItem.cs b/OneSTools.EventLog/EventLogItem.cs index 60c7239..9642c84 100644 --- a/OneSTools.EventLog/EventLogItem.cs +++ b/OneSTools.EventLog/EventLogItem.cs @@ -1,6 +1,4 @@ using System; -using System.Runtime.CompilerServices; -using System.Transactions; namespace OneSTools.EventLog { @@ -12,7 +10,7 @@ public class EventLogItem public virtual long LgfEndPosition { get; set; } = 0; public virtual DateTime DateTime { get; set; } = DateTime.MinValue; public virtual string TransactionStatus { get; set; } = ""; - public virtual DateTime TransactionDateTime { get; set; } = new DateTime(1970,1,1); + public virtual DateTime TransactionDateTime { get; set; } = new DateTime(1970, 1, 1); public virtual long TransactionNumber { get; set; } = 0; public virtual string UserUuid { get; set; } = ""; public virtual string User { get; set; } = ""; @@ -31,4 +29,4 @@ public class EventLogItem public virtual int AddPort { get; set; } = 0; public virtual long Session { get; set; } = 0; } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog/EventLogReader.cs b/OneSTools.EventLog/EventLogReader.cs index e9c2f77..225c504 100644 --- a/OneSTools.EventLog/EventLogReader.cs +++ b/OneSTools.EventLog/EventLogReader.cs @@ -1,30 +1,22 @@ -using NodaTime; -using System; -using System.Collections.Concurrent; +using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; -using System.Threading.Tasks; namespace OneSTools.EventLog { /// - /// Presents methods for reading 1C event log - /// - public class EventLogReader: IDisposable + /// Presents methods for reading 1C event log + /// + public class EventLogReader : IDisposable { - private ManualResetEvent _lgpChangedCreated; private readonly EventLogReaderSettings _settings; + private bool _disposedValue; private LgfReader _lgfReader; - private LgpReader _lgpReader; + private ManualResetEvent _lgpChangedCreated; private FileSystemWatcher _lgpFilesWatcher; - private bool _disposedValue; - - /// - /// Current reader's "lgp" file name - /// - public string LgpFileName => _lgpReader.LgpFileName; + private LgpReader _lgpReader; public EventLogReader(EventLogReaderSettings settings) { @@ -43,7 +35,20 @@ public EventLogReader(EventLogReaderSettings settings) } /// - /// The behaviour of the method depends on the mode of the reader. In the "live" mode it'll be waiting for an appearing of the new event item, otherwise It'll just return null + /// Current reader's "lgp" file name + /// + public string LgpFileName => _lgpReader.LgpFileName; + + public void Dispose() + { + // Не изменяйте этот код. Разместите код очистки в методе "Dispose(bool disposing)". + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// The behaviour of the method depends on the mode of the reader. In the "live" mode it'll be waiting for an appearing + /// of the new event item, otherwise It'll just return null /// /// Token for interrupting of the reader /// @@ -80,7 +85,8 @@ public EventLogItem ReadNextEventLogItem(CancellationToken cancellationToken = d { _lgpChangedCreated.Reset(); - var waitHandle = WaitHandle.WaitAny(new WaitHandle[] { _lgpChangedCreated, cancellationToken.WaitHandle }, _settings.ReadingTimeout); + var waitHandle = WaitHandle.WaitAny( + new[] {_lgpChangedCreated, cancellationToken.WaitHandle}, _settings.ReadingTimeout); if (_settings.ReadingTimeout != Timeout.Infinite && waitHandle == WaitHandle.WaitTimeout) throw new EventLogReaderTimeoutException(); @@ -118,31 +124,31 @@ private bool SetNextLgpReader() var files = Directory.GetFiles(_settings.LogFolder, "*.lgp"); foreach (var file in files) - { if (_lgpReader != null) { if (_lgpReader.LgpPath != file) filesDateTime.Add((file, new FileInfo(file).LastWriteTime)); } else + { filesDateTime.Add((file, new FileInfo(file).LastWriteTime)); - } + } var orderedFiles = filesDateTime.OrderBy(c => c.Item2).ToList(); - var nextFile = orderedFiles.FirstOrDefault(c => c.Item2 > currentReaderLastWriteDateTime); + var (item1, _) = orderedFiles.FirstOrDefault(c => c.Item2 > currentReaderLastWriteDateTime); - if (string.IsNullOrEmpty(nextFile.Item1)) - return false; - else + if (string.IsNullOrEmpty(item1)) { - _lgpReader?.Dispose(); - _lgpReader = null; + return false; + } - _lgpReader = new LgpReader(nextFile.Item1, _settings.TimeZone, _lgfReader); + _lgpReader?.Dispose(); + _lgpReader = null; - return true; - } + _lgpReader = new LgpReader(item1, _settings.TimeZone, _lgfReader); + + return true; } private void StartLgpFilesWatcher() @@ -168,11 +174,6 @@ protected virtual void Dispose(bool disposing) { if (!_disposedValue) { - if (disposing) - { - // TODO: освободить управляемое состояние (управляемые объекты) - } - _lgpFilesWatcher?.Dispose(); _lgpFilesWatcher = null; _lgpChangedCreated?.Dispose(); @@ -188,14 +189,7 @@ protected virtual void Dispose(bool disposing) ~EventLogReader() { - Dispose(disposing: false); - } - - public void Dispose() - { - // Не изменяйте этот код. Разместите код очистки в методе "Dispose(bool disposing)". - Dispose(disposing: true); - GC.SuppressFinalize(this); + Dispose(false); } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog/EventLogReaderSettings.cs b/OneSTools.EventLog/EventLogReaderSettings.cs index ee02dd7..4e7cee9 100644 --- a/OneSTools.EventLog/EventLogReaderSettings.cs +++ b/OneSTools.EventLog/EventLogReaderSettings.cs @@ -1,5 +1,5 @@ -using NodaTime; -using System.Threading; +using System.Threading; +using NodaTime; namespace OneSTools.EventLog { @@ -14,4 +14,4 @@ public class EventLogReaderSettings public int ReadingTimeout { get; set; } = Timeout.Infinite; public DateTimeZone TimeZone { get; set; } = DateTimeZoneProviders.Tzdb.GetSystemDefault(); } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog/EventLogReaderTimeoutException.cs b/OneSTools.EventLog/EventLogReaderTimeoutException.cs index 252f8c8..3eebe32 100644 --- a/OneSTools.EventLog/EventLogReaderTimeoutException.cs +++ b/OneSTools.EventLog/EventLogReaderTimeoutException.cs @@ -1,10 +1,8 @@ using System; -using System.Collections.Generic; -using System.Text; namespace OneSTools.EventLog { public class EventLogReaderTimeoutException : Exception { } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog/LgfReader.cs b/OneSTools.EventLog/LgfReader.cs index f260f56..c4ee1eb 100644 --- a/OneSTools.EventLog/LgfReader.cs +++ b/OneSTools.EventLog/LgfReader.cs @@ -1,47 +1,54 @@ -using OneSTools.BracketsFile; -using System; +using System; using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Text; using System.Threading; -using System.Threading.Tasks; +using OneSTools.BracketsFile; namespace OneSTools.EventLog { internal class LgfReader : IDisposable { - private FileStream _fileStream; private BracketsListReader _bracketsReader; + private bool _disposedValue; + private FileStream _fileStream; - public string LgfPath { get; } /// - /// Key tuple - first value is an object type, second value is an object number in the array + /// Key tuple - first value is an object type, second value is an object number in the array /// private Dictionary<(ObjectType, int), string> _objects = new Dictionary<(ObjectType, int), string>(); + /// - /// Key tuple - first value is an object type, second value is an object number in the array - /// Value tuple - first value is an object value, second value is a guid of the object value + /// Key tuple - first value is an object type, second value is an object number in the array + /// Value tuple - first value is an object value, second value is a guid of the object value /// - private Dictionary<(ObjectType, int), (string, string)> _referencedObjects = new Dictionary<(ObjectType, int), (string, string)>(); - private bool _disposedValue; + private Dictionary<(ObjectType, int), (string, string)> _referencedObjects = + new Dictionary<(ObjectType, int), (string, string)>(); public LgfReader(string lgfPath) { LgfPath = lgfPath; } - private void ReadTill(ObjectType objectType, int number, long position, CancellationToken cancellationToken = default) + public string LgfPath { get; } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + private void ReadTill(ObjectType objectType, int number, long position, + CancellationToken cancellationToken = default) { InitializeStreams(); - bool stop = false; + var stop = false; while (!stop && !_bracketsReader.EndOfStream && !cancellationToken.IsCancellationRequested) { var itemData = _bracketsReader.NextNode(); - var ot = (ObjectType)(int)itemData[0]; + var ot = (ObjectType) (int) itemData[0]; // Skip unknown object types if (ot >= ObjectType.Unknown) @@ -51,8 +58,8 @@ private void ReadTill(ObjectType objectType, int number, long position, Cancella { case ObjectType.Users: case ObjectType.Metadata: - var key = (ot, (int)itemData[3]); - var value = ((string)itemData[2], (string)itemData[1]); + var key = (ot, (int) itemData[3]); + var value = ((string) itemData[2], (string) itemData[1]); if (_referencedObjects.ContainsKey(key)) _referencedObjects.Remove(key); @@ -67,11 +74,11 @@ private void ReadTill(ObjectType objectType, int number, long position, Cancella if (ot == objectType && key.Item2 == number) stop = true; - + break; default: - var key1 = (ot, (int)itemData[2]); - var value1 = (string)itemData[1]; + var key1 = (ot, (int) itemData[2]); + var value1 = (string) itemData[1]; if (_objects.ContainsKey(key1)) _objects.Remove(key1); @@ -99,29 +106,27 @@ public string GetObjectValue(ObjectType objectType, int number, CancellationToke if (_objects.TryGetValue((objectType, number), out var value)) return value; - else - ReadTill(objectType, number, 0, cancellationToken); + ReadTill(objectType, number, 0, cancellationToken); if (_objects.TryGetValue((objectType, number), out value)) return value; - else - throw new Exception($"Cannot find objectType {objectType} with number {number} in objects collection"); + throw new Exception($"Cannot find objectType {objectType} with number {number} in objects collection"); } - public (string Value, string Uuid) GetReferencedObjectValue(ObjectType objectType, int number, CancellationToken cancellationToken = default) + public (string Value, string Uuid) GetReferencedObjectValue(ObjectType objectType, int number, + CancellationToken cancellationToken = default) { if (number == 0) return ("", ""); if (_referencedObjects.TryGetValue((objectType, number), out var value)) return value; - else - ReadTill(objectType, number, 0, cancellationToken); + ReadTill(objectType, number, 0, cancellationToken); if (_referencedObjects.TryGetValue((objectType, number), out value)) return value; - else - throw new Exception($"Cannot find objectType {objectType} with number {number} in referenced objects collection"); + throw new Exception( + $"Cannot find objectType {objectType} with number {number} in referenced objects collection"); } private void InitializeStreams() @@ -131,7 +136,8 @@ private void InitializeStreams() if (!File.Exists(LgfPath)) throw new Exception("Cannot find \"1Cv8.lgf\""); - _fileStream = new FileStream(LgfPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + _fileStream = new FileStream(LgfPath, FileMode.Open, FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete); _bracketsReader = new BracketsListReader(_fileStream); } } @@ -169,13 +175,7 @@ protected virtual void Dispose(bool disposing) ~LgfReader() { - Dispose(disposing: false); - } - - public void Dispose() - { - Dispose(disposing: true); - GC.SuppressFinalize(this); + Dispose(false); } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog/LgpReader.cs b/OneSTools.EventLog/LgpReader.cs index 06488bb..d0a36f5 100644 --- a/OneSTools.EventLog/LgpReader.cs +++ b/OneSTools.EventLog/LgpReader.cs @@ -1,32 +1,21 @@ -using OneSTools.BracketsFile; -using System; -using System.Collections.Generic; -using System.ComponentModel; +using System; using System.Globalization; using System.IO; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Security.Cryptography; using System.Text; -using System.Text.RegularExpressions; using System.Threading; -using System.Threading.Tasks; -using System.Resources; using NodaTime; +using OneSTools.BracketsFile; namespace OneSTools.EventLog { - internal class LgpReader: IDisposable + internal class LgpReader : IDisposable { - private LgfReader _lgfReader; - private FileStream _fileStream; + private readonly DateTimeZone _timeZone; private BracketsListReader _bracketsReader; - private FileSystemWatcher _lgpFileWatcher; private bool _disposedValue; - private readonly DateTimeZone _timeZone; - - public string LgpPath { get; } - public string LgpFileName => Path.GetFileName(LgpPath); + private FileStream _fileStream; + private LgfReader _lgfReader; + private FileSystemWatcher _lgpFileWatcher; public LgpReader(string lgpPath, DateTimeZone timeZone, LgfReader lgfReader) { @@ -35,6 +24,15 @@ public LgpReader(string lgpPath, DateTimeZone timeZone, LgfReader lgfReader) _lgfReader = lgfReader; } + public string LgpPath { get; } + public string LgpFileName => Path.GetFileName(LgpPath); + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + public EventLogItem ReadNextEventLogItem(CancellationToken cancellationToken = default) { if (_disposedValue) @@ -61,48 +59,46 @@ private void InitializeStreams() _lgpFileWatcher = new FileSystemWatcher(Path.GetDirectoryName(LgpPath)!, "*.lgp") { - NotifyFilter = NotifyFilters.CreationTime | NotifyFilters.LastWrite | NotifyFilters.FileName | NotifyFilters.Attributes + NotifyFilter = NotifyFilters.CreationTime | NotifyFilters.LastWrite | NotifyFilters.FileName | + NotifyFilters.Attributes }; _lgpFileWatcher.Deleted += LgpFileWatcher_Deleted; _lgpFileWatcher.EnableRaisingEvents = true; - _fileStream = new FileStream(LgpPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + _fileStream = new FileStream(LgpPath, FileMode.Open, FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete); _bracketsReader = new BracketsListReader(_fileStream); } } private void LgpFileWatcher_Deleted(object sender, FileSystemEventArgs e) { - if (e.ChangeType == WatcherChangeTypes.Deleted && LgpPath == e.FullPath) - { - Dispose(); - } + if (e.ChangeType == WatcherChangeTypes.Deleted && LgpPath == e.FullPath) Dispose(); } private (StringBuilder Data, long EndPosition) ReadNextEventLogItemData() { - StringBuilder data = _bracketsReader.NextNodeAsStringBuilder(); + var data = _bracketsReader.NextNodeAsStringBuilder(); return (data, _bracketsReader.Position); } private EventLogItem ReadEventLogItemData(CancellationToken cancellationToken = default) { - (StringBuilder data, long endPosition) = ReadNextEventLogItemData(); - - if (data.Length == 0) - return null; + var (data, endPosition) = ReadNextEventLogItemData(); - return ParseEventLogItemData(data, endPosition, cancellationToken); + return data.Length == 0 ? null : ParseEventLogItemData(data, endPosition, cancellationToken); } - private EventLogItem ParseEventLogItemData(StringBuilder eventLogItemData, long endPosition, CancellationToken cancellationToken = default) + private EventLogItem ParseEventLogItemData(StringBuilder eventLogItemData, long endPosition, + CancellationToken cancellationToken = default) { var parsedData = BracketsParser.ParseBlock(eventLogItemData); var eventLogItem = new EventLogItem { - DateTime = _timeZone.ToUtc(DateTime.ParseExact(parsedData[0], "yyyyMMddHHmmss", CultureInfo.InvariantCulture)), + DateTime = _timeZone.ToUtc(DateTime.ParseExact(parsedData[0], "yyyyMMddHHmmss", + CultureInfo.InvariantCulture)), TransactionStatus = GetTransactionPresentation(parsedData[1]), FileName = LgpFileName, EndPosition = endPosition, @@ -113,7 +109,9 @@ private EventLogItem ParseEventLogItemData(StringBuilder eventLogItemData, long eventLogItem.TransactionNumber = Convert.ToInt64(transactionData[1], 16); var transactionDate = new DateTime().AddSeconds(Convert.ToInt64(transactionData[0], 16) / 10000); - eventLogItem.TransactionDateTime = transactionDate == DateTime.MinValue ? transactionDate : _timeZone.ToUtc(transactionDate); + eventLogItem.TransactionDateTime = transactionDate == DateTime.MinValue + ? transactionDate + : _timeZone.ToUtc(transactionDate); var (value, uuid) = _lgfReader.GetReferencedObjectValue(ObjectType.Users, parsedData[3], cancellationToken); eventLogItem.UserUuid = uuid; @@ -129,7 +127,7 @@ private EventLogItem ParseEventLogItemData(StringBuilder eventLogItemData, long var ev = _lgfReader.GetObjectValue(ObjectType.Events, parsedData[7], cancellationToken); eventLogItem.Event = GetEventPresentation(ev); - var severity = (string)parsedData[8]; + var severity = (string) parsedData[8]; eventLogItem.Severity = GetSeverityPresentation(severity); eventLogItem.Comment = parsedData[9]; @@ -155,22 +153,22 @@ private EventLogItem ParseEventLogItemData(StringBuilder eventLogItemData, long return eventLogItem; } - private string GetData(BracketsNode node) + private static string GetData(BracketsNode node) { - var dataType = (string)node[0]; + var dataType = (string) node[0]; switch (dataType) { case "R": // Reference - return (string)node[1]; + return node[1]; case "U": // Undefined return ""; case "S": // String - return (string)node[1]; + return node[1]; case "B": // Boolean - return (string)node[1] == "0" ? "false" : "true"; + return (string) node[1] == "0" ? "false" : "true"; case "P": // Complex data - StringBuilder str = new StringBuilder(); + var str = new StringBuilder(); var subDataNode = node[1]; @@ -185,7 +183,7 @@ private string GetData(BracketsNode node) var subDataCount = subDataNode.Count - 1; if (subDataCount > 0) - for (int i = 1; i <= subDataCount; i++) + for (var i = 1; i <= subDataCount; i++) { var value = GetData(subDataNode[i]); @@ -199,7 +197,7 @@ private string GetData(BracketsNode node) } } - private string GetTransactionPresentation(string str) + private static string GetTransactionPresentation(string str) { return str switch { @@ -207,11 +205,11 @@ private string GetTransactionPresentation(string str) "C" => "Отменена", "R" => "Не завершена", "N" => "Нет транзакции", - _ => "", + _ => "" }; } - private string GetSeverityPresentation(string str) + private static string GetSeverityPresentation(string str) { return str switch { @@ -219,11 +217,11 @@ private string GetSeverityPresentation(string str) "E" => "Ошибка", "W" => "Предупреждение", "N" => "Примечание", - _ => "", + _ => "" }; } - - private string GetApplicationPresentation(string str) + + private static string GetApplicationPresentation(string str) { return str switch { @@ -240,11 +238,11 @@ private string GetApplicationPresentation(string str) "JobScheduler" => "Планировщик заданий", "Debugger" => "Отладчик", "RAS" => "Сервер администрирования", - _ => str, + _ => str }; } - private string GetEventPresentation(string str) + private static string GetEventPresentation(string str) { return str switch { @@ -258,7 +256,8 @@ private string GetEventPresentation(string str) "_$Data$_.NewVersion" => "Данные.Добавление версии", "_$Data$_.Pos" => "Данные.Проведение", "_$Data$_.PredefinedDataInitialization" => "Данные.Инициализация предопределенных данных", - "_$Data$_.PredefinedDataInitializationDataNotFound" => "Данные.Инициализация предопределенных данных.Данные не найдены", + "_$Data$_.PredefinedDataInitializationDataNotFound" => + "Данные.Инициализация предопределенных данных.Данные не найдены", "_$Data$_.SetPredefinedDataInitialization" => "Данные.Установка инициализации предопределенных данных", "_$Data$_.SetStandardODataInterfaceContent" => "Данные.Изменение состава стандартного интерфейса OData", "_$Data$_.TotalsMaxPeriodUpdate" => "Данные.Изменение максимального периода рассчитанных итогов", @@ -272,11 +271,14 @@ private string GetEventPresentation(string str) "_$InfoBase$_.ConfigUpdate" => "Информационная база.Изменение конфигурации", "_$InfoBase$_.DBConfigBackgroundUpdateCancel" => "Информационная база.Отмена фонового обновления", "_$InfoBase$_.DBConfigBackgroundUpdateFinish" => "Информационная база.Завершение фонового обновления", - "_$InfoBase$_.DBConfigBackgroundUpdateResume" => "Информационная база.Продолжение (после приостановки) процесса фонового обновления", + "_$InfoBase$_.DBConfigBackgroundUpdateResume" => + "Информационная база.Продолжение (после приостановки) процесса фонового обновления", "_$InfoBase$_.DBConfigBackgroundUpdateStart" => "Информационная база.Запуск фонового обновления", - "_$InfoBase$_.DBConfigBackgroundUpdateSuspend" => "Информационная база.Приостановка (пауза) процесса фонового обновления", + "_$InfoBase$_.DBConfigBackgroundUpdateSuspend" => + "Информационная база.Приостановка (пауза) процесса фонового обновления", "_$InfoBase$_.DBConfigExtensionUpdate" => "Информационная база.Изменение расширения конфигурации", - "_$InfoBase$_.DBConfigExtensionUpdateError" => "Информационная база.Ошибка изменения расширения конфигурации", + "_$InfoBase$_.DBConfigExtensionUpdateError" => + "Информационная база.Ошибка изменения расширения конфигурации", "_$InfoBase$_.DBConfigUpdate" => "Информационная база.Изменение конфигурации базы данных", "_$InfoBase$_.DBConfigUpdateStart" => "Информационная база.Запуск обновления конфигурации базы данных", "_$InfoBase$_.DumpError" => "Информационная база.Ошибка выгрузки в файл", @@ -286,21 +288,30 @@ private string GetEventPresentation(string str) "_$InfoBase$_.EventLogReduce" => "Информационная база.Сокращение журнала регистрации", "_$InfoBase$_.EventLogReduceError" => "Информационная база.Ошибка сокращения журнала регистрации", "_$InfoBase$_.EventLogSettingsUpdate" => "Информационная база.Изменение параметров журнала регистрации", - "_$InfoBase$_.EventLogSettingsUpdateError" => "Информационная база.Ошибка при изменение настроек журнала регистрации", - "_$InfoBase$_.InfoBaseAdmParamsUpdate" => "Информационная база.Изменение параметров информационной базы", - "_$InfoBase$_.InfoBaseAdmParamsUpdateError" => "Информационная база.Ошибка изменения параметров информационной базы", - "_$InfoBase$_.IntegrationServiceActiveUpdate" => "Информационная база.Изменение активности сервиса интеграции", - "_$InfoBase$_.IntegrationServiceSettingsUpdate" => "Информационная база.Изменение настроек сервиса интеграции", + "_$InfoBase$_.EventLogSettingsUpdateError" => + "Информационная база.Ошибка при изменение настроек журнала регистрации", + "_$InfoBase$_.InfoBaseAdmParamsUpdate" => + "Информационная база.Изменение параметров информационной базы", + "_$InfoBase$_.InfoBaseAdmParamsUpdateError" => + "Информационная база.Ошибка изменения параметров информационной базы", + "_$InfoBase$_.IntegrationServiceActiveUpdate" => + "Информационная база.Изменение активности сервиса интеграции", + "_$InfoBase$_.IntegrationServiceSettingsUpdate" => + "Информационная база.Изменение настроек сервиса интеграции", "_$InfoBase$_.MasterNodeUpdate" => "Информационная база.Изменение главного узла", "_$InfoBase$_.PredefinedDataUpdate" => "Информационная база.Обновление предопределенных данных", "_$InfoBase$_.RegionalSettingsUpdate" => "Информационная база.Изменение региональных установок", "_$InfoBase$_.RestoreError" => "Информационная база.Ошибка загрузки из файла", "_$InfoBase$_.RestoreFinish" => "Информационная база.Окончание загрузки из файла", "_$InfoBase$_.RestoreStart" => "Информационная база.Начало загрузки из файла", - "_$InfoBase$_.SecondFactorAuthTemplateDelete" => "Информационная база.Удаление шаблона вторго фактора аутентификации", - "_$InfoBase$_.SecondFactorAuthTemplateNew" => "Информационная база.Добавление шаблона вторго фактора аутентификации", - "_$InfoBase$_.SecondFactorAuthTemplateUpdate" => "Информационная база.Изменение шаблона вторго фактора аутентификации", - "_$InfoBase$_.SetPredefinedDataUpdate" => "Информационная база.Установить обновление предопределенных данных", + "_$InfoBase$_.SecondFactorAuthTemplateDelete" => + "Информационная база.Удаление шаблона вторго фактора аутентификации", + "_$InfoBase$_.SecondFactorAuthTemplateNew" => + "Информационная база.Добавление шаблона вторго фактора аутентификации", + "_$InfoBase$_.SecondFactorAuthTemplateUpdate" => + "Информационная база.Изменение шаблона вторго фактора аутентификации", + "_$InfoBase$_.SetPredefinedDataUpdate" => + "Информационная база.Установить обновление предопределенных данных", "_$InfoBase$_.TARImportant" => "Тестирование и исправление.Ошибка", "_$InfoBase$_.TARInfo" => "Тестирование и исправление.Сообщение", "_$InfoBase$_.TARMess" => "Тестирование и исправление.Предупреждение", @@ -336,35 +347,23 @@ private string GetEventPresentation(string str) protected virtual void Dispose(bool disposing) { - if (!_disposedValue) - { - if (disposing) - { - // TODO: освободить управляемое состояние (управляемые объекты) - } + if (_disposedValue) return; - _bracketsReader?.Dispose(); - _bracketsReader = null; - _fileStream = null; + _bracketsReader?.Dispose(); + _bracketsReader = null; + _fileStream = null; - _lgpFileWatcher?.Dispose(); - _lgpFileWatcher = null; + _lgpFileWatcher?.Dispose(); + _lgpFileWatcher = null; - _lgfReader = null; + _lgfReader = null; - _disposedValue = true; - } + _disposedValue = true; } ~LgpReader() { - Dispose(disposing: false); - } - - public void Dispose() - { - Dispose(disposing: true); - GC.SuppressFinalize(this); + Dispose(false); } } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog/ObjectType.cs b/OneSTools.EventLog/ObjectType.cs index b3ec848..9c204ea 100644 --- a/OneSTools.EventLog/ObjectType.cs +++ b/OneSTools.EventLog/ObjectType.cs @@ -13,4 +13,4 @@ internal enum ObjectType AddPorts = 8, Unknown = 9 } -} +} \ No newline at end of file diff --git a/OneSTools.EventLog/OneSTools.EventLog.csproj b/OneSTools.EventLog/OneSTools.EventLog.csproj index 8e7f63e..4341821 100644 --- a/OneSTools.EventLog/OneSTools.EventLog.csproj +++ b/OneSTools.EventLog/OneSTools.EventLog.csproj @@ -15,7 +15,7 @@ https://github.com/akpaevj/OneSTools.EventLog LICENSE - 1.2.3 + 1.2.5 @@ -32,8 +32,8 @@ - - + + diff --git a/OneSTools.EventLog/StreamReaderExtensions.cs b/OneSTools.EventLog/StreamReaderExtensions.cs index 942a2be..90218da 100644 --- a/OneSTools.EventLog/StreamReaderExtensions.cs +++ b/OneSTools.EventLog/StreamReaderExtensions.cs @@ -1,30 +1,33 @@ using System; -using System.Collections.Generic; using System.IO; using System.Reflection; -using System.Text; namespace OneSTools.EventLog { internal static class StreamReaderExtensions { - private static readonly FieldInfo charPosField = typeof(StreamReader).GetField("_charPos", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly); - private static readonly FieldInfo byteLenField = typeof(StreamReader).GetField("_byteLen", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly); - private static readonly FieldInfo charBufferField = typeof(StreamReader).GetField("_charBuffer", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly); + private static readonly FieldInfo charPosField = typeof(StreamReader).GetField("_charPos", + BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly); + + private static readonly FieldInfo byteLenField = typeof(StreamReader).GetField("_byteLen", + BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly); + + private static readonly FieldInfo charBufferField = typeof(StreamReader).GetField("_charBuffer", + BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly); public static long GetPosition(this StreamReader reader) { // shift position back from BaseStream.Position by the number of bytes read // into internal buffer. - int byteLen = (int)byteLenField.GetValue(reader); + var byteLen = (int) byteLenField.GetValue(reader); var position = reader.BaseStream.Position - byteLen; // if we have consumed chars from the buffer we need to calculate how many // bytes they represent in the current encoding and add that to the position. - int charPos = (int)charPosField.GetValue(reader); + var charPos = (int) charPosField.GetValue(reader); if (charPos > 0) { - var charBuffer = (char[])charBufferField.GetValue(reader); + var charBuffer = (char[]) charBufferField.GetValue(reader); var encoding = reader.CurrentEncoding; var bytesConsumed = encoding.GetBytes(charBuffer, 0, charPos).Length; position += bytesConsumed; @@ -42,4 +45,4 @@ public static void SetPosition(this StreamReader reader, long position) throw new Exception("Couldn't set the stream position"); } } -} +} \ No newline at end of file