minimal date for parcing

alex-bochkov · Feb 20, 2017 · eea782a · eea782a
1 parent d857ca7
commit eea782a
Show file tree

Hide file tree

Showing 6 changed files with 92 additions and 24 deletions.
diff --git a/EventLogLoaderManager/AddPath.Designer.vb b/EventLogLoaderManager/AddPath.Designer.vb
diff --git a/EventLogLoaderManager/Form1.Designer.vb b/EventLogLoaderManager/Form1.Designer.vb
diff --git a/EventLogLoaderManager/Form1.vb b/EventLogLoaderManager/Form1.vb
@@ -150,6 +150,7 @@ Public Class Form1
                 IBSetting.DatabaseName = Item.SubItems(0).Text
                 IBSetting.DatabaseCatalog = Item.SubItems(4).Text
                 IBSetting.ESServerName = Item.SubItems(5).Text
+                IBSetting.StartDate = Item.SubItems(6).Text
 
                 NewConfigSetting.Infobases.Add(IBSetting)
 
@@ -228,7 +229,7 @@ Public Class Form1
 
     Private Sub Form1_Load(sender As System.Object, e As System.EventArgs) Handles MyBase.Load
 
-        Text = My.Application.Info.ProductName + " / " + My.Application.Info.Copyright
+        Text = My.Application.Info.ProductName
 
         LoadConfigSetting()
 
@@ -278,6 +279,7 @@ Public Class Form1
                     item1.SubItems.Add(a.SizeEventLog.ToString)
                     item1.SubItems.Add(a.CatalogEventLog)
                     item1.SubItems.Add(SavedIB.ESServerName) 'ESServerName
+                    item1.SubItems.Add(SavedIB.StartDate) 'StartAt
 
                     ListView.Items.Add(item1)
 
@@ -302,6 +304,7 @@ Public Class Form1
                 item1.SubItems.Add(a.SizeEventLog.ToString)
                 item1.SubItems.Add(a.CatalogEventLog)
                 item1.SubItems.Add(SavedIB.ESServerName) 'ESServerName
+                item1.SubItems.Add(SavedIB.StartDate) 'StartAt
 
                 ListView.Items.Add(item1)
 
@@ -321,6 +324,7 @@ Public Class Form1
                 item1.SubItems.Add(CalcullateFolderSize(IB.DatabaseCatalog))
                 item1.SubItems.Add(IB.DatabaseCatalog)
                 item1.SubItems.Add(IB.ESServerName)
+                item1.SubItems.Add(IB.StartDate) 'StartAt
 
                 ListView.Items.Add(item1)
             End If
@@ -489,6 +493,11 @@ Public Class Form1
             item1.SubItems.Add(CalcullateFolderSize(AddPath.Path.Text))
             item1.SubItems.Add(AddPath.Path.Text)
             item1.SubItems.Add(AddPath.ESServerNameTextBox.Text)
+            If AddPath.UseFilterByDateCheckBox.Checked Then
+                item1.SubItems.Add(AddPath.FilterByDateDateTimePicker.Value.ToString("yyyy-MM-dd"))
+            Else
+                item1.SubItems.Add("")
+            End If
 
             ListView.Items.Add(item1)
 
@@ -511,6 +520,15 @@ Public Class Form1
         AddPath.ESServerNameTextBox.Text = item.SubItems(5).Text
         AddPath.Description.Text = item.SubItems(2).Text
         AddPath.Path.Text = item.SubItems(4).Text
+
+        Dim DateStr = item.SubItems(6).Text
+        If String.IsNullOrEmpty(DateStr) Then
+            AddPath.UseFilterByDateCheckBox.Checked = False
+        Else
+            AddPath.UseFilterByDateCheckBox.Checked = True
+            AddPath.FilterByDateDateTimePicker.Value = DateTime.Parse(DateStr)
+        End If
+
         AddPath.ExistedRow = True
         AddPath.ShowDialog()
 
@@ -524,15 +542,14 @@ Public Class Form1
             item.SubItems.Add(CalcullateFolderSize(AddPath.Path.Text))
             item.SubItems.Add(AddPath.Path.Text)
             item.SubItems.Add(AddPath.ESServerNameTextBox.Text)
+            If AddPath.UseFilterByDateCheckBox.Checked Then
+                item.SubItems.Add(AddPath.FilterByDateDateTimePicker.Value.ToString("yyyy-MM-dd"))
+            Else
+                item.SubItems.Add("")
+            End If
 
         End If
 
-
-
-        'End If
-
-
-
     End Sub
 
     Private Sub LinkLabel2_LinkClicked(sender As Object, e As LinkLabelLinkClickedEventArgs) Handles LinkLabel2.LinkClicked

diff --git a/EventLogLoaderService/ConfigSettings.vb b/EventLogLoaderService/ConfigSettings.vb
@@ -7,6 +7,7 @@ Public Module ConfigSettingsModule
         Public DatabaseID As String = ""
         Public DatabaseName As String = ""
         Public DatabaseCatalog As String = ""
+        Public StartDate As String = ""
         Public Found As Boolean = False
     End Class
 

diff --git a/EventLogLoaderService/EventLogProcessor.vb b/EventLogLoaderService/EventLogProcessor.vb
@@ -221,6 +221,7 @@ Public Class EventLogProcessor
     Public ItIsMySQL As Boolean = False
     Public ItIsES As Boolean = False
     Public ESUseSynonymsForFieldsNames As Boolean = False
+    Public LoadEventsStartingAt As Date
     Public ESFieldSynonyms As ElasticSearchFieldSynonymsClass = New ElasticSearchFieldSynonymsClass
     Public SleepTime As Integer = 60 * 1000 '1 минута
 
@@ -1238,19 +1239,23 @@ Public Class EventLogProcessor
                                             [eventCode],
                                             [comment],
                                             [metadataCodes],
-                                            [sessionDataSplitCode],--
+                                            [sessionDataSplitCode],
                                             [dataType],
                                             [data],
                                             [dataPresentation],
                                             [workServerCode],
                                             [primaryPortCode],
                                             [secondaryPortCode]
                                         FROM [EventLog] 
-                                        WHERE [rowID] > @LastEventNumber83 
+                                        WHERE [rowID] > @LastEventNumber83 AND [date] >= @MinimumDate
                                         ORDER BY 1
                                         LIMIT 1000"
 
                 Command.Parameters.AddWithValue("LastEventNumber83", LastEventNumber83)
+
+                Dim noOfSeconds As Int64 = (LoadEventsStartingAt - New Date).TotalSeconds
+                Command.Parameters.AddWithValue("MinimumDate", noOfSeconds * 10000)
+
                 Dim rs = Command.ExecuteReader
 
                 Dim HasData = rs.HasRows
@@ -1307,23 +1312,17 @@ Public Class EventLogProcessor
                     OneEvent.Transaction = ""
                     OneEvent.EventType = ""
 
-
-
                     EventsList.Add(OneEvent)
 
-                    If EventsList.Count >= 1000 Then
-                        'Console.WriteLine("Выгрузка 1000 событий: " + Now.ToString)
-                        SaveEventsToSQL()
-                    End If
-
                     LastEventNumber83 = OneEvent.RowID
 
                 End While
 
-
                 rs.Close()
 
-                SaveEventsToSQL()
+                If EventsList.Count > 0 Then
+                    SaveEventsToSQL()
+                End If
 
                 If Not HasData Then
                     Exit While
@@ -1332,8 +1331,6 @@ Public Class EventLogProcessor
             End While
 
 
-
-
             Command.Dispose()
             Conn.Close()
             Conn.Dispose()
@@ -1523,6 +1520,10 @@ Public Class EventLogProcessor
         OneEvent.DateTime = Date.ParseExact(Array(0), "yyyyMMddHHmmss", provider)
         OneEvent.TransactionStatus = Array(1)
 
+        If OneEvent.DateTime < LoadEventsStartingAt Then
+            Exit Sub
+        End If
+
         Dim TransStr = Array(2).ToString.Replace("}", "").Replace("{", "")
         Dim TransDate = From16To10(TransStr.Substring(0, TransStr.IndexOf(",")))
 

diff --git a/EventLogLoaderService/Service1.vb b/EventLogLoaderService/Service1.vb
@@ -66,6 +66,13 @@ Public Class EventLogLoaderService
                     EventLogProcessorObj.ESUseSynonymsForFieldsNames = ConfigSettingObj.ESUseSynonymsForFieldsNames
                     EventLogProcessorObj.ESFieldSynonyms = ConfigSettingObj.ESFieldSynonyms
 
+                    Try
+                        EventLogProcessorObj.LoadEventsStartingAt = Date.Parse(IBConfig.StartDate)
+                    Catch ex As Exception
+                        EventLogProcessorObj.LoadEventsStartingAt = New Date(1900, 1, 1)
+                    End Try
+
+
                     ListOfProcessors.Add(EventLogProcessorObj)
 
                 Next