Updates. Removed filename, row number. Added more visuals to Power BI…

… including time brush

web/iis/log-analysis/Power BI/prep-cs/IIS2CSV/Program.cs

@@ -18,7 +18,7 @@ static void Main(string[] args)
 
             //string FilePath = @"C:\temp\KCw\Sep 2019\IIS Analysis\us-wclclaweb03\W3SVC8989\u_ex190901.log";
             //IIS2CSV(FilePath);
-            string folderPath = @"C:\temp\KCw\Sep 2019\IIS Analysis\us-wclclaweb03\W3SVC8989\";
+            string folderPath = @"<folder which contains .log files>";
             foreach (string filePath in Directory.GetFiles(folderPath, "*.log"))
             {
                 IIS2CSV(filePath);

web/iis/log-analysis/Power BI/prep-powershell/IIS to CSV.ps1

@@ -1,28 +1,35 @@
 cls
-$input = "<folder which contains .log files>"
+$inputPath = "<folder which contains .log files>"
 $outExt = ".csv" 
-Get-ChildItem $input -Filter *.log | 
+$pathToLogParser= "C:\Program Files (x86)\Log Parser 2.2\logparser"
+Get-ChildItem $inputPath -Filter *.log | 
 Foreach-Object {
  $inFile = $_.FullName
- #$inFile
-
+ "Starting " + $inFile
  $outFile =  $_.FullName -replace "\.log",".csv"
  $outFile
- $cmd = "SELECT Logfilename,date,OUT_ROW_NUMBER() AS RowNumber, time, s-ip, cs-method, cs-uri-stem, cs-uri-query, s-port, cs-username, c-ip, cs(User-Agent) as cs-user-agent, sc-status, sc-substatus, sc-win32-status,sc-bytes,cs-bytes, time-taken INTO '"+$outFile+"' FROM '"+$inFile+"'"
+ $cmd = "SELECT date, time, s-ip, cs-method, cs-uri-stem, cs-uri-query, s-port, cs-username, c-ip, cs(User-Agent) as cs-user-agent, sc-status, sc-substatus, sc-win32-status,sc-bytes,cs-bytes, time-taken INTO '"+$outFile+"' FROM '"+$inFile+"'"
  #$cmd
  $output =  & "C:\Program Files (x86)\Log Parser 2.2\logparser" -i:W3C -o:csv $cmd | Out-String
- "output is " +$output.Length
- # Hack in case the log file dont have sc-bytes or cs-bytes columns as those are not enabled by default.
+ "Output of first conversion " +$output.Length + ". If it is 0, it will rerun with less fields. This takes long time based on log size. Make sure sc-bytes or cs-bytes columns available in IISLogs."
+
+ # Hack in case the log file dont have  as those are not enabled by default.
  # TODO move the default logic to PowerBI if possible.
- # Warning The below step takes time if the log size > 100 MB as PowerShell take time to write. Better make sure the IIS log files have cs-bytes & sc-bytes
+
  if ($output.Length -eq 0) {
     #There may be missing columns such as sc-bytes, cs-bytes. Rerun without those fields & Add default value 0
-    $cmd = "SELECT Logfilename,date,OUT_ROW_NUMBER() AS RowNumber, time, s-ip, cs-method, cs-uri-stem, cs-uri-query, s-port, cs-username, c-ip, cs(User-Agent) as cs-user-agent, sc-status, sc-substatus, sc-win32-status, time-taken INTO '"+$outFile+"' FROM '"+$inFile+"'"
-    $output =  & "C:\Program Files (x86)\Log Parser 2.2\logparser" -i:W3C -o:csv $cmd | Out-String
-    "output of rerun is " +$output.Length
+    $cmd = "SELECT date, time, s-ip, cs-method, cs-uri-stem, cs-uri-query, s-port, cs-username, c-ip, cs(User-Agent) as cs-user-agent, sc-status, sc-substatus, sc-win32-status, time-taken INTO '"+$outFile+"' FROM '"+$inFile+"'"
+    $output =  & $pathToLogParser -i:W3C -o:csv $cmd | Out-String
+    "output of rerun is " +$output.Length + ". Starting addition of missing columns"
     $csv = Import-Csv -Path $outFile 
-    $csv =  $csv | Select-Object LogFileName,RowNumber,time,date, s-ip,cs-method,cs-uri-stem, cs-uri-query, s-port, cs-username, c-ip, cs-User-Agent, sc-status, sc-substatus, sc-win32-status,sc-bytes,cs-bytes,time-taken 
+    "Parsed CSV into PowerShell"
+    $csv =  $csv | Select-Object date,time, s-ip,cs-method,cs-uri-stem, cs-uri-query, s-port, cs-username, c-ip, cs-User-Agent, sc-status, sc-substatus, sc-win32-status,sc-bytes,cs-bytes,time-taken 
+    "Added missing columns"
     $csv | Export-Csv $outFile -NoTypeInformation
+    "Completed column addition " + $inFile
+
+ }else {
+ "Completed file " + $inFile
  }
 }
-"completed"
+"completed all files"