Loki config/ export of forms options/ bug fixes (#415)

* Loki config/ export of forms options/ bug fixes * fix bug with loki credentials not sent * export form options * add tf secrets for loki config * fix currentVersion for iOS * consolidate loki credentials * configurable invalid credentials error message * set loki defaults --------- Co-authored-by: Andrei Ioniță <[email protected]>
code4romania · Oct 13, 2023 · c4eb93d · c4eb93d
1 parent c5df5d6
commit c4eb93d
Show file tree

Hide file tree

Showing 12 changed files with 164 additions and 55 deletions.
diff --git a/requests/requests.http b/requests/requests.http
@@ -321,6 +321,7 @@ Content-Type: text/csv
 ###
 # @name exportAll
 GET {{url}}/api/v2/export/all
+Authorization: Bearer {{adminJwtToken.response.body.$.access_token}}
 ###
 
 ###

diff --git a/src/api/VoteMonitor.Api.DataExport/FileGenerator/ExcelFile.cs b/src/api/VoteMonitor.Api.DataExport/FileGenerator/ExcelFile.cs
@@ -1,8 +1,5 @@
-using Microsoft.EntityFrameworkCore.Metadata.Internal;
-using NPOI.SS.Formula.Functions;
 using NPOI.SS.UserModel;
 using NPOI.XSSF.UserModel;
-using System.Collections;
 using System.ComponentModel;
 using System.Data;
 
@@ -62,8 +59,6 @@ public ExcelFile WithSheet(string sheetName, DataTable dataTable)
             var cell = header.CreateCell(i);
             cell.SetCellValue(headers[i]);
             cell.CellStyle = _headerStyle;
-            // It's heavy, it slows down your Excel if you have large data
-            sheet.AutoSizeColumn(i);
         }
         #endregion
 
@@ -95,8 +90,6 @@ public ExcelFile WithSheet<T>(string sheetName, List<T> exportData) where T:clas
             var cell = header.CreateCell(i);
             cell.SetCellValue(headers[i]);
             cell.CellStyle = _headerStyle;
-            // It's heavy, it slows down your Excel if you have large data
-            sheet.AutoSizeColumn(i);
         }
         #endregion
 

diff --git a/src/api/VoteMonitor.Api.DataExport/Handlers/GetExcelDbCommandHandler.cs b/src/api/VoteMonitor.Api.DataExport/Handlers/GetExcelDbCommandHandler.cs
@@ -1,7 +1,6 @@
 using MediatR;
 using Microsoft.EntityFrameworkCore;
 using System.Data;
-using System.Text;
 using VoteMonitor.Api.DataExport.FileGenerator;
 using VoteMonitor.Api.DataExport.Queries;
 using VoteMonitor.Entities;
@@ -20,11 +19,13 @@ public GetExcelDbCommandHandler(VoteMonitorContext context)
     public async Task<byte[]> Handle(GetExcelDbCommand request, CancellationToken cancellationToken)
     {
         var ngos = await _context.Ngos
+            .AsNoTracking()
             .Select(ngo => new { ngo.Id, ngo.Name, ngo.Organizer, })
             .OrderBy(x => x.Id)
             .ToListAsync(cancellationToken: cancellationToken);
 
         var observers = await _context.Observers
+            .AsNoTracking()
             .Select(observer => new
             {
                 observer.Id,
@@ -39,17 +40,20 @@ public async Task<byte[]> Handle(GetExcelDbCommand request, CancellationToken ca
             .ToListAsync(cancellationToken: cancellationToken);
 
         var provinces = await _context.Provinces
+            .AsNoTracking()
             .OrderBy(x => x.Order)
             .Select(province => new { province.Id, province.Code, province.Name })
             .ToListAsync(cancellationToken: cancellationToken);
 
         var counties = await _context.Counties
+            .AsNoTracking()
             .Include(x => x.Province)
             .OrderBy(x => x.Order)
             .Select(county => new { county.Id, county.Code, ProvinceCode = county.Province.Code, county.Name, county.Diaspora })
             .ToListAsync(cancellationToken: cancellationToken);
 
         var municipalities = await _context.Municipalities
+            .AsNoTracking()
             .Include(x => x.County)
             .OrderBy(x => x.Order)
             .Select(municipality => new { municipality.Id, municipality.Code, CountyCode = municipality.County.Code, municipality.Name })
@@ -67,9 +71,34 @@ public async Task<byte[]> Handle(GetExcelDbCommand request, CancellationToken ca
                 pollingStation.Number,
                 pollingStation.Address
             })
-
             .ToListAsync(cancellationToken: cancellationToken);
 
+        var aggregatedForms = await GetForms(cancellationToken);
+        var filledInForms = await GetFilledInForms(cancellationToken);
+        var notes = await GetNotesForExport(cancellationToken);
+
+        var excelBuilder = ExcelFile
+            .New()
+            .WithSheet("ngos", ngos)
+            .WithSheet("observers", observers)
+            .WithSheet("provinces", provinces)
+            .WithSheet("counties", counties)
+            .WithSheet("municipalities", municipalities)
+            .WithSheet("polling-stations", pollingStations)
+            .WithSheet("forms", aggregatedForms);
+
+        filledInForms.ForEach(f =>
+        {
+            excelBuilder = excelBuilder.WithSheet(f.Code, f.Data);
+        });
+
+        excelBuilder = excelBuilder.WithSheet("notes", notes);
+
+        return excelBuilder.Write();
+    }
+
+    private async Task<DataTable> GetForms(CancellationToken cancellationToken)
+    {
         var forms = await _context.Forms
             .AsNoTracking()
             .Include(f => f.FormSections)
@@ -80,17 +109,18 @@ public async Task<byte[]> Handle(GetExcelDbCommand request, CancellationToken ca
             .OrderBy(f => f.Order)
             .ToListAsync(cancellationToken);
 
-        var aggregatedForms = forms.SelectMany(form => form.FormSections
+        var questions = forms.SelectMany(form => form.FormSections
                 .OrderBy(x => x.OrderNumber)
                 .SelectMany(formSection => formSection.Questions.OrderBy(x => x.OrderNumber)
                     .Select(question => new
                     {
                         FormCode = form.Code,
+                        FormOrderNumber = form.Order,
                         FormSectionCode = formSection.Code,
-                        FormSectionDescription = formSection.Description,
                         QuestionCode = question.Code,
-                        QuestionQuestionType = question.QuestionType,
+                        QuestionType = question.QuestionType,
                         QuestionText = question.Text,
+                        QuestionOrderNumber = question.OrderNumber,
                         Options = question.OptionsToQuestions
                             .OrderBy(x => x.Option.OrderNumber)
                             .Select(optionToQuestion => new
@@ -101,30 +131,41 @@ public async Task<byte[]> Handle(GetExcelDbCommand request, CancellationToken ca
                             })
                             .ToList()
                     })))
+            .OrderBy(q => q.FormOrderNumber)
+            .ThenBy(q => q.QuestionOrderNumber)
             .ToList();
 
-        var filledInForms = await GetFilledInForms(cancellationToken);
+        DataTable dataTable = new DataTable();
 
-        var notes = await GetNotesForExport(cancellationToken);
+        dataTable.Columns.Add("FormCode", typeof(string));
+        dataTable.Columns.Add("FormSectionCode", typeof(string));
+        dataTable.Columns.Add("QuestionCode", typeof(string));
+        dataTable.Columns.Add("Question", typeof(string));
+        dataTable.Columns.Add("Type", typeof(string));
+        var maxNumberOfOptions = questions.Select(x => x.Options.Count).DefaultIfEmpty(0).Max();
 
-        var excelBuilder = ExcelFile
-            .New()
-            .WithSheet("ngos", ngos)
-            .WithSheet("observers", observers)
-            .WithSheet("provinces", provinces)
-            .WithSheet("counties", counties)
-            .WithSheet("municipalities", municipalities)
-            .WithSheet("polling-stations", pollingStations)
-            .WithSheet("forms", aggregatedForms);
+        for (int i = 1; i <= maxNumberOfOptions; i++)
+        {
+            dataTable.Columns.Add($"Options-{i}", typeof(string));
+        }
 
-        filledInForms.ForEach(f =>
+        foreach (var question in questions)
         {
-            excelBuilder = excelBuilder.WithSheet(f.Code, f.Data);
-        });
+            object?[] rowValues = new List<object?>
+                {
+                    question.FormCode,
+                    question.FormSectionCode,
+                    question.QuestionCode,
+                    question.QuestionText,
+                    GetFormattedQuestionType(question.QuestionType),
+                }
+                .Union(question.Options.Select(x => FormatOption(x.Text, x.IsFreeText, x.IsFlagged)))
+                .ToArray();
 
-        excelBuilder = excelBuilder.WithSheet("notes", notes);
+            dataTable.Rows.Add(rowValues);
+        }
 
-        return excelBuilder.Write();
+        return dataTable;
     }
 
     private async Task<DataTable> GetNotesForExport(CancellationToken cancellationToken)
@@ -201,6 +242,7 @@ private static string GetNoteLocation(string pollingStation, string formCode, st
     private async Task<List<(string Code, DataTable Data)>> GetFilledInForms(CancellationToken cancellationToken)
     {
         var answers = await _context.Answers
+            .AsNoTracking()
             .Include(a => a.Observer)
             .Include(a => a.PollingStation)
             .ThenInclude(x => x.Municipality)
@@ -296,7 +338,7 @@ private static string GetNoteLocation(string pollingStation, string formCode, st
         return result;
     }
 
-    private string GetSelectedOptionText(string optionText, string enteredFreeText)
+    private static string GetSelectedOptionText(string optionText, string enteredFreeText)
     {
         if (!string.IsNullOrWhiteSpace(enteredFreeText))
         {
@@ -305,4 +347,29 @@ private string GetSelectedOptionText(string optionText, string enteredFreeText)
 
         return optionText;
     }
+
+    private static string GetFormattedQuestionType(QuestionType questionType)
+    {
+        switch (questionType)
+        {
+            case QuestionType.MultipleOption:
+                return "multiple choice";
+            case QuestionType.SingleOption:
+                return "single choice";
+            case QuestionType.MultipleOptionWithText:
+                return "multiple choice with text";
+            case QuestionType.SingleOptionWithText:
+                return "single choice with text";
+            default:
+                return "unknown";
+        }
+    }
+
+    private static string FormatOption(string text, bool isFreeText, bool isFlagged)
+    {
+        string isFreeTextFlag = isFreeText ? "$text" : string.Empty;
+        string isFlaggedFlag = isFlagged ? "$flagged" : string.Empty;
+
+        return $"{text}{isFreeTextFlag}{isFlaggedFlag}";
+    }
 }
diff --git a/src/api/VoteMonitor.Api.Form/Models/FormDetailsModel.cs b/src/api/VoteMonitor.Api.Form/Models/FormDetailsModel.cs
@@ -13,7 +13,7 @@ public class FormDetailsModel
     [JsonPropertyName("description")]
     public string Description { get; set; }
 
-    [JsonPropertyName("version")]
+    [JsonPropertyName("currentVersion")]
     public int CurrentVersion { get; set; }
 
     // quick and dirty fix to sync iOS and Android

diff --git a/src/api/VoteMonitor.Api/Extensions/HealthChecks/S3HealthChecksExtensions.cs b/src/api/VoteMonitor.Api/Extensions/HealthChecks/S3HealthChecksExtensions.cs
@@ -42,4 +42,4 @@ public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context
             return new HealthCheckResult(context.Registration.FailureStatus, exception: ex);
         }
     }
-}
+}
diff --git a/src/api/VoteMonitor.Api/Extensions/LoggingConfiguration.cs b/src/api/VoteMonitor.Api/Extensions/LoggingConfiguration.cs
@@ -2,7 +2,6 @@
 using Serilog.Core;
 using Serilog.Events;
 using Serilog.Exceptions;
-using Serilog.Formatting.Compact;
 using Serilog.Sinks.Grafana.Loki;
 
 namespace VoteMonitor.Api.Extensions;
@@ -34,7 +33,7 @@ public static void AddLoggingConfiguration(this IHostBuilder host, IConfiguratio
             .Enrich.WithProperty("Application", env.ApplicationName)
             .Enrich.WithExceptionDetails()
             .WriteTo.Console()
-            .WriteTo.GrafanaLoki(configuration["LokiConfig:Uri"], propertiesAsLabels: new[] { "Environment", "Application" });
+            .WriteTo.GrafanaLoki(configuration["LokiConfig:Uri"], credentials: lokiCredentials, propertiesAsLabels: new[] { "Environment", "Application" });
 
         Log.Logger = logger.CreateLogger();
 

diff --git a/src/api/VoteMonitor.Api/appsettings.json b/src/api/VoteMonitor.Api/appsettings.json
@@ -8,7 +8,7 @@
         }
     },
     "LokiConfig": {
-        "Uri": "",
+        "Uri": "http://localhost:3100",
         "User": "",
         "Password": ""
     },

diff --git a/src/docker-compose.yml b/src/docker-compose.yml
@@ -70,3 +70,21 @@ services:
     ports:
       - '6379:6379'
     command: redis-server --save 20 1 --loglevel warning
+
+  loki:
+    image: grafana/loki:master
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+    networks:
+      - loki
+
+  grafana:
+    image: grafana/grafana:master
+    ports:
+      - "3000:3000"
+    networks:
+      - loki
+
+networks:
+  loki:
diff --git a/src/grafana-docker-compose.yml b/src/grafana-docker-compose.yml
diff --git a/terraform/secrets.tf b/terraform/secrets.tf
@@ -69,3 +69,17 @@ resource "aws_secretsmanager_secret_version" "firebase_serverkey" {
   secret_id     = aws_secretsmanager_secret.firebase_serverkey.id
   secret_string = var.firebase_serverkey
 }
+
+resource "aws_secretsmanager_secret" "loki" {
+  name = "${local.namespace}-loki-${random_string.secrets_suffix.result}"
+}
+
+resource "aws_secretsmanager_secret_version" "loki" {
+  secret_id = aws_secretsmanager_secret.loki.id
+
+  secret_string = jsonencode({
+    "uri"      = var.loki_uri
+    "user"     = var.loki_user
+    "password" = var.loki_password
+  })
+}
diff --git a/terraform/service_api.tf b/terraform/service_api.tf
@@ -66,7 +66,7 @@ module "ecs_api" {
     },
     {
       name  = "MobileSecurityOptions__InvalidCredentialsErrorMessage"
-      value = "{ \"error\": \"A aparut o eroare la logarea in aplicatie. Va rugam sa verificati ca ati introdus corect numarul de telefon si codul de acces, iar daca eroarea persista va rugam contactati serviciul de suport la numarul 07......\" }"
+      value = var.invalid_credentials_error_message
     },
     {
       name  = "MobileSecurityOptions__LockDevice"
@@ -143,6 +143,18 @@ module "ecs_api" {
       name      = "HashOptions__Salt"
       valueFrom = aws_secretsmanager_secret.hash_salt.arn
     },
+    {
+      name      = "LokiConfig__Uri"
+      valueFrom = "${aws_secretsmanager_secret.loki.arn}:uri::"
+    },
+    {
+      name      = "LokiConfig__User"
+      valueFrom = "${aws_secretsmanager_secret.loki.arn}:user::"
+    },
+    {
+      name      = "LokiConfig__Password"
+      valueFrom = "${aws_secretsmanager_secret.loki.arn}:password::"
+    },
 
   ]
 
@@ -151,6 +163,7 @@ module "ecs_api" {
     aws_secretsmanager_secret.jwt_signing_key.arn,
     aws_secretsmanager_secret.hash_salt.arn,
     aws_secretsmanager_secret.rds.arn,
+    aws_secretsmanager_secret.loki.arn,
   ]
 }