Loki config/ export of forms options/ bug fixes

requests/requests.http

@@ -321,6 +321,7 @@ Content-Type: text/csv
 ###
 # @name exportAll
 GET {{url}}/api/v2/export/all
+Authorization: Bearer {{adminJwtToken.response.body.$.access_token}}
 ###
 
 ###

src/api/VoteMonitor.Api.DataExport/FileGenerator/ExcelFile.cs

@@ -1,8 +1,5 @@
-using Microsoft.EntityFrameworkCore.Metadata.Internal;
-using NPOI.SS.Formula.Functions;
 using NPOI.SS.UserModel;
 using NPOI.XSSF.UserModel;
-using System.Collections;
 using System.ComponentModel;
 using System.Data;
 
@@ -62,8 +59,6 @@ public ExcelFile WithSheet(string sheetName, DataTable dataTable)
             var cell = header.CreateCell(i);
             cell.SetCellValue(headers[i]);
             cell.CellStyle = _headerStyle;
-            // It's heavy, it slows down your Excel if you have large data
-            sheet.AutoSizeColumn(i);
         }
         #endregion
 
@@ -95,8 +90,6 @@ public ExcelFile WithSheet<T>(string sheetName, List<T> exportData) where T:clas
             var cell = header.CreateCell(i);
             cell.SetCellValue(headers[i]);
             cell.CellStyle = _headerStyle;
-            // It's heavy, it slows down your Excel if you have large data
-            sheet.AutoSizeColumn(i);
         }
         #endregion
 

src/api/VoteMonitor.Api.DataExport/Handlers/GetExcelDbCommandHandler.cs

@@ -1,7 +1,6 @@
 using MediatR;
 using Microsoft.EntityFrameworkCore;
 using System.Data;
-using System.Text;
 using VoteMonitor.Api.DataExport.FileGenerator;
 using VoteMonitor.Api.DataExport.Queries;
 using VoteMonitor.Entities;
@@ -20,11 +19,13 @@
     public async Task<byte[]> Handle(GetExcelDbCommand request, CancellationToken cancellationToken)
     {
         var ngos = await _context.Ngos
+            .AsNoTracking()
             .Select(ngo => new { ngo.Id, ngo.Name, ngo.Organizer, })
             .OrderBy(x => x.Id)
             .ToListAsync(cancellationToken: cancellationToken);
 
         var observers = await _context.Observers
+            .AsNoTracking()
             .Select(observer => new
             {
                 observer.Id,
@@ -39,17 +40,20 @@
             .ToListAsync(cancellationToken: cancellationToken);
 
         var provinces = await _context.Provinces
+            .AsNoTracking()
             .OrderBy(x => x.Order)
             .Select(province => new { province.Id, province.Code, province.Name })
             .ToListAsync(cancellationToken: cancellationToken);
 
         var counties = await _context.Counties
+            .AsNoTracking()
             .Include(x => x.Province)
             .OrderBy(x => x.Order)
             .Select(county => new { county.Id, county.Code, ProvinceCode = county.Province.Code, county.Name, county.Diaspora })
             .ToListAsync(cancellationToken: cancellationToken);
 
         var municipalities = await _context.Municipalities
+            .AsNoTracking()
             .Include(x => x.County)
             .OrderBy(x => x.Order)
             .Select(municipality => new { municipality.Id, municipality.Code, CountyCode = municipality.County.Code, municipality.Name })
@@ -67,9 +71,34 @@
                 pollingStation.Number,
                 pollingStation.Address
             })
-
             .ToListAsync(cancellationToken: cancellationToken);
 
+        var aggregatedForms = await GetForms(cancellationToken);
+        var filledInForms = await GetFilledInForms(cancellationToken);
+        var notes = await GetNotesForExport(cancellationToken);
+
+        var excelBuilder = ExcelFile
+            .New()
+            .WithSheet("ngos", ngos)
+            .WithSheet("observers", observers)
+            .WithSheet("provinces", provinces)
+            .WithSheet("counties", counties)
+            .WithSheet("municipalities", municipalities)
+            .WithSheet("polling-stations", pollingStations)
+            .WithSheet("forms", aggregatedForms);
+
+        filledInForms.ForEach(f =>
+        {
+            excelBuilder = excelBuilder.WithSheet(f.Code, f.Data);
+        });
+
+        excelBuilder = excelBuilder.WithSheet("notes", notes);
+
+        return excelBuilder.Write();
+    }
+
+    private async Task<DataTable> GetForms(CancellationToken cancellationToken)
+    {
         var forms = await _context.Forms
             .AsNoTracking()
             .Include(f => f.FormSections)
@@ -80,17 +109,18 @@
             .OrderBy(f => f.Order)
             .ToListAsync(cancellationToken);
 
-        var aggregatedForms = forms.SelectMany(form => form.FormSections
+        var questions = forms.SelectMany(form => form.FormSections
                 .OrderBy(x => x.OrderNumber)
                 .SelectMany(formSection => formSection.Questions.OrderBy(x => x.OrderNumber)
                     .Select(question => new
                     {
                         FormCode = form.Code,
+                        FormOrderNumber = form.Order,
                         FormSectionCode = formSection.Code,
-                        FormSectionDescription = formSection.Description,
                         QuestionCode = question.Code,
-                        QuestionQuestionType = question.QuestionType,
+                        QuestionType = question.QuestionType,
                         QuestionText = question.Text,
+                        QuestionOrderNumber = question.OrderNumber,
                         Options = question.OptionsToQuestions
                             .OrderBy(x => x.Option.OrderNumber)
                             .Select(optionToQuestion => new
@@ -101,30 +131,41 @@
                             })
                             .ToList()
                     })))
+            .OrderBy(q => q.FormOrderNumber)
+            .ThenBy(q => q.QuestionOrderNumber)
             .ToList();
 
-        var filledInForms = await GetFilledInForms(cancellationToken);
+        DataTable dataTable = new DataTable();
 
-        var notes = await GetNotesForExport(cancellationToken);
+        dataTable.Columns.Add("FormCode", typeof(string));
+        dataTable.Columns.Add("FormSectionCode", typeof(string));
+        dataTable.Columns.Add("QuestionCode", typeof(string));
+        dataTable.Columns.Add("Question", typeof(string));
+        dataTable.Columns.Add("Type", typeof(string));
+        var maxNumberOfOptions = questions.Select(x => x.Options.Count).DefaultIfEmpty(0).Max();
 
-        var excelBuilder = ExcelFile
-            .New()
-            .WithSheet("ngos", ngos)
-            .WithSheet("observers", observers)
-            .WithSheet("provinces", provinces)
-            .WithSheet("counties", counties)
-            .WithSheet("municipalities", municipalities)
-            .WithSheet("polling-stations", pollingStations)
-            .WithSheet("forms", aggregatedForms);
+        for (int i = 1; i <= maxNumberOfOptions; i++)
+        {
+            dataTable.Columns.Add($"Options-{i}", typeof(string));
+        }
 
-        filledInForms.ForEach(f =>
+        foreach (var question in questions)
         {
-            excelBuilder = excelBuilder.WithSheet(f.Code, f.Data);
-        });
+            object?[] rowValues = new List<object?>
+                {
+                    question.FormCode,
+                    question.FormSectionCode,
+                    question.QuestionCode,
+                    question.QuestionText,
+                    GetFormattedQuestionType(question.QuestionType),
+                }
+                .Union(question.Options.Select(x => FormatOption(x.Text, x.IsFreeText, x.IsFlagged)))
+                .ToArray();
 
-        excelBuilder = excelBuilder.WithSheet("notes", notes);
+            dataTable.Rows.Add(rowValues);
+        }
 
-        return excelBuilder.Write();
+        return dataTable;
     }
 
     private async Task<DataTable> GetNotesForExport(CancellationToken cancellationToken)
@@ -170,7 +211,7 @@
 
         foreach (var note in notes)
         {
             object?[] rowValues = new List<object?>
                 {
                     note.ObserverId,
                     note.ObserverPhone,
@@ -201,6 +242,7 @@
     private async Task<List<(string Code, DataTable Data)>> GetFilledInForms(CancellationToken cancellationToken)
     {
         var answers = await _context.Answers
+            .AsNoTracking()
             .Include(a => a.Observer)
             .Include(a => a.PollingStation)
             .ThenInclude(x => x.Municipality)
@@ -296,7 +338,7 @@
         return result;
     }
 
-    private string GetSelectedOptionText(string optionText, string enteredFreeText)
+    private static string GetSelectedOptionText(string optionText, string enteredFreeText)
     {
         if (!string.IsNullOrWhiteSpace(enteredFreeText))
         {
@@ -305,4 +347,29 @@
 
         return optionText;
     }
+
+    private static string GetFormattedQuestionType(QuestionType questionType)
+    {
+        switch (questionType)
+        {
+            case QuestionType.MultipleOption:
+                return "multiple choice";
+            case QuestionType.SingleOption:
+                return "single choice";
+            case QuestionType.MultipleOptionWithText:
+                return "multiple choice with text";
+            case QuestionType.SingleOptionWithText:
+                return "single choice with text";
+            default:
+                return "unknown";
+        }
+    }
+
+    private static string FormatOption(string text, bool isFreeText, bool isFlagged)
+    {
+        string isFreeTextFlag = isFreeText ? "$text" : string.Empty;
+        string isFlaggedFlag = isFlagged ? "$flagged" : string.Empty;
+
+        return $"{text}{isFreeTextFlag}{isFlaggedFlag}";
+    }
 }

src/api/VoteMonitor.Api.Form/Models/FormDetailsModel.cs

@@ -13,7 +13,7 @@ public class FormDetailsModel
     [JsonPropertyName("description")]
     public string Description { get; set; }
 
-    [JsonPropertyName("version")]
+    [JsonPropertyName("currentVersion")]
     public int CurrentVersion { get; set; }
 
     // quick and dirty fix to sync iOS and Android

src/api/VoteMonitor.Api/Extensions/HealthChecks/S3HealthChecksExtensions.cs

@@ -42,4 +42,4 @@ public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context
             return new HealthCheckResult(context.Registration.FailureStatus, exception: ex);
         }
     }
-}
+}

src/api/VoteMonitor.Api/Extensions/LoggingConfiguration.cs

@@ -2,7 +2,6 @@
 using Serilog.Core;
 using Serilog.Events;
 using Serilog.Exceptions;
-using Serilog.Formatting.Compact;
 using Serilog.Sinks.Grafana.Loki;
 
 namespace VoteMonitor.Api.Extensions;
@@ -34,7 +33,7 @@ public static void AddLoggingConfiguration(this IHostBuilder host, IConfiguratio
             .Enrich.WithProperty("Application", env.ApplicationName)
             .Enrich.WithExceptionDetails()
             .WriteTo.Console()
-            .WriteTo.GrafanaLoki(configuration["LokiConfig:Uri"], propertiesAsLabels: new[] { "Environment", "Application" });
+            .WriteTo.GrafanaLoki(configuration["LokiConfig:Uri"], credentials: lokiCredentials, propertiesAsLabels: new[] { "Environment", "Application" });
 
         Log.Logger = logger.CreateLogger();
 

src/api/VoteMonitor.Api/appsettings.json

@@ -8,7 +8,7 @@
         }
     },
     "LokiConfig": {
-        "Uri": "",
+        "Uri": "http://localhost:3100",
         "User": "",
         "Password": ""
     },

src/docker-compose.yml

@@ -70,3 +70,21 @@ services:
     ports:
       - '6379:6379'
     command: redis-server --save 20 1 --loglevel warning
+
+  loki:
+    image: grafana/loki:master
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+    networks:
+      - loki
+
+  grafana:
+    image: grafana/grafana:master
+    ports:
+      - "3000:3000"
+    networks:
+      - loki
+
+networks:
+  loki:

terraform/secrets.tf

@@ -69,3 +69,17 @@ resource "aws_secretsmanager_secret_version" "firebase_serverkey" {
   secret_id     = aws_secretsmanager_secret.firebase_serverkey.id
   secret_string = var.firebase_serverkey
 }
+
+resource "aws_secretsmanager_secret" "loki" {
+  name = "${local.namespace}-loki-${random_string.secrets_suffix.result}"
+}
+
+resource "aws_secretsmanager_secret_version" "loki" {
+  secret_id = aws_secretsmanager_secret.loki.id
+
+  secret_string = jsonencode({
+    "uri"      = var.loki_uri
+    "user"     = var.loki_user
+    "password" = var.loki_password
+  })
+}

terraform/service_api.tf

@@ -66,7 +66,7 @@ module "ecs_api" {
     },
     {
       name  = "MobileSecurityOptions__InvalidCredentialsErrorMessage"
-      value = "{ \"error\": \"A aparut o eroare la logarea in aplicatie. Va rugam sa verificati ca ati introdus corect numarul de telefon si codul de acces, iar daca eroarea persista va rugam contactati serviciul de suport la numarul 07......\" }"
+      value = var.invalid_credentials_error_message
     },
     {
       name  = "MobileSecurityOptions__LockDevice"
@@ -143,6 +143,18 @@ module "ecs_api" {
       name      = "HashOptions__Salt"
       valueFrom = aws_secretsmanager_secret.hash_salt.arn
     },
+    {
+      name      = "LokiConfig__Uri"
+      valueFrom = "${aws_secretsmanager_secret.loki.arn}:uri::"
+    },
+    {
+      name      = "LokiConfig__User"
+      valueFrom = "${aws_secretsmanager_secret.loki.arn}:user::"
+    },
+    {
+      name      = "LokiConfig__Password"
+      valueFrom = "${aws_secretsmanager_secret.loki.arn}:password::"
+    },
 
   ]
 
@@ -151,6 +163,7 @@ module "ecs_api" {
     aws_secretsmanager_secret.jwt_signing_key.arn,
     aws_secretsmanager_secret.hash_salt.arn,
     aws_secretsmanager_secret.rds.arn,
+    aws_secretsmanager_secret.loki.arn,
   ]
 }