From c4eb93d3c9028cd38ee56acf4a5f1942c0e823cd Mon Sep 17 00:00:00 2001 From: Ion Dormenco Date: Fri, 13 Oct 2023 12:10:44 +0300 Subject: [PATCH] Loki config/ export of forms options/ bug fixes (#415) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Loki config/ export of forms options/ bug fixes * fix bug with loki credentials not sent * export form options * add tf secrets for loki config * fix currentVersion for iOS * consolidate loki credentials * configurable invalid credentials error message * set loki defaults --------- Co-authored-by: Andrei Ioniță --- requests/requests.http | 1 + .../FileGenerator/ExcelFile.cs | 7 -- .../Handlers/GetExcelDbCommandHandler.cs | 111 ++++++++++++++---- .../Models/FormDetailsModel.cs | 2 +- .../HealthChecks/S3HealthChecksExtensions.cs | 2 +- .../Extensions/LoggingConfiguration.cs | 3 +- src/api/VoteMonitor.Api/appsettings.json | 2 +- src/docker-compose.yml | 18 +++ src/grafana-docker-compose.yml | 20 ---- terraform/secrets.tf | 14 +++ terraform/service_api.tf | 15 ++- terraform/variables.tf | 24 ++++ 12 files changed, 164 insertions(+), 55 deletions(-) delete mode 100644 src/grafana-docker-compose.yml diff --git a/requests/requests.http b/requests/requests.http index 8a6a3cd5..c67f197a 100644 --- a/requests/requests.http +++ b/requests/requests.http @@ -321,6 +321,7 @@ Content-Type: text/csv ### # @name exportAll GET {{url}}/api/v2/export/all +Authorization: Bearer {{adminJwtToken.response.body.$.access_token}} ### ### diff --git a/src/api/VoteMonitor.Api.DataExport/FileGenerator/ExcelFile.cs b/src/api/VoteMonitor.Api.DataExport/FileGenerator/ExcelFile.cs index 5efcfd3d..a173b15c 100644 --- a/src/api/VoteMonitor.Api.DataExport/FileGenerator/ExcelFile.cs +++ b/src/api/VoteMonitor.Api.DataExport/FileGenerator/ExcelFile.cs @@ -1,8 +1,5 @@ -using Microsoft.EntityFrameworkCore.Metadata.Internal; -using NPOI.SS.Formula.Functions; using NPOI.SS.UserModel; using NPOI.XSSF.UserModel; -using System.Collections; using System.ComponentModel; using System.Data; @@ -62,8 +59,6 @@ public ExcelFile WithSheet(string sheetName, DataTable dataTable) var cell = header.CreateCell(i); cell.SetCellValue(headers[i]); cell.CellStyle = _headerStyle; - // It's heavy, it slows down your Excel if you have large data - sheet.AutoSizeColumn(i); } #endregion @@ -95,8 +90,6 @@ public ExcelFile WithSheet(string sheetName, List exportData) where T:clas var cell = header.CreateCell(i); cell.SetCellValue(headers[i]); cell.CellStyle = _headerStyle; - // It's heavy, it slows down your Excel if you have large data - sheet.AutoSizeColumn(i); } #endregion diff --git a/src/api/VoteMonitor.Api.DataExport/Handlers/GetExcelDbCommandHandler.cs b/src/api/VoteMonitor.Api.DataExport/Handlers/GetExcelDbCommandHandler.cs index e9073f21..a859eb27 100644 --- a/src/api/VoteMonitor.Api.DataExport/Handlers/GetExcelDbCommandHandler.cs +++ b/src/api/VoteMonitor.Api.DataExport/Handlers/GetExcelDbCommandHandler.cs @@ -1,7 +1,6 @@ using MediatR; using Microsoft.EntityFrameworkCore; using System.Data; -using System.Text; using VoteMonitor.Api.DataExport.FileGenerator; using VoteMonitor.Api.DataExport.Queries; using VoteMonitor.Entities; @@ -20,11 +19,13 @@ public GetExcelDbCommandHandler(VoteMonitorContext context) public async Task Handle(GetExcelDbCommand request, CancellationToken cancellationToken) { var ngos = await _context.Ngos + .AsNoTracking() .Select(ngo => new { ngo.Id, ngo.Name, ngo.Organizer, }) .OrderBy(x => x.Id) .ToListAsync(cancellationToken: cancellationToken); var observers = await _context.Observers + .AsNoTracking() .Select(observer => new { observer.Id, @@ -39,17 +40,20 @@ public async Task Handle(GetExcelDbCommand request, CancellationToken ca .ToListAsync(cancellationToken: cancellationToken); var provinces = await _context.Provinces + .AsNoTracking() .OrderBy(x => x.Order) .Select(province => new { province.Id, province.Code, province.Name }) .ToListAsync(cancellationToken: cancellationToken); var counties = await _context.Counties + .AsNoTracking() .Include(x => x.Province) .OrderBy(x => x.Order) .Select(county => new { county.Id, county.Code, ProvinceCode = county.Province.Code, county.Name, county.Diaspora }) .ToListAsync(cancellationToken: cancellationToken); var municipalities = await _context.Municipalities + .AsNoTracking() .Include(x => x.County) .OrderBy(x => x.Order) .Select(municipality => new { municipality.Id, municipality.Code, CountyCode = municipality.County.Code, municipality.Name }) @@ -67,9 +71,34 @@ public async Task Handle(GetExcelDbCommand request, CancellationToken ca pollingStation.Number, pollingStation.Address }) - .ToListAsync(cancellationToken: cancellationToken); + var aggregatedForms = await GetForms(cancellationToken); + var filledInForms = await GetFilledInForms(cancellationToken); + var notes = await GetNotesForExport(cancellationToken); + + var excelBuilder = ExcelFile + .New() + .WithSheet("ngos", ngos) + .WithSheet("observers", observers) + .WithSheet("provinces", provinces) + .WithSheet("counties", counties) + .WithSheet("municipalities", municipalities) + .WithSheet("polling-stations", pollingStations) + .WithSheet("forms", aggregatedForms); + + filledInForms.ForEach(f => + { + excelBuilder = excelBuilder.WithSheet(f.Code, f.Data); + }); + + excelBuilder = excelBuilder.WithSheet("notes", notes); + + return excelBuilder.Write(); + } + + private async Task GetForms(CancellationToken cancellationToken) + { var forms = await _context.Forms .AsNoTracking() .Include(f => f.FormSections) @@ -80,17 +109,18 @@ public async Task Handle(GetExcelDbCommand request, CancellationToken ca .OrderBy(f => f.Order) .ToListAsync(cancellationToken); - var aggregatedForms = forms.SelectMany(form => form.FormSections + var questions = forms.SelectMany(form => form.FormSections .OrderBy(x => x.OrderNumber) .SelectMany(formSection => formSection.Questions.OrderBy(x => x.OrderNumber) .Select(question => new { FormCode = form.Code, + FormOrderNumber = form.Order, FormSectionCode = formSection.Code, - FormSectionDescription = formSection.Description, QuestionCode = question.Code, - QuestionQuestionType = question.QuestionType, + QuestionType = question.QuestionType, QuestionText = question.Text, + QuestionOrderNumber = question.OrderNumber, Options = question.OptionsToQuestions .OrderBy(x => x.Option.OrderNumber) .Select(optionToQuestion => new @@ -101,30 +131,41 @@ public async Task Handle(GetExcelDbCommand request, CancellationToken ca }) .ToList() }))) + .OrderBy(q => q.FormOrderNumber) + .ThenBy(q => q.QuestionOrderNumber) .ToList(); - var filledInForms = await GetFilledInForms(cancellationToken); + DataTable dataTable = new DataTable(); - var notes = await GetNotesForExport(cancellationToken); + dataTable.Columns.Add("FormCode", typeof(string)); + dataTable.Columns.Add("FormSectionCode", typeof(string)); + dataTable.Columns.Add("QuestionCode", typeof(string)); + dataTable.Columns.Add("Question", typeof(string)); + dataTable.Columns.Add("Type", typeof(string)); + var maxNumberOfOptions = questions.Select(x => x.Options.Count).DefaultIfEmpty(0).Max(); - var excelBuilder = ExcelFile - .New() - .WithSheet("ngos", ngos) - .WithSheet("observers", observers) - .WithSheet("provinces", provinces) - .WithSheet("counties", counties) - .WithSheet("municipalities", municipalities) - .WithSheet("polling-stations", pollingStations) - .WithSheet("forms", aggregatedForms); + for (int i = 1; i <= maxNumberOfOptions; i++) + { + dataTable.Columns.Add($"Options-{i}", typeof(string)); + } - filledInForms.ForEach(f => + foreach (var question in questions) { - excelBuilder = excelBuilder.WithSheet(f.Code, f.Data); - }); + object?[] rowValues = new List + { + question.FormCode, + question.FormSectionCode, + question.QuestionCode, + question.QuestionText, + GetFormattedQuestionType(question.QuestionType), + } + .Union(question.Options.Select(x => FormatOption(x.Text, x.IsFreeText, x.IsFlagged))) + .ToArray(); - excelBuilder = excelBuilder.WithSheet("notes", notes); + dataTable.Rows.Add(rowValues); + } - return excelBuilder.Write(); + return dataTable; } private async Task GetNotesForExport(CancellationToken cancellationToken) @@ -201,6 +242,7 @@ private static string GetNoteLocation(string pollingStation, string formCode, st private async Task> GetFilledInForms(CancellationToken cancellationToken) { var answers = await _context.Answers + .AsNoTracking() .Include(a => a.Observer) .Include(a => a.PollingStation) .ThenInclude(x => x.Municipality) @@ -296,7 +338,7 @@ private static string GetNoteLocation(string pollingStation, string formCode, st return result; } - private string GetSelectedOptionText(string optionText, string enteredFreeText) + private static string GetSelectedOptionText(string optionText, string enteredFreeText) { if (!string.IsNullOrWhiteSpace(enteredFreeText)) { @@ -305,4 +347,29 @@ private string GetSelectedOptionText(string optionText, string enteredFreeText) return optionText; } + + private static string GetFormattedQuestionType(QuestionType questionType) + { + switch (questionType) + { + case QuestionType.MultipleOption: + return "multiple choice"; + case QuestionType.SingleOption: + return "single choice"; + case QuestionType.MultipleOptionWithText: + return "multiple choice with text"; + case QuestionType.SingleOptionWithText: + return "single choice with text"; + default: + return "unknown"; + } + } + + private static string FormatOption(string text, bool isFreeText, bool isFlagged) + { + string isFreeTextFlag = isFreeText ? "$text" : string.Empty; + string isFlaggedFlag = isFlagged ? "$flagged" : string.Empty; + + return $"{text}{isFreeTextFlag}{isFlaggedFlag}"; + } } diff --git a/src/api/VoteMonitor.Api.Form/Models/FormDetailsModel.cs b/src/api/VoteMonitor.Api.Form/Models/FormDetailsModel.cs index a8c01447..c4fb151a 100644 --- a/src/api/VoteMonitor.Api.Form/Models/FormDetailsModel.cs +++ b/src/api/VoteMonitor.Api.Form/Models/FormDetailsModel.cs @@ -13,7 +13,7 @@ public class FormDetailsModel [JsonPropertyName("description")] public string Description { get; set; } - [JsonPropertyName("version")] + [JsonPropertyName("currentVersion")] public int CurrentVersion { get; set; } // quick and dirty fix to sync iOS and Android diff --git a/src/api/VoteMonitor.Api/Extensions/HealthChecks/S3HealthChecksExtensions.cs b/src/api/VoteMonitor.Api/Extensions/HealthChecks/S3HealthChecksExtensions.cs index 4750fe53..b55d68bd 100644 --- a/src/api/VoteMonitor.Api/Extensions/HealthChecks/S3HealthChecksExtensions.cs +++ b/src/api/VoteMonitor.Api/Extensions/HealthChecks/S3HealthChecksExtensions.cs @@ -42,4 +42,4 @@ public async Task CheckHealthAsync(HealthCheckContext context return new HealthCheckResult(context.Registration.FailureStatus, exception: ex); } } -} \ No newline at end of file +} diff --git a/src/api/VoteMonitor.Api/Extensions/LoggingConfiguration.cs b/src/api/VoteMonitor.Api/Extensions/LoggingConfiguration.cs index ec5456d4..91871e20 100644 --- a/src/api/VoteMonitor.Api/Extensions/LoggingConfiguration.cs +++ b/src/api/VoteMonitor.Api/Extensions/LoggingConfiguration.cs @@ -2,7 +2,6 @@ using Serilog.Core; using Serilog.Events; using Serilog.Exceptions; -using Serilog.Formatting.Compact; using Serilog.Sinks.Grafana.Loki; namespace VoteMonitor.Api.Extensions; @@ -34,7 +33,7 @@ public static void AddLoggingConfiguration(this IHostBuilder host, IConfiguratio .Enrich.WithProperty("Application", env.ApplicationName) .Enrich.WithExceptionDetails() .WriteTo.Console() - .WriteTo.GrafanaLoki(configuration["LokiConfig:Uri"], propertiesAsLabels: new[] { "Environment", "Application" }); + .WriteTo.GrafanaLoki(configuration["LokiConfig:Uri"], credentials: lokiCredentials, propertiesAsLabels: new[] { "Environment", "Application" }); Log.Logger = logger.CreateLogger(); diff --git a/src/api/VoteMonitor.Api/appsettings.json b/src/api/VoteMonitor.Api/appsettings.json index 10fa48f7..b0a876e1 100644 --- a/src/api/VoteMonitor.Api/appsettings.json +++ b/src/api/VoteMonitor.Api/appsettings.json @@ -8,7 +8,7 @@ } }, "LokiConfig": { - "Uri": "", + "Uri": "http://localhost:3100", "User": "", "Password": "" }, diff --git a/src/docker-compose.yml b/src/docker-compose.yml index e526f2f6..ac4860c4 100644 --- a/src/docker-compose.yml +++ b/src/docker-compose.yml @@ -70,3 +70,21 @@ services: ports: - '6379:6379' command: redis-server --save 20 1 --loglevel warning + + loki: + image: grafana/loki:master + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + networks: + - loki + + grafana: + image: grafana/grafana:master + ports: + - "3000:3000" + networks: + - loki + +networks: + loki: \ No newline at end of file diff --git a/src/grafana-docker-compose.yml b/src/grafana-docker-compose.yml deleted file mode 100644 index 0dab779c..00000000 --- a/src/grafana-docker-compose.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '3.4' - -services: - loki: - image: grafana/loki:master - ports: - - "3100:3100" - command: -config.file=/etc/loki/local-config.yaml - networks: - - loki - - grafana: - image: grafana/grafana:master - ports: - - "3000:3000" - networks: - - loki - -networks: - loki: \ No newline at end of file diff --git a/terraform/secrets.tf b/terraform/secrets.tf index 352647a5..867a8133 100644 --- a/terraform/secrets.tf +++ b/terraform/secrets.tf @@ -69,3 +69,17 @@ resource "aws_secretsmanager_secret_version" "firebase_serverkey" { secret_id = aws_secretsmanager_secret.firebase_serverkey.id secret_string = var.firebase_serverkey } + +resource "aws_secretsmanager_secret" "loki" { + name = "${local.namespace}-loki-${random_string.secrets_suffix.result}" +} + +resource "aws_secretsmanager_secret_version" "loki" { + secret_id = aws_secretsmanager_secret.loki.id + + secret_string = jsonencode({ + "uri" = var.loki_uri + "user" = var.loki_user + "password" = var.loki_password + }) +} diff --git a/terraform/service_api.tf b/terraform/service_api.tf index 5abeed69..833b6cb5 100644 --- a/terraform/service_api.tf +++ b/terraform/service_api.tf @@ -66,7 +66,7 @@ module "ecs_api" { }, { name = "MobileSecurityOptions__InvalidCredentialsErrorMessage" - value = "{ \"error\": \"A aparut o eroare la logarea in aplicatie. Va rugam sa verificati ca ati introdus corect numarul de telefon si codul de acces, iar daca eroarea persista va rugam contactati serviciul de suport la numarul 07......\" }" + value = var.invalid_credentials_error_message }, { name = "MobileSecurityOptions__LockDevice" @@ -143,6 +143,18 @@ module "ecs_api" { name = "HashOptions__Salt" valueFrom = aws_secretsmanager_secret.hash_salt.arn }, + { + name = "LokiConfig__Uri" + valueFrom = "${aws_secretsmanager_secret.loki.arn}:uri::" + }, + { + name = "LokiConfig__User" + valueFrom = "${aws_secretsmanager_secret.loki.arn}:user::" + }, + { + name = "LokiConfig__Password" + valueFrom = "${aws_secretsmanager_secret.loki.arn}:password::" + }, ] @@ -151,6 +163,7 @@ module "ecs_api" { aws_secretsmanager_secret.jwt_signing_key.arn, aws_secretsmanager_secret.hash_salt.arn, aws_secretsmanager_secret.rds.arn, + aws_secretsmanager_secret.loki.arn, ] } diff --git a/terraform/variables.tf b/terraform/variables.tf index 7a1e6696..3e134055 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -58,3 +58,27 @@ variable "docker_tag" { description = "Docker image tag" type = string } + +variable "loki_uri" { + description = "Loki uri" + type = string + default = null +} + +variable "loki_user" { + description = "Loki user" + type = string + default = null +} + +variable "loki_password" { + description = "Loki password" + type = string + default = null +} + +variable "invalid_credentials_error_message" { + description = "Error message to show when invalid credentials are provided" + type = string + default = "{ \"error\": \"A apărut o eroare la logarea în aplicație. Vă rugăm să verificați că ați introdus corect numărul de telefon și codul de acces, iar dacă eroarea persistă, vă rugăm contactați serviciul de suport la numarul 07......\" }" +}