From d48330140355ecdfa57ec620e78adab542360a85 Mon Sep 17 00:00:00 2001 From: Ion Dormenco Date: Fri, 13 Oct 2023 11:45:56 +0300 Subject: [PATCH 1/4] Loki config/ export of forms options/ bug fixes * fix bug with loki credentials not sent * export form options * add tf secrets for loki config * fix currentVersion for iOS --- requests/requests.http | 1 + .../FileGenerator/ExcelFile.cs | 7 -- .../Handlers/GetExcelDbCommandHandler.cs | 111 ++++++++++++++---- .../Models/FormDetailsModel.cs | 2 +- .../HealthChecks/S3HealthChecksExtensions.cs | 2 +- .../Extensions/LoggingConfiguration.cs | 3 +- src/api/VoteMonitor.Api/appsettings.json | 2 +- src/docker-compose.yml | 18 +++ src/grafana-docker-compose.yml | 20 ---- terraform/secrets.tf | 30 +++++ terraform/service_api.tf | 15 +++ terraform/variables.tf | 15 +++ 12 files changed, 172 insertions(+), 54 deletions(-) delete mode 100644 src/grafana-docker-compose.yml diff --git a/requests/requests.http b/requests/requests.http index 8a6a3cd5..c67f197a 100644 --- a/requests/requests.http +++ b/requests/requests.http @@ -321,6 +321,7 @@ Content-Type: text/csv ### # @name exportAll GET {{url}}/api/v2/export/all +Authorization: Bearer {{adminJwtToken.response.body.$.access_token}} ### ### diff --git a/src/api/VoteMonitor.Api.DataExport/FileGenerator/ExcelFile.cs b/src/api/VoteMonitor.Api.DataExport/FileGenerator/ExcelFile.cs index 5efcfd3d..a173b15c 100644 --- a/src/api/VoteMonitor.Api.DataExport/FileGenerator/ExcelFile.cs +++ b/src/api/VoteMonitor.Api.DataExport/FileGenerator/ExcelFile.cs @@ -1,8 +1,5 @@ -using Microsoft.EntityFrameworkCore.Metadata.Internal; -using NPOI.SS.Formula.Functions; using NPOI.SS.UserModel; using NPOI.XSSF.UserModel; -using System.Collections; using System.ComponentModel; using System.Data; @@ -62,8 +59,6 @@ public ExcelFile WithSheet(string sheetName, DataTable dataTable) var cell = header.CreateCell(i); cell.SetCellValue(headers[i]); cell.CellStyle = _headerStyle; - // It's heavy, it slows down your Excel if you have large data - sheet.AutoSizeColumn(i); } #endregion @@ -95,8 +90,6 @@ public ExcelFile WithSheet(string sheetName, List exportData) where T:clas var cell = header.CreateCell(i); cell.SetCellValue(headers[i]); cell.CellStyle = _headerStyle; - // It's heavy, it slows down your Excel if you have large data - sheet.AutoSizeColumn(i); } #endregion diff --git a/src/api/VoteMonitor.Api.DataExport/Handlers/GetExcelDbCommandHandler.cs b/src/api/VoteMonitor.Api.DataExport/Handlers/GetExcelDbCommandHandler.cs index e9073f21..a859eb27 100644 --- a/src/api/VoteMonitor.Api.DataExport/Handlers/GetExcelDbCommandHandler.cs +++ b/src/api/VoteMonitor.Api.DataExport/Handlers/GetExcelDbCommandHandler.cs @@ -1,7 +1,6 @@ using MediatR; using Microsoft.EntityFrameworkCore; using System.Data; -using System.Text; using VoteMonitor.Api.DataExport.FileGenerator; using VoteMonitor.Api.DataExport.Queries; using VoteMonitor.Entities; @@ -20,11 +19,13 @@ public GetExcelDbCommandHandler(VoteMonitorContext context) public async Task Handle(GetExcelDbCommand request, CancellationToken cancellationToken) { var ngos = await _context.Ngos + .AsNoTracking() .Select(ngo => new { ngo.Id, ngo.Name, ngo.Organizer, }) .OrderBy(x => x.Id) .ToListAsync(cancellationToken: cancellationToken); var observers = await _context.Observers + .AsNoTracking() .Select(observer => new { observer.Id, @@ -39,17 +40,20 @@ public async Task Handle(GetExcelDbCommand request, CancellationToken ca .ToListAsync(cancellationToken: cancellationToken); var provinces = await _context.Provinces + .AsNoTracking() .OrderBy(x => x.Order) .Select(province => new { province.Id, province.Code, province.Name }) .ToListAsync(cancellationToken: cancellationToken); var counties = await _context.Counties + .AsNoTracking() .Include(x => x.Province) .OrderBy(x => x.Order) .Select(county => new { county.Id, county.Code, ProvinceCode = county.Province.Code, county.Name, county.Diaspora }) .ToListAsync(cancellationToken: cancellationToken); var municipalities = await _context.Municipalities + .AsNoTracking() .Include(x => x.County) .OrderBy(x => x.Order) .Select(municipality => new { municipality.Id, municipality.Code, CountyCode = municipality.County.Code, municipality.Name }) @@ -67,9 +71,34 @@ public async Task Handle(GetExcelDbCommand request, CancellationToken ca pollingStation.Number, pollingStation.Address }) - .ToListAsync(cancellationToken: cancellationToken); + var aggregatedForms = await GetForms(cancellationToken); + var filledInForms = await GetFilledInForms(cancellationToken); + var notes = await GetNotesForExport(cancellationToken); + + var excelBuilder = ExcelFile + .New() + .WithSheet("ngos", ngos) + .WithSheet("observers", observers) + .WithSheet("provinces", provinces) + .WithSheet("counties", counties) + .WithSheet("municipalities", municipalities) + .WithSheet("polling-stations", pollingStations) + .WithSheet("forms", aggregatedForms); + + filledInForms.ForEach(f => + { + excelBuilder = excelBuilder.WithSheet(f.Code, f.Data); + }); + + excelBuilder = excelBuilder.WithSheet("notes", notes); + + return excelBuilder.Write(); + } + + private async Task GetForms(CancellationToken cancellationToken) + { var forms = await _context.Forms .AsNoTracking() .Include(f => f.FormSections) @@ -80,17 +109,18 @@ public async Task Handle(GetExcelDbCommand request, CancellationToken ca .OrderBy(f => f.Order) .ToListAsync(cancellationToken); - var aggregatedForms = forms.SelectMany(form => form.FormSections + var questions = forms.SelectMany(form => form.FormSections .OrderBy(x => x.OrderNumber) .SelectMany(formSection => formSection.Questions.OrderBy(x => x.OrderNumber) .Select(question => new { FormCode = form.Code, + FormOrderNumber = form.Order, FormSectionCode = formSection.Code, - FormSectionDescription = formSection.Description, QuestionCode = question.Code, - QuestionQuestionType = question.QuestionType, + QuestionType = question.QuestionType, QuestionText = question.Text, + QuestionOrderNumber = question.OrderNumber, Options = question.OptionsToQuestions .OrderBy(x => x.Option.OrderNumber) .Select(optionToQuestion => new @@ -101,30 +131,41 @@ public async Task Handle(GetExcelDbCommand request, CancellationToken ca }) .ToList() }))) + .OrderBy(q => q.FormOrderNumber) + .ThenBy(q => q.QuestionOrderNumber) .ToList(); - var filledInForms = await GetFilledInForms(cancellationToken); + DataTable dataTable = new DataTable(); - var notes = await GetNotesForExport(cancellationToken); + dataTable.Columns.Add("FormCode", typeof(string)); + dataTable.Columns.Add("FormSectionCode", typeof(string)); + dataTable.Columns.Add("QuestionCode", typeof(string)); + dataTable.Columns.Add("Question", typeof(string)); + dataTable.Columns.Add("Type", typeof(string)); + var maxNumberOfOptions = questions.Select(x => x.Options.Count).DefaultIfEmpty(0).Max(); - var excelBuilder = ExcelFile - .New() - .WithSheet("ngos", ngos) - .WithSheet("observers", observers) - .WithSheet("provinces", provinces) - .WithSheet("counties", counties) - .WithSheet("municipalities", municipalities) - .WithSheet("polling-stations", pollingStations) - .WithSheet("forms", aggregatedForms); + for (int i = 1; i <= maxNumberOfOptions; i++) + { + dataTable.Columns.Add($"Options-{i}", typeof(string)); + } - filledInForms.ForEach(f => + foreach (var question in questions) { - excelBuilder = excelBuilder.WithSheet(f.Code, f.Data); - }); + object?[] rowValues = new List + { + question.FormCode, + question.FormSectionCode, + question.QuestionCode, + question.QuestionText, + GetFormattedQuestionType(question.QuestionType), + } + .Union(question.Options.Select(x => FormatOption(x.Text, x.IsFreeText, x.IsFlagged))) + .ToArray(); - excelBuilder = excelBuilder.WithSheet("notes", notes); + dataTable.Rows.Add(rowValues); + } - return excelBuilder.Write(); + return dataTable; } private async Task GetNotesForExport(CancellationToken cancellationToken) @@ -201,6 +242,7 @@ private static string GetNoteLocation(string pollingStation, string formCode, st private async Task> GetFilledInForms(CancellationToken cancellationToken) { var answers = await _context.Answers + .AsNoTracking() .Include(a => a.Observer) .Include(a => a.PollingStation) .ThenInclude(x => x.Municipality) @@ -296,7 +338,7 @@ private static string GetNoteLocation(string pollingStation, string formCode, st return result; } - private string GetSelectedOptionText(string optionText, string enteredFreeText) + private static string GetSelectedOptionText(string optionText, string enteredFreeText) { if (!string.IsNullOrWhiteSpace(enteredFreeText)) { @@ -305,4 +347,29 @@ private string GetSelectedOptionText(string optionText, string enteredFreeText) return optionText; } + + private static string GetFormattedQuestionType(QuestionType questionType) + { + switch (questionType) + { + case QuestionType.MultipleOption: + return "multiple choice"; + case QuestionType.SingleOption: + return "single choice"; + case QuestionType.MultipleOptionWithText: + return "multiple choice with text"; + case QuestionType.SingleOptionWithText: + return "single choice with text"; + default: + return "unknown"; + } + } + + private static string FormatOption(string text, bool isFreeText, bool isFlagged) + { + string isFreeTextFlag = isFreeText ? "$text" : string.Empty; + string isFlaggedFlag = isFlagged ? "$flagged" : string.Empty; + + return $"{text}{isFreeTextFlag}{isFlaggedFlag}"; + } } diff --git a/src/api/VoteMonitor.Api.Form/Models/FormDetailsModel.cs b/src/api/VoteMonitor.Api.Form/Models/FormDetailsModel.cs index a8c01447..c4fb151a 100644 --- a/src/api/VoteMonitor.Api.Form/Models/FormDetailsModel.cs +++ b/src/api/VoteMonitor.Api.Form/Models/FormDetailsModel.cs @@ -13,7 +13,7 @@ public class FormDetailsModel [JsonPropertyName("description")] public string Description { get; set; } - [JsonPropertyName("version")] + [JsonPropertyName("currentVersion")] public int CurrentVersion { get; set; } // quick and dirty fix to sync iOS and Android diff --git a/src/api/VoteMonitor.Api/Extensions/HealthChecks/S3HealthChecksExtensions.cs b/src/api/VoteMonitor.Api/Extensions/HealthChecks/S3HealthChecksExtensions.cs index 4750fe53..b55d68bd 100644 --- a/src/api/VoteMonitor.Api/Extensions/HealthChecks/S3HealthChecksExtensions.cs +++ b/src/api/VoteMonitor.Api/Extensions/HealthChecks/S3HealthChecksExtensions.cs @@ -42,4 +42,4 @@ public async Task CheckHealthAsync(HealthCheckContext context return new HealthCheckResult(context.Registration.FailureStatus, exception: ex); } } -} \ No newline at end of file +} diff --git a/src/api/VoteMonitor.Api/Extensions/LoggingConfiguration.cs b/src/api/VoteMonitor.Api/Extensions/LoggingConfiguration.cs index ec5456d4..91871e20 100644 --- a/src/api/VoteMonitor.Api/Extensions/LoggingConfiguration.cs +++ b/src/api/VoteMonitor.Api/Extensions/LoggingConfiguration.cs @@ -2,7 +2,6 @@ using Serilog.Core; using Serilog.Events; using Serilog.Exceptions; -using Serilog.Formatting.Compact; using Serilog.Sinks.Grafana.Loki; namespace VoteMonitor.Api.Extensions; @@ -34,7 +33,7 @@ public static void AddLoggingConfiguration(this IHostBuilder host, IConfiguratio .Enrich.WithProperty("Application", env.ApplicationName) .Enrich.WithExceptionDetails() .WriteTo.Console() - .WriteTo.GrafanaLoki(configuration["LokiConfig:Uri"], propertiesAsLabels: new[] { "Environment", "Application" }); + .WriteTo.GrafanaLoki(configuration["LokiConfig:Uri"], credentials: lokiCredentials, propertiesAsLabels: new[] { "Environment", "Application" }); Log.Logger = logger.CreateLogger(); diff --git a/src/api/VoteMonitor.Api/appsettings.json b/src/api/VoteMonitor.Api/appsettings.json index 10fa48f7..b0a876e1 100644 --- a/src/api/VoteMonitor.Api/appsettings.json +++ b/src/api/VoteMonitor.Api/appsettings.json @@ -8,7 +8,7 @@ } }, "LokiConfig": { - "Uri": "", + "Uri": "http://localhost:3100", "User": "", "Password": "" }, diff --git a/src/docker-compose.yml b/src/docker-compose.yml index e526f2f6..ac4860c4 100644 --- a/src/docker-compose.yml +++ b/src/docker-compose.yml @@ -70,3 +70,21 @@ services: ports: - '6379:6379' command: redis-server --save 20 1 --loglevel warning + + loki: + image: grafana/loki:master + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + networks: + - loki + + grafana: + image: grafana/grafana:master + ports: + - "3000:3000" + networks: + - loki + +networks: + loki: \ No newline at end of file diff --git a/src/grafana-docker-compose.yml b/src/grafana-docker-compose.yml deleted file mode 100644 index 0dab779c..00000000 --- a/src/grafana-docker-compose.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '3.4' - -services: - loki: - image: grafana/loki:master - ports: - - "3100:3100" - command: -config.file=/etc/loki/local-config.yaml - networks: - - loki - - grafana: - image: grafana/grafana:master - ports: - - "3000:3000" - networks: - - loki - -networks: - loki: \ No newline at end of file diff --git a/terraform/secrets.tf b/terraform/secrets.tf index 352647a5..822acc9a 100644 --- a/terraform/secrets.tf +++ b/terraform/secrets.tf @@ -69,3 +69,33 @@ resource "aws_secretsmanager_secret_version" "firebase_serverkey" { secret_id = aws_secretsmanager_secret.firebase_serverkey.id secret_string = var.firebase_serverkey } + +# LOKI URI +resource "aws_secretsmanager_secret" "loki_uri" { + name = "${local.namespace}-loki_uri-${random_string.secrets_suffix.result}" +} + +resource "aws_secretsmanager_secret_version" "loki_uri" { + secret_id = aws_secretsmanager_secret.loki_uri.id + secret_string = var.loki_uri +} + +# LOKI USER +resource "aws_secretsmanager_secret" "loki_user" { + name = "${local.namespace}-loki_user-${random_string.secrets_suffix.result}" +} + +resource "aws_secretsmanager_secret_version" "loki_user" { + secret_id = aws_secretsmanager_secret.loki_user.id + secret_string = var.loki_user +} + +# LOKI PASSWORD +resource "aws_secretsmanager_secret" "loki_password" { + name = "${local.namespace}-loki_password-${random_string.secrets_suffix.result}" +} + +resource "aws_secretsmanager_secret_version" "loki_password" { + secret_id = aws_secretsmanager_secret.loki_password.id + secret_string = var.loki_password +} \ No newline at end of file diff --git a/terraform/service_api.tf b/terraform/service_api.tf index 5abeed69..3f9289c3 100644 --- a/terraform/service_api.tf +++ b/terraform/service_api.tf @@ -143,6 +143,18 @@ module "ecs_api" { name = "HashOptions__Salt" valueFrom = aws_secretsmanager_secret.hash_salt.arn }, + { + name = "LokiConfig__Uri" + valueFrom = aws_secretsmanager_secret.loki_uri.arn + }, + { + name = "LokiConfig__User" + valueFrom = aws_secretsmanager_secret.loki_user.arn + }, + { + name = "LokiConfig__Password" + valueFrom = aws_secretsmanager_secret.loki_password.arn + }, ] @@ -151,6 +163,9 @@ module "ecs_api" { aws_secretsmanager_secret.jwt_signing_key.arn, aws_secretsmanager_secret.hash_salt.arn, aws_secretsmanager_secret.rds.arn, + aws_secretsmanager_secret.loki_uri.arn, + aws_secretsmanager_secret.loki_user.arn, + aws_secretsmanager_secret.loki_password.arn, ] } diff --git a/terraform/variables.tf b/terraform/variables.tf index 7a1e6696..cae34a72 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -58,3 +58,18 @@ variable "docker_tag" { description = "Docker image tag" type = string } + +variable "loki_uri" { + description = "Loki uri" + type = string +} + +variable "loki_user" { + description = "Loki user" + type = string +} + +variable "loki_password" { + description = "Loki password" + type = string +} \ No newline at end of file From 22a3dcafc6a58328e3b14ae981bf75846574e847 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20Ioni=C8=9B=C4=83?= Date: Fri, 13 Oct 2023 10:04:58 +0100 Subject: [PATCH 2/4] consolidate loki credentials --- terraform/secrets.tf | 34 +++++++++------------------------- terraform/service_api.tf | 10 ++++------ 2 files changed, 13 insertions(+), 31 deletions(-) diff --git a/terraform/secrets.tf b/terraform/secrets.tf index 822acc9a..867a8133 100644 --- a/terraform/secrets.tf +++ b/terraform/secrets.tf @@ -70,32 +70,16 @@ resource "aws_secretsmanager_secret_version" "firebase_serverkey" { secret_string = var.firebase_serverkey } -# LOKI URI -resource "aws_secretsmanager_secret" "loki_uri" { - name = "${local.namespace}-loki_uri-${random_string.secrets_suffix.result}" +resource "aws_secretsmanager_secret" "loki" { + name = "${local.namespace}-loki-${random_string.secrets_suffix.result}" } -resource "aws_secretsmanager_secret_version" "loki_uri" { - secret_id = aws_secretsmanager_secret.loki_uri.id - secret_string = var.loki_uri -} - -# LOKI USER -resource "aws_secretsmanager_secret" "loki_user" { - name = "${local.namespace}-loki_user-${random_string.secrets_suffix.result}" -} +resource "aws_secretsmanager_secret_version" "loki" { + secret_id = aws_secretsmanager_secret.loki.id -resource "aws_secretsmanager_secret_version" "loki_user" { - secret_id = aws_secretsmanager_secret.loki_user.id - secret_string = var.loki_user + secret_string = jsonencode({ + "uri" = var.loki_uri + "user" = var.loki_user + "password" = var.loki_password + }) } - -# LOKI PASSWORD -resource "aws_secretsmanager_secret" "loki_password" { - name = "${local.namespace}-loki_password-${random_string.secrets_suffix.result}" -} - -resource "aws_secretsmanager_secret_version" "loki_password" { - secret_id = aws_secretsmanager_secret.loki_password.id - secret_string = var.loki_password -} \ No newline at end of file diff --git a/terraform/service_api.tf b/terraform/service_api.tf index 3f9289c3..7849f226 100644 --- a/terraform/service_api.tf +++ b/terraform/service_api.tf @@ -145,15 +145,15 @@ module "ecs_api" { }, { name = "LokiConfig__Uri" - valueFrom = aws_secretsmanager_secret.loki_uri.arn + valueFrom = "${aws_secretsmanager_secret.loki.arn}:uri::" }, { name = "LokiConfig__User" - valueFrom = aws_secretsmanager_secret.loki_user.arn + valueFrom = "${aws_secretsmanager_secret.loki.arn}:user::" }, { name = "LokiConfig__Password" - valueFrom = aws_secretsmanager_secret.loki_password.arn + valueFrom = "${aws_secretsmanager_secret.loki.arn}:password::" }, ] @@ -163,9 +163,7 @@ module "ecs_api" { aws_secretsmanager_secret.jwt_signing_key.arn, aws_secretsmanager_secret.hash_salt.arn, aws_secretsmanager_secret.rds.arn, - aws_secretsmanager_secret.loki_uri.arn, - aws_secretsmanager_secret.loki_user.arn, - aws_secretsmanager_secret.loki_password.arn, + aws_secretsmanager_secret.loki.arn, ] } From 0567eb062410da9e95838a70c0afee6537223f93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20Ioni=C8=9B=C4=83?= Date: Fri, 13 Oct 2023 10:06:42 +0100 Subject: [PATCH 3/4] configurable invalid credentials error message --- terraform/service_api.tf | 2 +- terraform/variables.tf | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/terraform/service_api.tf b/terraform/service_api.tf index 7849f226..833b6cb5 100644 --- a/terraform/service_api.tf +++ b/terraform/service_api.tf @@ -66,7 +66,7 @@ module "ecs_api" { }, { name = "MobileSecurityOptions__InvalidCredentialsErrorMessage" - value = "{ \"error\": \"A aparut o eroare la logarea in aplicatie. Va rugam sa verificati ca ati introdus corect numarul de telefon si codul de acces, iar daca eroarea persista va rugam contactati serviciul de suport la numarul 07......\" }" + value = var.invalid_credentials_error_message }, { name = "MobileSecurityOptions__LockDevice" diff --git a/terraform/variables.tf b/terraform/variables.tf index cae34a72..58939e34 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -72,4 +72,10 @@ variable "loki_user" { variable "loki_password" { description = "Loki password" type = string -} \ No newline at end of file +} + +variable "invalid_credentials_error_message" { + description = "Error message to show when invalid credentials are provided" + type = string + default = "{ \"error\": \"A apărut o eroare la logarea în aplicație. Vă rugăm să verificați că ați introdus corect numărul de telefon și codul de acces, iar dacă eroarea persistă, vă rugăm contactați serviciul de suport la numarul 07......\" }" +} From ffeb140d42ca5d63c6cfdb137887ac4fe8b1cad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20Ioni=C8=9B=C4=83?= Date: Fri, 13 Oct 2023 10:09:05 +0100 Subject: [PATCH 4/4] set loki defaults --- terraform/variables.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform/variables.tf b/terraform/variables.tf index 58939e34..3e134055 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -62,16 +62,19 @@ variable "docker_tag" { variable "loki_uri" { description = "Loki uri" type = string + default = null } variable "loki_user" { description = "Loki user" type = string + default = null } variable "loki_password" { description = "Loki password" type = string + default = null } variable "invalid_credentials_error_message" {