Merge branch 'master' into word-card-tests

Backend.Tests/Otel/LocationProviderTests.cs

@@ -15,7 +15,7 @@ namespace Backend.Tests.Otel
 {
     public class LocationProviderTests
     {
-        private readonly IPAddress TestIpAddress = new(new byte[] { 100, 0, 0, 0 });
+        private readonly IPAddress TestIpAddress = new([100, 0, 0, 0]);
         private IHttpContextAccessor _contextAccessor = null!;
         private IMemoryCache _memoryCache = null!;
         private Mock<HttpMessageHandler> _handlerMock = null!;
@@ -26,15 +26,9 @@ public class LocationProviderTests
         public void Setup()
         {
             // Set up HttpContextAccessor with mocked IP
-            _contextAccessor = new HttpContextAccessor();
-            var httpContext = new DefaultHttpContext()
-            {
-                Connection =
-                {
-                    RemoteIpAddress = TestIpAddress
-                }
-            };
-            _contextAccessor.HttpContext = httpContext;
+            var httpContext = new DefaultHttpContext();
+            httpContext.Request.Headers["X-Original-Forwarded-For"] = TestIpAddress.ToString();
+            _contextAccessor = new HttpContextAccessor { HttpContext = httpContext };
 
             // Set up MemoryCache
             var services = new ServiceCollection();

Backend.Tests/Otel/OtelKernelTests.cs

@@ -3,7 +3,6 @@
 using System.Diagnostics;
 using System.Linq;
 using Backend.Tests.Mocks;
-using BackendFramework.Otel;
 using Microsoft.AspNetCore.Http;
 using NUnit.Framework;
 using static BackendFramework.Otel.OtelKernel;
@@ -12,9 +11,6 @@ namespace Backend.Tests.Otel
 {
     public class OtelKernelTests : IDisposable
     {
-        private const string FrontendSessionIdKey = "sessionId";
-        private const string OtelSessionIdKey = "sessionId";
-        private const string OtelSessionBaggageKey = "sessionBaggage";
         private LocationEnricher _locationEnricher = null!;
 
         public void Dispose()
@@ -32,41 +28,63 @@ protected virtual void Dispose(bool disposing)
         }
 
         [Test]
-        public void BuildersSetSessionBaggageFromHeader()
+        public void BuildersSetBaggageFromHeaderAllAnalytics()
         {
             // Arrange
             var httpContext = new DefaultHttpContext();
-            httpContext.Request.Headers[FrontendSessionIdKey] = "123";
+            httpContext.Request.Headers[AnalyticsOnHeader] = "true";
+            httpContext.Request.Headers[SessionIdHeader] = "123";
             var activity = new Activity("testActivity").Start();
 
             // Act
+            TrackConsent(activity, httpContext.Request);
             TrackSession(activity, httpContext.Request);
 
             // Assert
-            Assert.That(activity.Baggage.Any(_ => _.Key == OtelSessionBaggageKey));
+            Assert.That(activity.Baggage.Any(_ => _.Key == ConsentBaggage));
+            Assert.That(activity.Baggage.Any(_ => _.Key == SessionIdBaggage));
         }
 
         [Test]
-        public void OnEndSetsSessionTagFromBaggage()
+        public void BuildersSetBaggageFromHeaderNecessaryAnalytics()
         {
             // Arrange
+            var httpContext = new DefaultHttpContext();
+            httpContext.Request.Headers[AnalyticsOnHeader] = "false";
             var activity = new Activity("testActivity").Start();
-            activity.SetBaggage(OtelSessionBaggageKey, "test session id");
 
             // Act
-            _locationEnricher.OnEnd(activity);
+            TrackConsent(activity, httpContext.Request);
+            TrackSession(activity, httpContext.Request);
 
             // Assert
-            Assert.That(activity.Tags.Any(_ => _.Key == OtelSessionIdKey));
+            Assert.That(activity.Baggage.Any(_ => _.Key == ConsentBaggage));
+            Assert.That(!activity.Baggage.Any(_ => _.Key == SessionIdBaggage));
         }
 
+        [Test]
+        public void OnEndSetsTagsFromBaggage()
+        {
+            // Arrange
+            var activity = new Activity("testActivity").Start();
+            activity.SetBaggage(ConsentBaggage, "true");
+            activity.SetBaggage(SessionIdBaggage, "test session id");
+
+            // Act
+            _locationEnricher.OnEnd(activity);
+
+            // Assert
+            Assert.That(activity.Tags.Any(_ => _.Key == ConsentTag));
+            Assert.That(activity.Tags.Any(_ => _.Key == SessionIdTag));
+        }
 
         [Test]
-        public void OnEndSetsLocationTags()
+        public void OnEndSetsLocationTagsAllAnalytics()
         {
             // Arrange
             _locationEnricher = new LocationEnricher(new LocationProviderMock());
             var activity = new Activity("testActivity").Start();
+            activity.SetBaggage(ConsentBaggage, "true");
 
             // Act
             _locationEnricher.OnEnd(activity);
@@ -81,19 +99,21 @@ public void OnEndSetsLocationTags()
             Assert.That(activity.Tags, Is.SupersetOf(testLocation));
         }
 
-        public void OnEndRedactsIp()
+        [Test]
+        public void OnEndSetsLocationTagsNecessaryAnalytics()
         {
             // Arrange
             _locationEnricher = new LocationEnricher(new LocationProviderMock());
             var activity = new Activity("testActivity").Start();
-            activity.SetTag("url.full", $"{LocationProvider.locationGetterUri}100.0.0.0");
+            activity.SetBaggage(ConsentBaggage, "false");
 
             // Act
             _locationEnricher.OnEnd(activity);
 
             // Assert
-            Assert.That(activity.Tags.Any(_ => _.Key == "url.full" && _.Value == ""));
-            Assert.That(activity.Tags.Any(_ => _.Key == "url.redacted.ip" && _.Value == LocationProvider.locationGetterUri));
+            Assert.That(!activity.Tags.Any(_ => _.Key == "country"));
+            Assert.That(!activity.Tags.Any(_ => _.Key == "regionName"));
+            Assert.That(!activity.Tags.Any(_ => _.Key == "city"));
         }
     }
 }

Backend/Models/User.cs

@@ -66,6 +66,19 @@ public class User
         [BsonElement("username")]
         public string Username { get; set; }
 
+        /// <summary>
+        /// Is false if user rejects analytics, true otherwise.
+        /// User can update consent anytime.
+        /// </summary>
+        [BsonElement("analyticsOn")]
+        public bool AnalyticsOn { get; set; }
+
+        /// <summary>
+        /// Is set permanently to true once user first accepts or rejects analytics upon login.
+        /// </summary>
+        [BsonElement("answeredConsent")]
+        public bool AnsweredConsent { get; set; }
+
         [BsonElement("uiLang")]
         public string UILang { get; set; }
 
@@ -97,6 +110,8 @@ public User()
             Agreement = false;
             Password = "";
             Username = "";
+            AnalyticsOn = true;
+            AnsweredConsent = false;
             UILang = "";
             GlossSuggestion = OffOnSetting.On;
             Token = "";
@@ -119,6 +134,8 @@ public User Clone()
                 Agreement = Agreement,
                 Password = Password,
                 Username = Username,
+                AnalyticsOn = AnalyticsOn,
+                AnsweredConsent = AnsweredConsent,
                 UILang = UILang,
                 GlossSuggestion = GlossSuggestion,
                 Token = Token,
@@ -141,6 +158,8 @@ public bool ContentEquals(User other)
                 other.Agreement == Agreement &&
                 other.Password.Equals(Password, StringComparison.Ordinal) &&
                 other.Username.Equals(Username, StringComparison.Ordinal) &&
+                other.AnalyticsOn == AnalyticsOn &&
+                other.AnsweredConsent == AnsweredConsent &&
                 other.UILang.Equals(UILang, StringComparison.Ordinal) &&
                 other.GlossSuggestion.Equals(GlossSuggestion) &&
                 other.Token.Equals(Token, StringComparison.Ordinal) &&
@@ -178,6 +197,8 @@ public override int GetHashCode()
             hash.Add(Agreement);
             hash.Add(Password);
             hash.Add(Username);
+            hash.Add(AnalyticsOn);
+            hash.Add(AnsweredConsent);
             hash.Add(UILang);
             hash.Add(GlossSuggestion);
             hash.Add(Token);

Backend/Otel/LocationProvider.cs

@@ -27,24 +27,18 @@ public LocationProvider(IHttpContextAccessor contextAccessor, IMemoryCache memor
             // because OtelKernel calls the function for each activity
             if (_contextAccessor.HttpContext is { } context)
             {
-                var ipAddress = context.GetServerVariable("HTTP_X_FORWARDED_FOR") ?? context.Connection.RemoteIpAddress?.ToString();
-                var ipAddressWithoutPort = ipAddress?.Split(':')[0];
+                var ipAddress = context.Request.Headers["X-Original-Forwarded-For"].ToString();
+                if (string.IsNullOrEmpty(ipAddress))
+                {
+                    return null;
+                }
 
                 return await _memoryCache.GetOrCreateAsync(
-                    "location_" + ipAddressWithoutPort,
+                    "location_" + ipAddress,
                     async (cacheEntry) =>
                     {
                         cacheEntry.SlidingExpiration = TimeSpan.FromHours(1);
-                        try
-                        {
-                            return await GetLocationFromIp(ipAddressWithoutPort);
-                        }
-                        catch
-                        {
-                            // TODO consider what to have in catch
-                            Console.WriteLine("Attempted to get location but exception");
-                            throw;
-                        }
+                        return await GetLocationFromIp(ipAddress);
                     }
                 );
             }

Backend/Otel/OtelKernel.cs

@@ -17,6 +17,12 @@ namespace BackendFramework.Otel
     public static class OtelKernel
     {
         public const string SourceName = "Backend-Otel";
+        public const string AnalyticsOnHeader = "analyticsOn";
+        public const string SessionIdHeader = "sessionId";
+        public const string ConsentBaggage = "consentBaggage";
+        public const string SessionIdBaggage = "sessionIdBaggage";
+        public const string ConsentTag = "otelConsent";
+        public const string SessionIdTag = "sessionId";
 
         public static void AddOpenTelemetryInstrumentation(this IServiceCollection services)
         {
@@ -33,12 +39,19 @@ public static void AddOpenTelemetryInstrumentation(this IServiceCollection servi
             );
         }
 
+        internal static void TrackConsent(Activity activity, HttpRequest request)
+        {
+            request.Headers.TryGetValue(AnalyticsOnHeader, out var consentString);
+            var consent = bool.Parse(consentString!);
+            activity.SetBaggage(ConsentBaggage, consent.ToString());
+        }
+
         internal static void TrackSession(Activity activity, HttpRequest request)
         {
-            var sessionId = request.Headers.TryGetValue("sessionId", out var values) ? values.FirstOrDefault() : null;
+            var sessionId = request.Headers.TryGetValue(SessionIdHeader, out var values) ? values.FirstOrDefault() : null;
             if (sessionId is not null)
             {
-                activity.SetBaggage("sessionBaggage", sessionId);
+                activity.SetBaggage(SessionIdBaggage, sessionId);
             }
         }
 
@@ -67,6 +80,7 @@ private static void AspNetCoreBuilder(AspNetCoreTraceInstrumentationOptions opti
             options.EnrichWithHttpRequest = (activity, request) =>
             {
                 GetContentLengthAspNet(activity, request.Headers, "inbound.http.request.body.size");
+                TrackConsent(activity, request);
                 TrackSession(activity, request);
             };
             options.EnrichWithHttpResponse = (activity, response) =>
@@ -98,22 +112,20 @@ internal class LocationEnricher(ILocationProvider locationProvider) : BaseProces
         {
             public override async void OnEnd(Activity data)
             {
-                var uriPath = (string?)data.GetTagItem("url.full");
-                var locationUri = LocationProvider.locationGetterUri;
-                if (uriPath is null || !uriPath.Contains(locationUri))
+                var consentString = data.GetBaggageItem(ConsentBaggage);
+                data.AddTag(ConsentTag, consentString);
+                if (bool.TryParse(consentString, out bool consent) && consent)
                 {
-                    var location = await locationProvider.GetLocation();
-                    data?.AddTag("country", location?.Country);
-                    data?.AddTag("regionName", location?.RegionName);
-                    data?.AddTag("city", location?.City);
-                }
-                data?.SetTag("sessionId", data?.GetBaggageItem("sessionBaggage"));
-                if (uriPath is not null && uriPath.Contains(locationUri))
-                {
-                    // When getting location externally, url.full includes site URI and user IP. 
-                    // In such cases, only add url without IP information to traces.
-                    data?.SetTag("url.full", "");
-                    data?.SetTag("url.redacted.ip", LocationProvider.locationGetterUri);
+                    var uriPath = (string?)data.GetTagItem("url.full");
+                    var locationUri = LocationProvider.locationGetterUri;
+                    if (uriPath is null || !uriPath.Contains(locationUri))
+                    {
+                        var location = await locationProvider.GetLocation();
+                        data.AddTag("country", location?.Country);
+                        data.AddTag("regionName", location?.RegionName);
+                        data.AddTag("city", location?.City);
+                    }
+                    data.AddTag(SessionIdTag, data.GetBaggageItem(SessionIdBaggage));
                 }
             }
         }

Backend/Repositories/UserRepository.cs

@@ -196,6 +196,8 @@ public async Task<ResultOfUpdate> Update(string userId, User user, bool updateIs
                 .Set(x => x.ProjectRoles, user.ProjectRoles)
                 .Set(x => x.Agreement, user.Agreement)
                 .Set(x => x.Username, user.Username)
+                .Set(x => x.AnalyticsOn, user.AnalyticsOn)
+                .Set(x => x.AnsweredConsent, user.AnsweredConsent)
                 .Set(x => x.UILang, user.UILang)
                 .Set(x => x.GlossSuggestion, user.GlossSuggestion);
 

docs/tutorial_subtitles/_in_progress_transcripts_/data_entry_1_basics.eng.txt → docs/tutorial_subtitles/data_entry_1_basics.eng/data_entry_1_basics.eng.txt

@@ -26,11 +26,11 @@ Click on the play button to listen.
 Hold the shift-key and click to delete it.
 A dialog box will appear to confirm whether you want to delete the recording.
 (If you are using a touch-screen, you can tap on the play button to play, or press and hold the button to bring up a menu.)
-When you are done entering words, click the Exit button to return to the semantic domain tree.
+When you are done entering words, click the “Exit” button to return to the semantic domain tree.
 (Don’t worry—the words you entered are already saved even if you close the window without clicking the exit button.)
 If we select the same domain to enter more words, see how the words we previously entered in this domain are listed in a panel on the side?
 If you are working in a narrow web browser window, the panel of previously entered words will not automatically appear.
-You can bring it up by pressing the sideways carat icon at the bottom of the data entry box.
+You can bring it up by clicking the sideways carat icon at the bottom of the data entry box.
 Let’s enter more words!
 If you want to delete one of the words you added, click on the delete icon at the end of its row.
 Warning: this will permanently remove the word and all its content!
@@ -39,5 +39,5 @@ It will reset the vernacular field, the gloss field, the note, and the audio rec
 That covers how to gather words with Data Entry in The Combine!
 When gathering words by semantic domain, often the same word will be added to multiple domains, resulting in lots of duplicate words.
 The Combine has ways to help avoid or manage that issue.
-In the next video, we will look at entering multiple words with the same vernacular form.
+In the next Data Entry video, we will look at entering multiple words with the same vernacular form.
 Have a wonderful day!

docs/tutorial_subtitles/data_entry_1_basics.eng/data_entry_1_basics.spa.txt

@@ -0,0 +1,43 @@
+The Combine está diseñado para Rapid Word Collection, un método de recopilación de palabras por dominio semántico.
+En este video veremos cómo hacer Entrada de datos en The Combine para recoger palabras.
+Vamos a thecombine.app e iniciemos sesión.
+Al hacer clic en un proyecto, aparece el árbol de dominios semánticos.
+La selección del dominio es el primer paso en la Entrada de datos.
+Si está haciendo una tarea de proyecto diferente (por ejemplo, en la limpieza de datos o en la configuración del proyecto) puede volver aquí haciendo clic en el botón “Entrada de datos” (“Data Entry”) en la barra superior.
+Hay otro vídeo tutorial sobre cómo navegar por el árbol de dominios semánticos o cambiar su idioma.
+Para este vídeo, vamos a seleccionar el dominio “2: Persona” (“2: Person”) y empezar a recopilar palabras!
+Hay 4 cosas que pueden incluirse en una palabra nueva.
+En primer lugar, la forma vernácula de la palabra en el idioma vernáculo del proyecto.
+En segundo lugar, una glosa de la palabra en el idioma de análisis principal del proyecto.
+(Puede cambiar el idioma de análisis de un proyecto en los ajustes del proyecto.)
+En tercer lugar, una nota sobre la palabra.
+En cuarto lugar, grabaciones de audio de la pronunciación de la palabra.
+Tras añadir el contenido de la nueva palabra, pulse la tecla Enter.
+¿Ve cómo aparece en la tabla la palabra que acabamos de introducir?
+Si pasamos el cursor por encima del icono de nota de esa palabra, aparece el texto de la nota.
+¡Añadamos otra palabra!
+Pues la forma vernácula es necesaria para una nueva entrada, pero la glosa, la nota y el audio son opcionales.
+En cualquier momento puede modificar las palabras introducidas.
+Añadamos una glosa y una nota a la segunda entrada.
+Cambiemos la forma vernácula y eliminemos la nota de la primera entrada.
+¿Qué podemos hacer con las grabaciones de audio?
+Si pasamos el cursor por encima del botón de reproducción (el icono del triángulo verde), aparece un texto que describe las opciones disponibles.
+Haga clic en el botón de reproducción para escuchar.
+Mantenga pulsada la tecla Mayús y haga clic para eliminarla.
+Aparecerá un cuadro de diálogo para confirmar si desea eliminar la grabación.
+(Si usa una pantalla táctil, puede tocar el botón de reproducción para reproducir o mantener pulsado el botón para abrir un menú.)
+Cuando haya terminado de introducir palabras, haga clic en el botón “Salir” (“Exit”) para volver al árbol de dominios semánticos.
+(No se preocupe—las palabras introducidas ya se guardan aunque cierre la ventana sin pulsar el botón de salida.)
+Si seleccionamos el mismo dominio para introducir más palabras, ¿ve cómo las palabras que hemos introducido previamente en este dominio aparecen en un panel al lado?
+Si trabaja en una ventana estrecha del navegador, el panel de palabras introducidas previamente no aparecerá automáticamente.
+Puede activarlo haciendo clic en el icono del caret lateral en la parte inferior del cuadro de entrada de datos.
+¡Introduzcamos más palabras!
+Si quiere eliminar una de las palabras introducidas, haga clic en el icono de eliminación al final de su fila.
+Advertencia: ¡esto eliminará permanentemente la palabra y todo su contenido!
+Si quiere empezar de nuevo en una palabra que está agregando, haga clic en el icono de eliminación aquí en la fila inferior.
+Reiniciará el campo vernáculo, el campo de glosas, la nota y las grabaciones de audio para la nueva palabra.
+¡Eso cubre cómo recopilar palabras con la Entrada de datos en The Combine!
+Cuando se recopilan palabras por dominio semántico, a menudo se añade la misma palabra a múltiples dominios, resultando en muchas palabras duplicadas.
+The Combine tiene maneras de ayudar a evitar o manejar ese problema.
+En el próximo vídeo de Entrada de datos, veremos cómo introducir varias palabras con la misma forma vernácula.
+¡Que tenga un día maravilloso!