Merge pull request #407 from praekeltfoundation/add-language-column-i…

…n-import-export Add language column in import export
praekeltfoundation · Jan 30, 2025 · dbde0a0 · dbde0a0
2 parents 1d87d47 + 116241e
commit dbde0a0
Show file tree

Hide file tree

Showing 20 changed files with 134 additions and 85 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## Unreleased
 ### Added
 - Answer responses for Forms
+- Language_code field on imports and exports
+### Removed
+- Locale field on exports 
 -->
 
 ## v1.4.0 - 2024-12-18

diff --git a/home/export_content_pages.py b/home/export_content_pages.py
@@ -72,14 +72,14 @@ class ExportRow:
     tags: str = ""
     quick_replies: str = ""
     triggers: str = ""
-    locale: str = ""
     next_prompt: str = ""
     buttons: str = ""
     image_link: str = ""
     doc_link: str = ""
     media_link: str = ""
     related_pages: str = ""
     footer: str = ""
+    language_code: str = ""
 
     @classmethod
     def headings(cls) -> list[str]:
@@ -203,8 +203,6 @@ def _export_content_page(self, page: ContentPage, structure: str) -> None:
         Export a ContentPage.
 
         FIXME:
-         * The locale should be a language code rather than a language name to
-           make importing less messy.
          * We should use the parent slug (which is expected to be unique per
            locale (probably?)) instead of the parent title.
         """
@@ -228,8 +226,8 @@ def _export_content_page(self, page: ContentPage, structure: str) -> None:
             tags=self._comma_sep_qs(page.tags.all()),
             quick_replies=self._comma_sep_qs(page.quick_replies.all()),
             triggers=self._comma_sep_qs(page.triggers.all()),
-            locale=str(page.locale),
             related_pages=self._comma_sep_qs(self._related_pages(page)),
+            language_code=page.locale.language_code,
         )
         self.rows.append(row)
         message_bodies = list(
@@ -260,8 +258,6 @@ def _export_cpi(self, page: ContentPageIndex, structure: str) -> None:
         Export a ContentPageIndex.
 
         FIXME:
-         * The locale should be a language code rather than a language name to
-           make importing less messy.
          * We should use the parent slug (which is expected to be unique per
            locale (probably?)) instead of the parent title.
         """
@@ -272,7 +268,7 @@ def _export_cpi(self, page: ContentPageIndex, structure: str) -> None:
             parent=self._parent_title(page),
             web_title=page.title,
             translation_tag=str(page.translation_key),
-            locale=str(page.locale),
+            language_code=page.locale.language_code,
         )
         self.rows.append(row)
 
@@ -353,14 +349,14 @@ def _set_xlsx_styles(wb: Workbook, sheet: Worksheet) -> None:
         "tags": 118,
         "quick_replies": 118,
         "triggers": 118,
-        "locale": 118,
         "next_prompt": 118,
         "buttons": 118,
         "image_link": 118,
         "doc_link": 118,
         "media_link": 118,
         "related": 118,
         "footer": 118,
+        "language_code": 118,
     }
 
     for index, column_width in enumerate(column_widths_in_pts.values(), 2):

diff --git a/home/import_content_pages.py b/home/import_content_pages.py
@@ -11,9 +11,7 @@
 from django.core.exceptions import ObjectDoesNotExist, ValidationError  # type: ignore
 from taggit.models import Tag  # type: ignore
 from treebeard.exceptions import NodeAlreadySaved  # type: ignore
-from wagtail.blocks import (  # type: ignore
-    StructValue,  # type: ignore
-)
+from wagtail.blocks import StructValue  # type: ignore
 from wagtail.coreutils import get_content_languages  # type: ignore
 from wagtail.models import Locale, Page  # type: ignore
 from wagtail.models.sites import Site  # type: ignore
@@ -99,14 +97,14 @@ def process_rows(self, rows: list["ContentRow"]) -> None:
         for i, row in enumerate(rows, start=2):
             try:
                 if row.is_page_index:
-                    if self.locale and row.locale != self.locale.get_display_name():
+                    prev_locale = self._get_locale_from_row(row)
+                    if self.locale and self.locale != prev_locale:
                         # This page index isn't for the locale we're importing, so skip it.
                         continue
                     self.create_content_page_index_from_row(row)
-                    prev_locale = self.locale_from_display_name(row.locale)
                 elif row.is_content_page:
                     self.create_shadow_content_page_from_row(row, i)
-                    prev_locale = self.locale_from_display_name(row.locale)
+                    prev_locale = self._get_locale_from_row(row)
                 elif row.is_variation_message:
                     self.add_variation_to_shadow_content_page_from_row(row, prev_locale)
                 else:
@@ -117,6 +115,15 @@ def process_rows(self, rows: list["ContentRow"]) -> None:
                 e.locale = row.locale
                 raise e
 
+    def _get_locale_from_row(self, row: "ContentRow") -> Locale:
+        if row.language_code:
+            try:
+                return Locale.objects.get(language_code=row.language_code)
+            except Locale.DoesNotExist:
+                raise ImportException(f"Language not found: {row.language_code}")
+        else:
+            return self.locale_from_display_name(row.locale)
+
     def save_pages(self) -> None:
         for i, page in enumerate(self.shadow_pages.values()):
             if self.locale and page.locale != self.locale:
@@ -242,7 +249,7 @@ def default_locale(self) -> Locale:
         return site.root_page.locale
 
     def create_content_page_index_from_row(self, row: "ContentRow") -> None:
-        locale = self.locale_from_display_name(row.locale)
+        locale = self._get_locale_from_row(row)
         try:
             index = ContentPageIndex.objects.get(slug=row.slug, locale=locale)
         except ContentPageIndex.DoesNotExist:
@@ -252,7 +259,6 @@ def create_content_page_index_from_row(self, row: "ContentRow") -> None:
         # but optional for the default locale.
         if row.translation_tag or locale != self.default_locale():
             index.translation_key = row.translation_tag
-        locale = self.locale_from_display_name(row.locale)
         try:
             with contextlib.suppress(NodeAlreadySaved):
                 self.home_page(locale).add_child(instance=index)
@@ -268,7 +274,7 @@ def create_content_page_index_from_row(self, row: "ContentRow") -> None:
     def create_shadow_content_page_from_row(
         self, row: "ContentRow", row_num: int
     ) -> None:
-        locale = self.locale_from_display_name(row.locale)
+        locale = self._get_locale_from_row(row)
         page = ShadowContentPage(
             row_num=row_num,
             slug=row.slug,
@@ -738,6 +744,7 @@ class ContentRow:
     media_link: str = ""
     related_pages: list[str] = field(default_factory=list)
     footer: str = ""
+    language_code: str = ""
 
     @classmethod
     def from_flat(cls, row: dict[str, str], row_num: int) -> "ContentRow":

diff --git a/home/tests/import-export-data/content2.csv b/home/tests/import-export-data/content2.csv
@@ -1,5 +1,5 @@
-structure,message,page_id,Slug,parent,web_title,web_subtitle,web_body,whatsapp_title,whatsapp_body,whatsapp_template_name,whatsapp_template_category,example_values,variation_title,variation_body,list_title,list_items,sms_title,sms_body,ussd_title,ussd_body,messenger_title,messenger_body,viber_title,viber_body,translation_tag,tags,quick_replies,triggers,locale,next_prompt,buttons,image_link,doc_link,media_link,related_pages,footer
-Menu 1,0,4,main-menu,,Main Menu,,,,,,,,,,,,,,,,,,,,a0b85075-d01b-46bf-8997-8591e87ba171,,,,English,,,,,,,
+structure,message,page_id,Slug,parent,web_title,web_subtitle,web_body,whatsapp_title,whatsapp_body,whatsapp_template_name,whatsapp_template_category,example_values,variation_title,variation_body,list_title,list_items,sms_title,sms_body,ussd_title,ussd_body,messenger_title,messenger_body,viber_title,viber_body,translation_tag,tags,quick_replies,triggers,next_prompt,buttons,image_link,doc_link,media_link,related_pages,footer,language_code
+Menu 1,0,4,main-menu,,Main Menu,,,,,,,,,,,,,,,,,,,,a0b85075-d01b-46bf-8997-8591e87ba171,,,,,,,,,,,en
 Sub 1.1,1,5,main-menu-first-time-user,Main Menu,main menu first time user,,,main menu first time user,"*Welcome to HealthAlert* 🌍
 
 This is a messaging service created by the _*World Health Organization*_ *(WHO)* that provides information on COVID-19 as well as emergency resources for disease outbreaks, natural, and man-made disasters.
@@ -12,12 +12,12 @@ This is a messaging service created by the World Health Organization (WHO) that
 
 You can return to this main menu at any time by replying 🏠
 
-Choose what you'd like to know more about by tapping a button below ⬇️",,,5892bccd-8025-419d-9a8e-a6a37b755dbf,menu,"Self-help🌬️, Settings⚙️, Health Info🏥",Main menu🏠,English,,[],,,,,
+Choose what you'd like to know more about by tapping a button below ⬇️",,,5892bccd-8025-419d-9a8e-a6a37b755dbf,menu,"Self-help🌬️, Settings⚙️, Health Info🏥",Main menu🏠,,[],,,,,,en
 Sub 1.1.1,1,6,health-info,main menu first time user,health info,,,health info,"*Health information* 🏥
 
 Get information and advice from WHO by tapping on a button below ⬇️",,UTILITY,[],,,,[],Health Info SMS Title,*Health Info SMS Body*,Health Info USSD Title,*Health Info USSD Body*,health info,"*Health information* 🏥
 
-Get information and advice from WHO by tapping on a button below ⬇️",,,c9d6309e-173f-4c1d-bbaf-440b1fd4415f,health_info,,,English,,[],,,,,
+Get information and advice from WHO by tapping on a button below ⬇️",,,c9d6309e-173f-4c1d-bbaf-440b1fd4415f,health_info,,,,[],,,,,,en
 Sub 1.1.2,1,7,self-help,main menu first time user,self-help,,,self-help,"*Self-help programs* 🌬️
 
 Reply with a number to take part in a *free* self-help program created by WHO.
@@ -32,4 +32,4 @@ Reply with a number to take part in a *free* self-help program created by WHO.
 1. Quit tobacco 🚭
 _Stop smoking with the help of a guided, 42-day program._
 2. Manage your stress 🧘🏽‍♀️
-_Learn how to cope with stress and improve your wellbeing._",,,3e5d77f7-4d34-430d-aad7-d9ca01f79732,self_help,,,English,,[],,,,,
+_Learn how to cope with stress and improve your wellbeing._",,,3e5d77f7-4d34-430d-aad7-d9ca01f79732,self_help,,,,[],,,,,,en
diff --git a/home/tests/import-export-data/contentpage_index_required_fields.csv b/home/tests/import-export-data/contentpage_index_required_fields.csv
@@ -1,2 +1,2 @@
-message,slug,parent,web_title,locale
-0,main-menu,,Main Menu,English
+message,slug,parent,web_title,language_code
+0,main-menu,,Main Menu,en
diff --git a/home/tests/import-export-data/contentpage_required_fields.csv b/home/tests/import-export-data/contentpage_required_fields.csv
@@ -1,4 +1,4 @@
-message,slug,parent,web_title,locale
-0,main_menu,,Main Menu,English
-1,first_time_user,Main Menu,main menu first time user,English
-1,health_info,main menu first time user,health info,English
+message,slug,parent,web_title,language_code
+0,main_menu,,Main Menu,en
+1,first_time_user,Main Menu,main menu first time user,en
+1,health_info,main menu first time user,health info,en
diff --git a/home/tests/import-export-data/exported_content_20230911-variations-linked-page.csv b/home/tests/import-export-data/exported_content_20230911-variations-linked-page.csv
@@ -1,12 +1,12 @@
-structure,message,page_id,slug,parent,web_title,web_subtitle,web_body,whatsapp_title,whatsapp_body,whatsapp_template_name,variation_title,variation_body,list_title,list_items,sms_title,sms_body,ussd_title,ussd_body,messenger_title,messenger_body,viber_title,viber_body,translation_tag,tags,quick_replies,triggers,locale,next_prompt,buttons,image_link,doc_link,media_link,related_pages,example_values,footer
-Menu 1,0,5,appointment-reminders,,Appointment reminders,,,,,,,,,,,,,,,,,,8cff04aa-cf08-4120-88b4-e2269b7d5d80,,,,English,,,,,,,,
-Menu 2,0,6,stage-based-messages,,Stage-based messages,,,,,,,,,,,,,,,,,,5f2d9e63-f047-41ab-921a-c5ca7c04d643,,,,English,,,,,,,,
-Menu 3,0,7,health-info-messages,,Health info messages,,,,,,,,,,,,,,,,,,3db069a4-7112-4e66-a9a2-f35c6c18055a,,,,English,,,,,,,,
-Menu 4,0,17,whatsapp-template-testing,,whatsapp template testing,,,,,,,,,,,,,,,,,,5f7221f4-146a-48c2-b2e3-c8491aaead9d,,,,English,,,,,,,,
-Menu 5,0,164,import-export,,Import Export,,,,,,,,,,,,,,,,,,497bdc1f-43fc-4925-80a1-e68cb942faa4,,,,English,,,,,,,,
-Sub 5.1,1,165,cp-import-export,Import Export,CP-Import/export,,,WA import export data,Message 1 contains an image,,,,,[],,,,,,,,,8ac50daf-de21-4d05-b697-6d983b7ed3d5,"Tag2, Tag1",Quick reply1,"Trigger2, Trigger1",English,,[],/admin/images/usage/4/,,,ma_qa_temp,,
+structure,message,page_id,slug,parent,web_title,web_subtitle,web_body,whatsapp_title,whatsapp_body,whatsapp_template_name,variation_title,variation_body,list_title,list_items,sms_title,sms_body,ussd_title,ussd_body,messenger_title,messenger_body,viber_title,viber_body,translation_tag,tags,quick_replies,triggers,next_prompt,buttons,image_link,doc_link,media_link,related_pages,example_values,footer,language_code
+Menu 1,0,5,appointment-reminders,,Appointment reminders,,,,,,,,,,,,,,,,,,8cff04aa-cf08-4120-88b4-e2269b7d5d80,,,,,,,,,,,,en
+Menu 2,0,6,stage-based-messages,,Stage-based messages,,,,,,,,,,,,,,,,,,5f2d9e63-f047-41ab-921a-c5ca7c04d643,,,,,,,,,,,,en
+Menu 3,0,7,health-info-messages,,Health info messages,,,,,,,,,,,,,,,,,,3db069a4-7112-4e66-a9a2-f35c6c18055a,,,,,,,,,,,,en
+Menu 4,0,17,whatsapp-template-testing,,whatsapp template testing,,,,,,,,,,,,,,,,,,5f7221f4-146a-48c2-b2e3-c8491aaead9d,,,,,,,,,,,,en
+Menu 5,0,164,import-export,,Import Export,,,,,,,,,,,,,,,,,,497bdc1f-43fc-4925-80a1-e68cb942faa4,,,,,,,,,,,,en
+Sub 5.1,1,165,cp-import-export,Import Export,CP-Import/export,,,WA import export data,Message 1 contains an image,,,,,[],,,,,,,,,8ac50daf-de21-4d05-b697-6d983b7ed3d5,"Tag2, Tag1",Quick reply1,"Trigger2, Trigger1",,[],/admin/images/usage/4/,,,ma_qa_temp,,,en
 ,1,165,cp-import-export,,,,,,,,gender: male,Variation message one for Gender Male,,,,,,,,,,,,,,,,,,,,,,,
-,2,165,cp-import-export,,,,,,"Message2 has a document attached",,,,,[],,,,,,,,,,,,,,,[],,/admin/documents/usage/1/,,,,
+,2,165,cp-import-export,,,,,,Message2 has a document attached,,,,,[],,,,,,,,,,,,,,[],,/admin/documents/usage/1/,,,,,
 ,2,165,cp-import-export,,,,,,,,gender: empty,Variation message one for Gender Rather not say,,,,,,,,,,,,,,,,,,,,,,,
-,3,165,cp-import-export,,,,,,Message 3 with no variation but has an audio clip,,,,,[],,,,,,,,,,,,,,,[],,,/admin/media/usage/1/,,,
-Sub 5.2,1,166,ma_qa_temp,Import Export,MA QA Temp,,,,,,,,,,,,,,,,,,e9793b5f-f8c7-46c5-8a5e-bd9b8f00fee9,,,,English,,,,,,,,
+,3,165,cp-import-export,,,,,,Message 3 with no variation but has an audio clip,,,,,[],,,,,,,,,,,,,,[],,,/admin/media/usage/1/,,,,
+Sub 5.2,1,166,ma_qa_temp,Import Export,MA QA Temp,,,,,,,,,,,,,,,,,,e9793b5f-f8c7-46c5-8a5e-bd9b8f00fee9,,,,,,,,,,,,en
diff --git a/home/tests/import-export-data/language_code_import.csv b/home/tests/import-export-data/language_code_import.csv
@@ -0,0 +1,15 @@
+structure,message,page_id,Slug,parent,web_title,web_subtitle,web_body,whatsapp_title,whatsapp_body,whatsapp_template_name,whatsapp_template_category,example_values,variation_title,variation_body,list_title,list_items,sms_title,sms_body,ussd_title,ussd_body,messenger_title,messenger_body,viber_title,viber_body,translation_tag,tags,quick_replies,triggers,locale,next_prompt,buttons,image_link,doc_link,media_link,related_pages,footer,language_code
+Menu 1,0,4,main-menu,,Main Menu,,,,,,,,,,,,,,,,,,,,a0b85075-d01b-46bf-8997-8591e87ba171,,,,English,,,,,,,,
+Sub 1.1,1,5,main-menu-first-time-user,Main Menu,main menu first time user,,,main menu first time user,"*Welcome to HealthAlert* 🌍
+
+This is a messaging service created by the _*World Health Organization*_ *(WHO)* that provides information on COVID-19 as well as emergency resources for disease outbreaks, natural, and man-made disasters.
+
+_You can return to this main menu at any time by replying *0*_ 🏠
+
+Choose what you'd like to know more about by tapping a button below ⬇️",,UTILITY,[],,,,[],,,,,main menu first time user,"Welcome to HealthAlert 🌍
+
+This is a messaging service created by the World Health Organization (WHO) that provides information on COVID-19 as well as emergency resources for disease outbreaks, natural, and man-made disasters.
+
+You can return to this main menu at any time by replying 🏠
+
+Choose what you'd like to know more about by tapping a button below ⬇️",,,5892bccd-8025-419d-9a8e-a6a37b755dbf,menu,"Self-help🌬️, Settings⚙️, Health Info🏥",Main menu🏠,English,,[],,,,,,
diff --git a/home/tests/import-export-data/language_code_import_output.csv b/home/tests/import-export-data/language_code_import_output.csv
@@ -0,0 +1,15 @@
+structure,message,page_id,Slug,parent,web_title,web_subtitle,web_body,whatsapp_title,whatsapp_body,whatsapp_template_name,whatsapp_template_category,example_values,variation_title,variation_body,list_title,list_items,sms_title,sms_body,ussd_title,ussd_body,messenger_title,messenger_body,viber_title,viber_body,translation_tag,tags,quick_replies,triggers,next_prompt,buttons,image_link,doc_link,media_link,related_pages,footer,language_code
+Menu 1,0,4,main-menu,,Main Menu,,,,,,,,,,,,,,,,,,,,a0b85075-d01b-46bf-8997-8591e87ba171,,,,,,,,,,,en
+Sub 1.1,1,5,main-menu-first-time-user,Main Menu,main menu first time user,,,main menu first time user,"*Welcome to HealthAlert* 🌍
+
+This is a messaging service created by the _*World Health Organization*_ *(WHO)* that provides information on COVID-19 as well as emergency resources for disease outbreaks, natural, and man-made disasters.
+
+_You can return to this main menu at any time by replying *0*_ 🏠
+
+Choose what you'd like to know more about by tapping a button below ⬇️",,UTILITY,[],,,,[],,,,,main menu first time user,"Welcome to HealthAlert 🌍
+
+This is a messaging service created by the World Health Organization (WHO) that provides information on COVID-19 as well as emergency resources for disease outbreaks, natural, and man-made disasters.
+
+You can return to this main menu at any time by replying 🏠
+
+Choose what you'd like to know more about by tapping a button below ⬇️",,,5892bccd-8025-419d-9a8e-a6a37b755dbf,menu,"Self-help🌬️, Settings⚙️, Health Info🏥",Main menu🏠,,[],,,,,,en
diff --git a/home/tests/import-export-data/list_items.csv b/home/tests/import-export-data/list_items.csv
@@ -1,3 +1,3 @@
-structure,message,page_id,slug,parent,web_title,web_subtitle,web_body,whatsapp_title,whatsapp_body,whatsapp_template_name,whatsapp_template_category,variation_title,variation_body,list_title,list_items,sms_title,sms_body,ussd_title,ussd_body,messenger_title,messenger_body,viber_title,viber_body,translation_tag,tags,quick_replies,triggers,locale,next_prompt,buttons,image_link,doc_link,media_link,related_pages,footer
-Menu 1,0,600,main-menu,,Main Menu,,,,,,,,,,,,,,,,,,,35a7f12c-7373-42a6-9ca6-a1caedea5822,,,,English,,,,,,,
-Sub 1.1,1,601,ha-menu,Main Menu,HealthAlert menu,,,HealthAlert menu,*Welcome to HealthAlert* WA,,UTILITY,,,List Title,"'Item 1','Item 2'",,,,,HealthAlert menu,Welcome to HealthAlert M,HealthAlert menu,Welcome to HealthAlert V,5ab08854-228b-4f83-ae54-ec2dc6ebaf69,,,,English,,[],,,,,
+structure,message,page_id,slug,parent,web_title,web_subtitle,web_body,whatsapp_title,whatsapp_body,whatsapp_template_name,whatsapp_template_category,variation_title,variation_body,list_title,list_items,sms_title,sms_body,ussd_title,ussd_body,messenger_title,messenger_body,viber_title,viber_body,translation_tag,tags,quick_replies,triggers,next_prompt,buttons,image_link,doc_link,media_link,related_pages,footer,language_code
+Menu 1,0,600,main-menu,,Main Menu,,,,,,,,,,,,,,,,,,,35a7f12c-7373-42a6-9ca6-a1caedea5822,,,,,,,,,,,en
+Sub 1.1,1,601,ha-menu,Main Menu,HealthAlert menu,,,HealthAlert menu,*Welcome to HealthAlert* WA,,UTILITY,,,List Title,"'Item 1','Item 2'",,,,,HealthAlert menu,Welcome to HealthAlert M,HealthAlert menu,Welcome to HealthAlert V,5ab08854-228b-4f83-ae54-ec2dc6ebaf69,,,,,[],,,,,,en