1.8 Translate to English.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from mlds import data_loader\n",
    "import warnings, re, os\n",
    "warnings.filterwarnings('ignore')\n",
    "from transformers import pipeline\n",
    "from typing import List\n",
    "\n",
    "\n",
    "translater = pipeline(\"translation\", model=\"facebook/nllb-200-3.3B\", device=\"cuda:0\")\n",
    "manager = data_loader.SlotDataManager(data_folder=\"data/output\")\n",
    "mapper = data_loader.NLLB_MAPPER\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Translating 3200 texts from amh_Ethi to eng_Latn\n",
      "[{'translation_text': 'which account has the most money'}]\n",
      "Translating 3200 texts from ewe_Latn to eng_Latn\n",
      "[{'translation_text': 'If I were a goat, how would I cry out?'}]\n",
      "Translating 3200 texts from hau_Latn to eng_Latn\n",
      "[{'translation_text': 'Is there enough Naira in my zenith bank account to buy the latest newspaper?'}]\n",
      "Translating 3200 texts from ibo_Latn to eng_Latn\n",
      "[{'translation_text': 'How much did the Access Bank say I had left?'}]\n",
      "Translating 3200 texts from kin_Latn to eng_Latn\n",
      "[{'translation_text': 'i want to set up an alarm on the computer'}]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Translating 3200 texts from lin_Latn to eng_Latn\n",
      "[{'translation_text': 'How much money is in my bank account?'}]\n",
      "Translating 3200 texts from lug_Latn to eng_Latn\n",
      "[{'translation_text': 'How much money is left in my Cairo account'}]\n",
      "Translating 3200 texts from som_Latn to eng_Latn\n",
      "[{'translation_text': \"I can't even tell you how much money there is.\"}]\n",
      "Translating 3200 texts from sna_Latn to eng_Latn\n",
      "[{'translation_text': 'How much money is left in my pockets at Cabs?'}]\n",
      "Translating 3200 texts from sot_Latn to eng_Latn\n",
      "[{'translation_text': 'How much money do I have in my Discovery account'}]\n",
      "Translating 3200 texts from swh_Latn to eng_Latn\n",
      "[{'translation_text': 'How many shillings do I have in my NBC Savings bank account?'}]\n",
      "Translating 3200 texts from twi_Latn to eng_Latn\n",
      "[{'translation_text': 'Let me see the amount of money in my Citibank Senegal account.'}]\n",
      "Translating 3200 texts from wol_Latn to eng_Latn\n",
      "[{'translation_text': 'I reserved a seat at the restaurant Chez L Homme Tranquille in Saly at 9pm.'}]\n",
      "Translating 3200 texts from xho_Latn to eng_Latn\n",
      "[{'translation_text': 'Do I still have money in my purse?'}]\n",
      "Translating 3200 texts from yor_Latn to eng_Latn\n",
      "[{'translation_text': 'Please check my Dorm wallet and tell me how much I have in it.'}]\n",
      "Translating 3200 texts from zul_Latn to eng_Latn\n",
      "[{'translation_text': 'where can I find the bus to Mlazi?'}]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>domain</th>\n",
       "      <th>intent</th>\n",
       "      <th>text</th>\n",
       "      <th>spans</th>\n",
       "      <th>logical_form</th>\n",
       "      <th>xtreme-up</th>\n",
       "      <th>source_language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>travel</td>\n",
       "      <td>translate</td>\n",
       "      <td>where can I find the bus to Mlazi?</td>\n",
       "      <td>44:50:SL:PLACE_NAME</td>\n",
       "      <td>[IN:translate [SL:PLACE_NAME eMlazi] ]</td>\n",
       "      <td>PLACE_NAME: eMlazi</td>\n",
       "      <td>zul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>travel</td>\n",
       "      <td>translate</td>\n",
       "      <td>a plant traveling in a subway?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[IN:translate ]</td>\n",
       "      <td></td>\n",
       "      <td>zul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>travel</td>\n",
       "      <td>translate</td>\n",
       "      <td>greet Clara in Tswana.</td>\n",
       "      <td>11:16:SL:PERSONAL_NAME,22:28:SL:LANGUAGE_NAME</td>\n",
       "      <td>[IN:translate [SL:PERSONAL_NAME Clara] [SL:LAN...</td>\n",
       "      <td>PERSONAL_NAME: Clara $$ LANGUAGE_NAME: Tswana</td>\n",
       "      <td>zul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>travel</td>\n",
       "      <td>translate</td>\n",
       "      <td>what is the fastest train in south africa?</td>\n",
       "      <td>30:46:SL:COUNTRY</td>\n",
       "      <td>[IN:translate [SL:COUNTRY Ningizimu Afrika] ]</td>\n",
       "      <td>COUNTRY: Ningizimu Afrika</td>\n",
       "      <td>zul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>travel</td>\n",
       "      <td>translate</td>\n",
       "      <td>what is the main language spoken in Johannesburg?</td>\n",
       "      <td>34:39:SL:CITY_OR_PROVINCE</td>\n",
       "      <td>[IN:translate [SL:CITY_OR_PROVINCE eGoli] ]</td>\n",
       "      <td>CITY_OR_PROVINCE: eGoli</td>\n",
       "      <td>zul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3195</th>\n",
       "      <td>kitchen_and_dining</td>\n",
       "      <td>recipe</td>\n",
       "      <td>what's the delicious recipe for spicy bunny chow?</td>\n",
       "      <td>26:44:SL:DISH_OR_FOOD</td>\n",
       "      <td>[IN:recipe [SL:DISH_OR_FOOD bunny chow ebabayo] ]</td>\n",
       "      <td>DISH_OR_FOOD: bunny chow ebabayo</td>\n",
       "      <td>zul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3196</th>\n",
       "      <td>kitchen_and_dining</td>\n",
       "      <td>recipe</td>\n",
       "      <td>see atchar recipes without mango</td>\n",
       "      <td>19:25:SL:DISH_OR_FOOD,35:41:SL:DISH_OR_FOOD</td>\n",
       "      <td>[IN:recipe [SL:DISH_OR_FOOD atchar] [SL:DISH_O...</td>\n",
       "      <td>DISH_OR_FOOD: atchar $$ DISH_OR_FOOD: umango</td>\n",
       "      <td>zul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3197</th>\n",
       "      <td>kitchen_and_dining</td>\n",
       "      <td>recipe</td>\n",
       "      <td>can you please look for me recipes for making ...</td>\n",
       "      <td>40:54:SL:DISH_OR_FOOD</td>\n",
       "      <td>[IN:recipe [SL:DISH_OR_FOOD mutton ebabayo] ]</td>\n",
       "      <td>DISH_OR_FOOD: mutton ebabayo</td>\n",
       "      <td>zul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3198</th>\n",
       "      <td>kitchen_and_dining</td>\n",
       "      <td>recipe</td>\n",
       "      <td>i need a recipe for atchar and chocolate</td>\n",
       "      <td>21:27:SL:DISH_OR_FOOD,30:39:SL:DISH_OR_FOOD</td>\n",
       "      <td>[IN:recipe [SL:DISH_OR_FOOD atchar] [SL:DISH_O...</td>\n",
       "      <td>DISH_OR_FOOD: atchar $$ DISH_OR_FOOD: chakalaka</td>\n",
       "      <td>zul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3199</th>\n",
       "      <td>kitchen_and_dining</td>\n",
       "      <td>recipe</td>\n",
       "      <td>I'll show you some vegan food ideas.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[IN:recipe ]</td>\n",
       "      <td></td>\n",
       "      <td>zul</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3200 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  domain     intent  \\\n",
       "0                 travel  translate   \n",
       "1                 travel  translate   \n",
       "2                 travel  translate   \n",
       "3                 travel  translate   \n",
       "4                 travel  translate   \n",
       "...                  ...        ...   \n",
       "3195  kitchen_and_dining     recipe   \n",
       "3196  kitchen_and_dining     recipe   \n",
       "3197  kitchen_and_dining     recipe   \n",
       "3198  kitchen_and_dining     recipe   \n",
       "3199  kitchen_and_dining     recipe   \n",
       "\n",
       "                                                   text  \\\n",
       "0                    where can I find the bus to Mlazi?   \n",
       "1                        a plant traveling in a subway?   \n",
       "2                                greet Clara in Tswana.   \n",
       "3            what is the fastest train in south africa?   \n",
       "4     what is the main language spoken in Johannesburg?   \n",
       "...                                                 ...   \n",
       "3195  what's the delicious recipe for spicy bunny chow?   \n",
       "3196                   see atchar recipes without mango   \n",
       "3197  can you please look for me recipes for making ...   \n",
       "3198           i need a recipe for atchar and chocolate   \n",
       "3199               I'll show you some vegan food ideas.   \n",
       "\n",
       "                                              spans  \\\n",
       "0                               44:50:SL:PLACE_NAME   \n",
       "1                                               NaN   \n",
       "2     11:16:SL:PERSONAL_NAME,22:28:SL:LANGUAGE_NAME   \n",
       "3                                  30:46:SL:COUNTRY   \n",
       "4                         34:39:SL:CITY_OR_PROVINCE   \n",
       "...                                             ...   \n",
       "3195                          26:44:SL:DISH_OR_FOOD   \n",
       "3196    19:25:SL:DISH_OR_FOOD,35:41:SL:DISH_OR_FOOD   \n",
       "3197                          40:54:SL:DISH_OR_FOOD   \n",
       "3198    21:27:SL:DISH_OR_FOOD,30:39:SL:DISH_OR_FOOD   \n",
       "3199                                            NaN   \n",
       "\n",
       "                                           logical_form  \\\n",
       "0                [IN:translate [SL:PLACE_NAME eMlazi] ]   \n",
       "1                                       [IN:translate ]   \n",
       "2     [IN:translate [SL:PERSONAL_NAME Clara] [SL:LAN...   \n",
       "3         [IN:translate [SL:COUNTRY Ningizimu Afrika] ]   \n",
       "4           [IN:translate [SL:CITY_OR_PROVINCE eGoli] ]   \n",
       "...                                                 ...   \n",
       "3195  [IN:recipe [SL:DISH_OR_FOOD bunny chow ebabayo] ]   \n",
       "3196  [IN:recipe [SL:DISH_OR_FOOD atchar] [SL:DISH_O...   \n",
       "3197      [IN:recipe [SL:DISH_OR_FOOD mutton ebabayo] ]   \n",
       "3198  [IN:recipe [SL:DISH_OR_FOOD atchar] [SL:DISH_O...   \n",
       "3199                                       [IN:recipe ]   \n",
       "\n",
       "                                            xtreme-up source_language  \n",
       "0                                  PLACE_NAME: eMlazi             zul  \n",
       "1                                                                 zul  \n",
       "2       PERSONAL_NAME: Clara $$ LANGUAGE_NAME: Tswana             zul  \n",
       "3                           COUNTRY: Ningizimu Afrika             zul  \n",
       "4                             CITY_OR_PROVINCE: eGoli             zul  \n",
       "...                                               ...             ...  \n",
       "3195                 DISH_OR_FOOD: bunny chow ebabayo             zul  \n",
       "3196     DISH_OR_FOOD: atchar $$ DISH_OR_FOOD: umango             zul  \n",
       "3197                     DISH_OR_FOOD: mutton ebabayo             zul  \n",
       "3198  DISH_OR_FOOD: atchar $$ DISH_OR_FOOD: chakalaka             zul  \n",
       "3199                                                              zul  \n",
       "\n",
       "[3200 rows x 7 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def translate(source_language: str, target_language: str, texts: List[str]) -> List[str]:\n",
    "    \"\"\"Translate texts from source language to target language.\"\"\"\n",
    "    print(f\"Translating {len(texts)} texts from {source_language} to {target_language}\")\n",
    "    sample = texts[:1]\n",
    "    print(translater(sample, src_lang=source_language, tgt_lang=target_language))\n",
    "    return [content['translation_text']  for content in translater(texts, src_lang=source_language, tgt_lang=target_language, batch_size=128)]\n",
    "    # return [translater(text, src_lang=source_language, tgt_lang=target_language)[0] for text in texts]\n",
    "\n",
    "for language in data_loader.LANGUAGES:\n",
    "    data = manager.load_data(language, \"full\")\n",
    "    # domain,intent,text,spans,logical_form\n",
    "    # banking,balance,ብዙ ብር ያለው የትኛው አካውንቴ ነው,3:5:SL:CURRENCY,[IN:balance [SL:CURRENCY ብር] ]\n",
    "    # banking,balance,የባንክ አካውንቶቼ ውስጥ አሁን ያለኝን ቀሪ ሂሳብ ንገሩኝ።,,[IN:balance ]\n",
    "    # banking,balance,በባንክ አካውንቴ አሁን ያለኝ አጠቃላይ ገንዘብ ምን ያህል ነው።,,[IN:balance ]\n",
    "    \n",
    "    data['text'] = translate(mapper[language], mapper[\"eng\"], data['text'].tolist())\n",
    "    # break\n",
    "    data.to_csv(f\"data/output/{language}_eng.csv\", index=False)\n",
    "data"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rag",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}