back_translation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pororo import Pororo\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 데이터 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/data/ephemeral/level2_datacentric/data/train.csv\") # 데이터 경로 맞게 고치기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 번역기 생성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mt = Pororo(task=\"translation\", lang=\"multi\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mt(\"케빈은 아직도 일을 하고 있다.\", src=\"ko\", tgt=\"en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mt(\"死神は りんごしか食べない。\", src=\"ja\", tgt=\"ko\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### back translation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def back_translation(text, lang='en'):\n",
    "    trans_text = mt(text, src='ko',tgt=lang)\n",
    "    backtrans_text = mt(trans_text, src=lang, tgt='ko')\n",
    "    return backtrans_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test\n",
    "for row in df.head(5).iterrows():\n",
    "    print(f\"원본 문장: {row[1][1]}\")\n",
    "    print(f\"역번역 문장(en): {back_translation(row[1][1])}\")\n",
    "    print(f\"역번역 문장(ja): {back_translation(row[1][1],lang='ja')}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['bt_en_sentence'] = df['text'].apply(back_translation)\n",
    "df['bt_ja_sentence'] = df['text'].apply(lambda x: back_translation(x,lang='ja'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"/data/ephemeral/level2_datacentric/data/train_bt.csv\",index=False)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}