tts_finetune.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4ea88f2e-ff27-483c-9ff5-d35d0aeebb5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "!git clone -q https://github.com/ntdgo/Dataset-TTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "beb3d478-7478-46ad-b85d-a37fdf27b4e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "\n",
    "from trainer import Trainer, TrainerArgs\n",
    "\n",
    "from TTS.config.shared_configs import BaseDatasetConfig\n",
    "from TTS.tts.datasets import load_tts_samples\n",
    "from TTS.tts.layers.xtts.trainer.gpt_trainer import GPTArgs, GPTTrainer, GPTTrainerConfig, XttsAudioConfig\n",
    "from TTS.utils.manage import ModelManager"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b730c9b-b404-475e-8a4e-bacdaf7d9570",
   "metadata": {},
   "outputs": [],
   "source": [
    "current_dir = os.getcwd()\n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "806a8f4d-d90b-47ca-948d-414ea1cc0d62",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Logging parameters\n",
    "RUN_NAME = \"GPT_XTTS_v2.0_LJSpeech_FT\"\n",
    "PROJECT_NAME = \"XTTS_trainer\"\n",
    "DASHBOARD_LOGGER = \"tensorboard\"\n",
    "LOGGER_URI = None\n",
    "\n",
    "# Set here the path that the checkpoints will be saved. Default: ./run/training/\n",
    "OUT_PATH = os.path.join(current_dir, \"run\", \"training\")\n",
    "\n",
    "# Training Parameters\n",
    "OPTIMIZER_WD_ONLY_ON_WEIGHTS = True  # for multi-gpu training please make it False\n",
    "START_WITH_EVAL = True  # if True it will star with evaluation\n",
    "BATCH_SIZE = 3  # set here the batch size\n",
    "GRAD_ACUMM_STEPS = 84  # set here the grad accumulation steps\n",
    "# Note: we recommend that BATCH_SIZE * GRAD_ACUMM_STEPS need to be at least 252 for more efficient training. You can increase/decrease BATCH_SIZE but then set GRAD_ACUMM_STEPS accordingly.\n",
    "\n",
    "# Define here the dataset that you want to use for the fine-tuning on.\n",
    "config_dataset = BaseDatasetConfig(\n",
    "    formatter=\"thorsten\",\n",
    "    dataset_name=\"ft_dataset\",\n",
    "    path=os.path.join(current_dir, \"Dataset-TTS\" , \"dataset\", \"TupLeuBacTom\"),\n",
    "    meta_file_train=os.path.join(current_dir, \"Dataset-TTS\" , \"dataset\", \"TupLeuBacTom.txt\"),\n",
    "    language=\"vi-VN\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3ca5a7c2-df2a-49d8-a34e-4ad122b7615b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add here the configs of the datasets\n",
    "DATASETS_CONFIG_LIST = [config_dataset]\n",
    "\n",
    "# Define the path where XTTS v2.0.1 files will be downloaded\n",
    "CHECKPOINTS_OUT_PATH = os.path.join(OUT_PATH, \"XTTS_v2.0_original_model_files/\")\n",
    "os.makedirs(CHECKPOINTS_OUT_PATH, exist_ok=True)\n",
    "\n",
    "\n",
    "# DVAE files\n",
    "DVAE_CHECKPOINT_LINK = \"https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/dvae.pth\"\n",
    "MEL_NORM_LINK = \"https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/mel_stats.pth\"\n",
    "\n",
    "# Set the path to the downloaded files\n",
    "DVAE_CHECKPOINT = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(DVAE_CHECKPOINT_LINK))\n",
    "MEL_NORM_FILE = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(MEL_NORM_LINK))\n",
    "\n",
    "# download DVAE files if needed\n",
    "if not os.path.isfile(DVAE_CHECKPOINT) or not os.path.isfile(MEL_NORM_FILE):\n",
    "    print(\" > Downloading DVAE files!\")\n",
    "    ModelManager._download_model_files([MEL_NORM_LINK, DVAE_CHECKPOINT_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True)\n",
    "\n",
    "\n",
    "# Download XTTS v2.0 checkpoint if needed\n",
    "TOKENIZER_FILE_LINK = \"https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/vocab.json\"\n",
    "XTTS_CHECKPOINT_LINK = \"https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/model.pth\"\n",
    "\n",
    "# XTTS transfer learning parameters: You we need to provide the paths of XTTS model checkpoint that you want to do the fine tuning.\n",
    "TOKENIZER_FILE = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(TOKENIZER_FILE_LINK))  # vocab.json file\n",
    "XTTS_CHECKPOINT = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(XTTS_CHECKPOINT_LINK))  # model.pth file\n",
    "\n",
    "# download XTTS v2.0 files if needed\n",
    "if not os.path.isfile(TOKENIZER_FILE) or not os.path.isfile(XTTS_CHECKPOINT):\n",
    "    print(\" > Downloading XTTS v2.0 files!\")\n",
    "    ModelManager._download_model_files(\n",
    "        [TOKENIZER_FILE_LINK, XTTS_CHECKPOINT_LINK], CHECKPOINTS_OUT_PATH, progress_bar=False\n",
    "    )\n",
    "\n",
    "\n",
    "# Training sentences generations\n",
    "# speaker reference to be used in training test sentences\n",
    "SPEAKER_REFERENCE = [ os.path.join(current_dir, \"Dataset-TTS\" , \"dataset\", \"TupLeuBacTom\", \"wavs\", \"TupLeuBacTom_000.wav\")]\n",
    "LANGUAGE = config_dataset.language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "200bdd06-02b9-474a-8d34-abd15b45bdc3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">> DVAE weights restored from: content/run/training/XTTS_v2.0_original_model_files/dvae.pth\n"
     ]
    }
   ],
   "source": [
    "model_args = GPTArgs(\n",
    "    max_conditioning_length=132300,  # 6 secs\n",
    "    min_conditioning_length=66150,  # 3 secs\n",
    "    debug_loading_failures=False,\n",
    "    max_wav_length=255995,  # ~11.6 seconds\n",
    "    max_text_length=200,\n",
    "    mel_norm_file=MEL_NORM_FILE,\n",
    "    dvae_checkpoint=DVAE_CHECKPOINT,\n",
    "    xtts_checkpoint=XTTS_CHECKPOINT,  # checkpoint path of the model that you want to fine-tune\n",
    "    tokenizer_file=TOKENIZER_FILE,\n",
    "    gpt_num_audio_tokens=1026,\n",
    "    gpt_start_audio_token=1024,\n",
    "    gpt_stop_audio_token=1025,\n",
    "    gpt_use_masking_gt_prompt_approach=True,\n",
    "    gpt_use_perceiver_resampler=True,\n",
    ")\n",
    "# define audio config\n",
    "audio_config = XttsAudioConfig(sample_rate=22050, dvae_sample_rate=22050, output_sample_rate=24000)\n",
    "# training parameters config\n",
    "config = GPTTrainerConfig(\n",
    "    output_path=OUT_PATH,\n",
    "    model_args=model_args,\n",
    "    run_name=RUN_NAME,\n",
    "    project_name=PROJECT_NAME,\n",
    "    run_description=\"\"\"\n",
    "        GPT XTTS training\n",
    "        \"\"\",\n",
    "    dashboard_logger=DASHBOARD_LOGGER,\n",
    "    logger_uri=LOGGER_URI,\n",
    "    audio=audio_config,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    batch_group_size=48,\n",
    "    eval_batch_size=BATCH_SIZE,\n",
    "    num_loader_workers=8,\n",
    "    eval_split_max_size=256,\n",
    "    print_step=50,\n",
    "    plot_step=100,\n",
    "    log_model_step=1000,\n",
    "    save_step=10000,\n",
    "    save_n_checkpoints=1,\n",
    "    save_checkpoints=True,\n",
    "    # target_loss=\"loss\",\n",
    "    print_eval=False,\n",
    "    # Optimizer values like tortoise, pytorch implementation with modifications to not apply WD to non-weight parameters.\n",
    "    optimizer=\"AdamW\",\n",
    "    optimizer_wd_only_on_weights=OPTIMIZER_WD_ONLY_ON_WEIGHTS,\n",
    "    optimizer_params={\"betas\": [0.9, 0.96], \"eps\": 1e-8, \"weight_decay\": 1e-2},\n",
    "    lr=5e-06,  # learning rate\n",
    "    lr_scheduler=\"MultiStepLR\",\n",
    "    # it was adjusted accordly for the new step scheme\n",
    "    lr_scheduler_params={\"milestones\": [50000 * 18, 150000 * 18, 300000 * 18], \"gamma\": 0.5, \"last_epoch\": -1},\n",
    "    test_sentences=[\n",
    "        {\n",
    "            \"text\": \"It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.\",\n",
    "            \"speaker_wav\": SPEAKER_REFERENCE,\n",
    "            \"language\": LANGUAGE,\n",
    "        },\n",
    "        {\n",
    "            \"text\": \"This cake is great. It's so delicious and moist.\",\n",
    "            \"speaker_wav\": SPEAKER_REFERENCE,\n",
    "            \"language\": LANGUAGE,\n",
    "        },\n",
    "    ],\n",
    ")\n",
    "\n",
    "# init the model from config\n",
    "model = GPTTrainer.init_from_config(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "4856c973-75b7-409e-9158-59533f4e04c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "fatal: not a git repository (or any parent up to mount point /mnt)\n",
      "Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set).\n",
      "fatal: not a git repository (or any parent up to mount point /mnt)\n",
      "Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set).\n",
      " > Training Environment:\n",
      " | > Backend: Torch\n",
      " | > Mixed precision: False\n",
      " | > Precision: float32\n",
      " | > Current device: 0\n",
      " | > Num. of GPUs: 1\n",
      " | > Num. of CPUs: 32\n",
      " | > Num. of Torch Threads: 1\n",
      " | > Torch seed: 1\n",
      " | > Torch CUDNN: True\n",
      " | > Torch CUDNN deterministic: False\n",
      " | > Torch CUDNN benchmark: False\n",
      " | > Torch TF32 MatMul: False\n",
      " > Start Tensorboard: tensorboard --logdir=content/run/training/GPT_XTTS_v2.0_LJSpeech_FT-April-18-2024_10+23PM-0000000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " | > Found 219 files in /mnt/c/Users/ThanhDat/wsl_data/Dataset-TTS/dataset/TupLeuBacTom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " > Model has 518442047 parameters\n",
      "\n",
      "\u001b[4m\u001b[1m > EPOCH: 0/1000\u001b[0m\n",
      " --> content/run/training/GPT_XTTS_v2.0_LJSpeech_FT-April-18-2024_10+23PM-0000000\n",
      " ! Run is removed from content/run/training/GPT_XTTS_v2.0_LJSpeech_FT-April-18-2024_10+23PM-0000000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " > Filtering invalid eval samples!!\n",
      " > Total eval samples after filtering: 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/home/thanhdat/miniconda3/envs/tts/lib/python3.10/site-packages/trainer/trainer.py\", line 1833, in fit\n",
      "    self._fit()\n",
      "  File \"/home/thanhdat/miniconda3/envs/tts/lib/python3.10/site-packages/trainer/trainer.py\", line 1787, in _fit\n",
      "    self.eval_epoch()\n",
      "  File \"/home/thanhdat/miniconda3/envs/tts/lib/python3.10/site-packages/trainer/trainer.py\", line 1628, in eval_epoch\n",
      "    self.get_eval_dataloader(\n",
      "  File \"/home/thanhdat/miniconda3/envs/tts/lib/python3.10/site-packages/trainer/trainer.py\", line 990, in get_eval_dataloader\n",
      "    return self._get_loader(\n",
      "  File \"/home/thanhdat/miniconda3/envs/tts/lib/python3.10/site-packages/trainer/trainer.py\", line 914, in _get_loader\n",
      "    len(loader) > 0\n",
      "AssertionError:  ❗ len(DataLoader) returns 0. Make sure your dataset is not empty or len(dataset) > 0. \n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'tuple' object has no attribute 'tb_frame'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/tts/lib/python3.10/site-packages/trainer/trainer.py:1833\u001b[0m, in \u001b[0;36mTrainer.fit\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1832\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1833\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fit\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1834\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mrank \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
      "File \u001b[0;32m~/miniconda3/envs/tts/lib/python3.10/site-packages/trainer/trainer.py:1787\u001b[0m, in \u001b[0;36mTrainer._fit\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1786\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mrun_eval:\n\u001b[0;32m-> 1787\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43meval_epoch\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1788\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m epoch \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mtest_delay_epochs \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mrank \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
      "File \u001b[0;32m~/miniconda3/envs/tts/lib/python3.10/site-packages/trainer/trainer.py:1628\u001b[0m, in \u001b[0;36mTrainer.eval_epoch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1626\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39meval_loader \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1627\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39meval_loader \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m-> 1628\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_eval_dataloader\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1629\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtraining_assets\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1630\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43meval_samples\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1631\u001b[0m \u001b[43m            \u001b[49m\u001b[43mverbose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m   1632\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1633\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mrun_eval\n\u001b[1;32m   1634\u001b[0m         \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m   1635\u001b[0m     )\n\u001b[1;32m   1637\u001b[0m torch\u001b[38;5;241m.\u001b[39mset_grad_enabled(\u001b[38;5;28;01mFalse\u001b[39;00m)\n",
      "File \u001b[0;32m~/miniconda3/envs/tts/lib/python3.10/site-packages/trainer/trainer.py:990\u001b[0m, in \u001b[0;36mTrainer.get_eval_dataloader\u001b[0;34m(self, training_assets, samples, verbose)\u001b[0m\n\u001b[1;32m    988\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m loader\n\u001b[0;32m--> 990\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_loader\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    991\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    992\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    993\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtraining_assets\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    994\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    995\u001b[0m \u001b[43m    \u001b[49m\u001b[43msamples\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    996\u001b[0m \u001b[43m    \u001b[49m\u001b[43mverbose\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    997\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnum_gpus\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    998\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/tts/lib/python3.10/site-packages/trainer/trainer.py:914\u001b[0m, in \u001b[0;36mTrainer._get_loader\u001b[0;34m(self, model, config, assets, is_eval, samples, verbose, num_gpus)\u001b[0m\n\u001b[1;32m    909\u001b[0m         loader \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mget_data_loader(\n\u001b[1;32m    910\u001b[0m             config\u001b[38;5;241m=\u001b[39mconfig, assets\u001b[38;5;241m=\u001b[39massets, is_eval\u001b[38;5;241m=\u001b[39mis_eval, samples\u001b[38;5;241m=\u001b[39msamples, verbose\u001b[38;5;241m=\u001b[39mverbose, num_gpus\u001b[38;5;241m=\u001b[39mnum_gpus\n\u001b[1;32m    911\u001b[0m         )\n\u001b[1;32m    913\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m (\n\u001b[0;32m--> 914\u001b[0m     \u001b[38;5;28mlen\u001b[39m(loader) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m    915\u001b[0m ), \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m ❗ len(DataLoader) returns 0. Make sure your dataset is not empty or len(dataset) > 0. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    916\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m loader\n",
      "\u001b[0;31mAssertionError\u001b[0m:  ❗ len(DataLoader) returns 0. Make sure your dataset is not empty or len(dataset) > 0. ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mSystemExit\u001b[0m                                Traceback (most recent call last)",
      "    \u001b[0;31m[... skipping hidden 1 frame]\u001b[0m\n",
      "Cell \u001b[0;32mIn[20], line 23\u001b[0m\n\u001b[1;32m     10\u001b[0m trainer \u001b[38;5;241m=\u001b[39m Trainer(\n\u001b[1;32m     11\u001b[0m     TrainerArgs(\n\u001b[1;32m     12\u001b[0m         restore_path\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,  \u001b[38;5;66;03m# xtts checkpoint is restored via xtts_checkpoint key so no need of restore it using Trainer restore_path parameter\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     21\u001b[0m     eval_samples\u001b[38;5;241m=\u001b[39meval_samples,\n\u001b[1;32m     22\u001b[0m )\n\u001b[0;32m---> 23\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/tts/lib/python3.10/site-packages/trainer/trainer.py:1862\u001b[0m, in \u001b[0;36mTrainer.fit\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1861\u001b[0m traceback\u001b[38;5;241m.\u001b[39mprint_exc()\n\u001b[0;32m-> 1862\u001b[0m \u001b[43msys\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexit\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mSystemExit\u001b[0m: 1",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "    \u001b[0;31m[... skipping hidden 1 frame]\u001b[0m\n",
      "File \u001b[0;32m~/miniconda3/envs/tts/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2121\u001b[0m, in \u001b[0;36mInteractiveShell.showtraceback\u001b[0;34m(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)\u001b[0m\n\u001b[1;32m   2118\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m exception_only:\n\u001b[1;32m   2119\u001b[0m     stb \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mAn exception has occurred, use \u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mtb to see \u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m   2120\u001b[0m            \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mthe full traceback.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m'\u001b[39m]\n\u001b[0;32m-> 2121\u001b[0m     stb\u001b[38;5;241m.\u001b[39mextend(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mInteractiveTB\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_exception_only\u001b[49m\u001b[43m(\u001b[49m\u001b[43metype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2122\u001b[0m \u001b[43m                                                     \u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m   2123\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   2125\u001b[0m     \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcontains_exceptiongroup\u001b[39m(val):\n",
      "File \u001b[0;32m~/miniconda3/envs/tts/lib/python3.10/site-packages/IPython/core/ultratb.py:710\u001b[0m, in \u001b[0;36mListTB.get_exception_only\u001b[0;34m(self, etype, value)\u001b[0m\n\u001b[1;32m    702\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_exception_only\u001b[39m(\u001b[38;5;28mself\u001b[39m, etype, value):\n\u001b[1;32m    703\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Only print the exception type and message, without a traceback.\u001b[39;00m\n\u001b[1;32m    704\u001b[0m \n\u001b[1;32m    705\u001b[0m \u001b[38;5;124;03m    Parameters\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    708\u001b[0m \u001b[38;5;124;03m    value : exception value\u001b[39;00m\n\u001b[1;32m    709\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 710\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mListTB\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstructured_traceback\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43metype\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/tts/lib/python3.10/site-packages/IPython/core/ultratb.py:568\u001b[0m, in \u001b[0;36mListTB.structured_traceback\u001b[0;34m(self, etype, evalue, etb, tb_offset, context)\u001b[0m\n\u001b[1;32m    565\u001b[0m     chained_exc_ids\u001b[38;5;241m.\u001b[39madd(\u001b[38;5;28mid\u001b[39m(exception[\u001b[38;5;241m1\u001b[39m]))\n\u001b[1;32m    566\u001b[0m     chained_exceptions_tb_offset \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m    567\u001b[0m     out_list \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m--> 568\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstructured_traceback\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    569\u001b[0m \u001b[43m            \u001b[49m\u001b[43metype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    570\u001b[0m \u001b[43m            \u001b[49m\u001b[43mevalue\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    571\u001b[0m \u001b[43m            \u001b[49m\u001b[43m(\u001b[49m\u001b[43metb\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mchained_exc_ids\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# type: ignore\u001b[39;49;00m\n\u001b[1;32m    572\u001b[0m \u001b[43m            \u001b[49m\u001b[43mchained_exceptions_tb_offset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    573\u001b[0m \u001b[43m            \u001b[49m\u001b[43mcontext\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    574\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    575\u001b[0m         \u001b[38;5;241m+\u001b[39m chained_exception_message\n\u001b[1;32m    576\u001b[0m         \u001b[38;5;241m+\u001b[39m out_list)\n\u001b[1;32m    578\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m out_list\n",
      "File \u001b[0;32m~/miniconda3/envs/tts/lib/python3.10/site-packages/IPython/core/ultratb.py:1435\u001b[0m, in \u001b[0;36mAutoFormattedTB.structured_traceback\u001b[0;34m(self, etype, evalue, etb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1433\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1434\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtb \u001b[38;5;241m=\u001b[39m etb\n\u001b[0;32m-> 1435\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mFormattedTB\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstructured_traceback\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1436\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43metype\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43metb\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtb_offset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnumber_of_lines_of_context\u001b[49m\n\u001b[1;32m   1437\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/tts/lib/python3.10/site-packages/IPython/core/ultratb.py:1326\u001b[0m, in \u001b[0;36mFormattedTB.structured_traceback\u001b[0;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1323\u001b[0m mode \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmode\n\u001b[1;32m   1324\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m mode \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mverbose_modes:\n\u001b[1;32m   1325\u001b[0m     \u001b[38;5;66;03m# Verbose modes need a full traceback\u001b[39;00m\n\u001b[0;32m-> 1326\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mVerboseTB\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstructured_traceback\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1327\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43metype\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtb\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtb_offset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnumber_of_lines_of_context\u001b[49m\n\u001b[1;32m   1328\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1329\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m mode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mMinimal\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m   1330\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m ListTB\u001b[38;5;241m.\u001b[39mget_exception_only(\u001b[38;5;28mself\u001b[39m, etype, value)\n",
      "File \u001b[0;32m~/miniconda3/envs/tts/lib/python3.10/site-packages/IPython/core/ultratb.py:1173\u001b[0m, in \u001b[0;36mVerboseTB.structured_traceback\u001b[0;34m(self, etype, evalue, etb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1164\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstructured_traceback\u001b[39m(\n\u001b[1;32m   1165\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m   1166\u001b[0m     etype: \u001b[38;5;28mtype\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1170\u001b[0m     number_of_lines_of_context: \u001b[38;5;28mint\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m5\u001b[39m,\n\u001b[1;32m   1171\u001b[0m ):\n\u001b[1;32m   1172\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Return a nice text document describing the traceback.\"\"\"\u001b[39;00m\n\u001b[0;32m-> 1173\u001b[0m     formatted_exception \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mformat_exception_as_a_whole\u001b[49m\u001b[43m(\u001b[49m\u001b[43metype\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43metb\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnumber_of_lines_of_context\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1174\u001b[0m \u001b[43m                                                           \u001b[49m\u001b[43mtb_offset\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1176\u001b[0m     colors \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mColors  \u001b[38;5;66;03m# just a shorthand + quicker name lookup\u001b[39;00m\n\u001b[1;32m   1177\u001b[0m     colorsnormal \u001b[38;5;241m=\u001b[39m colors\u001b[38;5;241m.\u001b[39mNormal  \u001b[38;5;66;03m# used a lot\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/tts/lib/python3.10/site-packages/IPython/core/ultratb.py:1063\u001b[0m, in \u001b[0;36mVerboseTB.format_exception_as_a_whole\u001b[0;34m(self, etype, evalue, etb, number_of_lines_of_context, tb_offset)\u001b[0m\n\u001b[1;32m   1060\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(tb_offset, \u001b[38;5;28mint\u001b[39m)\n\u001b[1;32m   1061\u001b[0m head \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_header(\u001b[38;5;28mstr\u001b[39m(etype), \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlong_header)\n\u001b[1;32m   1062\u001b[0m records \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m-> 1063\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_records\u001b[49m\u001b[43m(\u001b[49m\u001b[43metb\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnumber_of_lines_of_context\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtb_offset\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m etb \u001b[38;5;28;01melse\u001b[39;00m []\n\u001b[1;32m   1064\u001b[0m )\n\u001b[1;32m   1066\u001b[0m frames \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m   1067\u001b[0m skipped \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n",
      "File \u001b[0;32m~/miniconda3/envs/tts/lib/python3.10/site-packages/IPython/core/ultratb.py:1131\u001b[0m, in \u001b[0;36mVerboseTB.get_records\u001b[0;34m(self, etb, number_of_lines_of_context, tb_offset)\u001b[0m\n\u001b[1;32m   1129\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m cf \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1130\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1131\u001b[0m         mod \u001b[38;5;241m=\u001b[39m inspect\u001b[38;5;241m.\u001b[39mgetmodule(\u001b[43mcf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtb_frame\u001b[49m)\n\u001b[1;32m   1132\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m mod \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1133\u001b[0m             mod_name \u001b[38;5;241m=\u001b[39m mod\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'tuple' object has no attribute 'tb_frame'"
     ]
    }
   ],
   "source": [
    "# load training samples\n",
    "train_samples, eval_samples = load_tts_samples(\n",
    "    DATASETS_CONFIG_LIST,\n",
    "    eval_split=True,\n",
    "    eval_split_max_size=config.eval_split_max_size,\n",
    "    eval_split_size=config.eval_split_size,\n",
    ")\n",
    "\n",
    "# init the trainer and 🚀\n",
    "trainer = Trainer(\n",
    "    TrainerArgs(\n",
    "        restore_path=None,  # xtts checkpoint is restored via xtts_checkpoint key so no need of restore it using Trainer restore_path parameter\n",
    "        skip_train_epoch=False,\n",
    "        start_with_eval=START_WITH_EVAL,\n",
    "        grad_accum_steps=GRAD_ACUMM_STEPS,\n",
    "    ),\n",
    "    config,\n",
    "    output_path=OUT_PATH,\n",
    "    model=model,\n",
    "    train_samples=train_samples,\n",
    "    eval_samples=eval_samples,\n",
    ")\n",
    "trainer.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6db92e9a-b226-4ebf-9188-583964b46c38",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc275e2e-7e42-4c4c-a7a3-7ebbe3c75654",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}