Fine_Tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: You are using pip version 20.2.3; however, version 23.1.2 is available.\n",
      "You should consider upgrading via the '/share/apps/python/3.8.6/intel/bin/python -m pip install --upgrade pip' command.\u001b[0m\n",
      "Hostname: gr048.hpc.nyu.edu\n",
      "Processor: Intel(R) Xeon(R) Platinum 8268 CPU @ 2.90GHz\n",
      "RAM: 165Gi\n"
     ]
    }
   ],
   "source": [
    "!module load cuda/11.6.2\n",
    "!module load cudnn/8.6.0.163-cuda11\n",
    "!pip install -q --use-feature=2020-resolver pynvml zstandard datasets psutil transformers torch bitsandbytes accelerate loralib peft\n",
    "!echo \"Hostname: $(hostname)\"\n",
    "!echo \"Processor: $(lscpu | grep 'Model name' | awk -F ':' '{print $2}' | xargs)\"\n",
    "!echo \"RAM: $(free -h | grep 'Mem:' | awk '{print $4}')\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU: Quadro RTX 8000\n",
      "GPU Memory: 0MiB/46080MiB\n"
     ]
    }
   ],
   "source": [
    "!echo \"GPU: $(nvidia-smi --query-gpu=name --format=csv,noheader)\"\n",
    "!echo \"GPU Memory: $(nvidia-smi | grep MiB |  awk '{print $9 $10 $11}')\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please run\n",
      "\n",
      "python -m bitsandbytes\n",
      "\n",
      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "bin /home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cuda118.so\n",
      "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n",
      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n",
      "CUDA SETUP: Detected CUDA version 118\n",
      "CUDA SETUP: Loading binary /home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cuda118.so...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: /share/apps/r/4.0.3/intel/lib64/R/lib:/share/apps/intel/19.1.2/mkl/lib/intel64:/share/apps/intel/19.1.2/lib/intel64:/share/apps/intel/19.1.2/lib:/share/apps/python/3.8.6/intel/lib::/share/apps/centos/8/usr/lib:/share/apps/centos/8/usr/lib64:/share/apps/centos/8/lib64 did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('r/intel')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('1;/opt/slurm/bin'), PosixPath('1;/share/apps/intel/19.1.2/bin'), PosixPath('1;/usr/lpp/mmfs/bin'), PosixPath('1'), PosixPath('1;/usr/local/bin'), PosixPath('1;/usr/local/sbin'), PosixPath('1;/share/apps/centos/8/bin'), PosixPath('1;/share/apps/python/3.8.6/intel/bin'), PosixPath('1;/usr/sbin'), PosixPath('1;/usr/bin'), PosixPath('1;/share/apps/singularity/bin'), PosixPath('1;/share/apps/local/bin'), PosixPath('1;/home/vgn2004/.local/bin'), PosixPath('1;/home/vgn2004/bin'), PosixPath('1;/share/apps/utils/vnc')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/var/www/ood/apps/sys/dashboard')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('1;/share/apps/modulefiles/intel/19.1.2.lua'), PosixPath('1;/share/apps/modulefiles/r/intel/4.0.3.lua'), PosixPath('1')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/share/zsh/5.5.1/functions')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('1;/share/apps/intel/19.1.2/lib'), PosixPath('1;/share/apps/centos/8/usr/lib64'), PosixPath('1'), PosixPath('1;/share/apps/intel/19.1.2/mkl/lib/intel64'), PosixPath('1;/share/apps/centos/8/usr/lib'), PosixPath('1;/share/apps/centos/8/lib64'), PosixPath('1;/share/apps/python/3.8.6/intel/lib'), PosixPath('1;/share/apps/intel/19.1.2/lib/intel64')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('1'), PosixPath('1;/share/apps/python/3.8.6/intel/lib/pkgconfig'), PosixPath('1;/share/apps/intel/19.1.2/mkl/bin/pkgconfig')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('1')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('python/intel')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('1'), PosixPath('1;intel/19.1.2'), PosixPath('1;r/intel/4.0.3'), PosixPath('python/intel/3.8.6')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/mnt,/share/apps,/vast')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/state/partition1/vgn2004-singularity-cache')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('1'), PosixPath('1;/share/apps/intel/19.1.2/mkl/lib/intel64'), PosixPath('1;/share/apps/intel/19.1.2/lib/intel64'), PosixPath('1;/share/apps/python/3.8.6/intel/lib')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('r/intel/4.0.3'), PosixPath('python/intel/3.8.6'), PosixPath('intel/19.1.2')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('1'), PosixPath('1;/share/apps/intel/19.1.2/man/common'), PosixPath('1;/share/apps/python/3.8.6/intel/share/man'), PosixPath('1;/share/apps/intel/19.1.2/man')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('1;/share/apps/python/3.8.6/intel/include'), PosixPath('1')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('-8}\\n}'), PosixPath(\"() {  tr -cd 'a-zA-Z0-9' < /dev/urandom 2> /dev/null | head -c${1\")}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('() {  ( alias;\\n eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@\\n}')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('() {  local port=\"${1}\";\\n local time=\"${2'), PosixPath('-30}\";\\n for ((i=1; i<=time*2; i++))\\n do\\n port_used \"${port}\";\\n port_status=$?;\\n if [ \"$port_status\" == \"0\" ]; then\\n return 0;\\n else\\n if [ \"$port_status\" == \"127\" ]; then\\n echo \"commands to find port were either not found or inaccessible.\";\\n echo \"command options are lsof, nc, bash\\'s /dev/tcp, or python (or python3) with socket lib.\";\\n return 127;\\n fi;\\n fi;\\n sleep 0.5;\\n done;\\n return 1\\n}')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('-65535}\");\\n while port_used \"${host}'), PosixPath('}\";\\n local host=$((expr \"${1}\" '), PosixPath('\\' || echo \"localhost\") | awk \\'END{print $NF}\\');\\n local port_strategies=(port_used_nc port_used_lsof port_used_bash port_used_python port_used_python3);\\n for strategy in ${port_strategies[@]};\\n do\\n $strategy $host $port;\\n status=$?;\\n if [[ \"$status\" == \"0\" ]] || [[ \"$status\" == \"1\" ]]; then\\n return $status;\\n fi;\\n done;\\n return 127\\n };\\n export -f port_used;\\n find_port () \\n { \\n local host=\"${1'), PosixPath('() {  random_number () \\n { \\n shuf -i ${1}-${2} -n 1\\n };\\n export -f random_number;\\n port_used_python () \\n { \\n python -c \"import socket; socket.socket().connect((\\'$1\\',$2))\" > /dev/null 2>&1\\n };\\n port_used_python3 () \\n { \\n python3 -c \"import socket; socket.socket().connect((\\'$1\\',$2))\" > /dev/null 2>&1\\n };\\n port_used_nc () \\n { \\n nc -w 2 \"$1\" \"$2\" < /dev/null > /dev/null 2>&1\\n };\\n port_used_lsof () \\n { \\n lsof -i '), PosixPath(\" '\\\\(.*\\\\)\"), PosixPath('\"$2\" > /dev/null 2>&1\\n };\\n port_used_bash () \\n { \\n local bash_supported=$(strings /bin/bash 2>/dev/null | grep tcp);\\n if [ \"$bash_supported\" == \"/dev/tcp/*/*\" ]; then\\n ( '), PosixPath('-65535}\");\\n done;\\n echo \"${port}\"\\n };\\n export -f find_port;\\n wait_until_port_used () \\n { \\n local port=\"${1}\";\\n local time=\"${2'), PosixPath('-8}\\n };\\n export -f create_passwd\\n}'), PosixPath(' < /dev/tcp/$1/$2 ) > /dev/null 2>&1;\\n else\\n return 127;\\n fi\\n };\\n port_used () \\n { \\n local port=\"${1#*'), PosixPath('-30}\";\\n for ((i=1; i<=time*2; i++))\\n do\\n port_used \"${port}\";\\n port_status=$?;\\n if [ \"$port_status\" == \"0\" ]; then\\n return 0;\\n else\\n if [ \"$port_status\" == \"127\" ]; then\\n echo \"commands to find port were either not found or inaccessible.\";\\n echo \"command options are lsof, nc, bash\\'s /dev/tcp, or python (or python3) with socket lib.\";\\n return 127;\\n fi;\\n fi;\\n sleep 0.5;\\n done;\\n return 1\\n };\\n export -f wait_until_port_used;\\n create_passwd () \\n { \\n tr -cd \\'a-zA-Z0-9\\' < /dev/urandom 2> /dev/null | head -c${1'), PosixPath('-localhost}\";\\n local port=$(random_number \"${2'), PosixPath('-2000}\" \"${3'), PosixPath('${port}\"; do\\n port=$(random_number \"${2')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('() {  eval $($LMOD_DIR/ml_cmd \"$@\")\\n}')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//ipykernel.pylab.backend_inline')}\n",
      "  warn(msg)\n",
      "/home/vgn2004/.local/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so'), PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0')}.. We'll flip a coin and try one of these, in order to fail forward.\n",
      "Either way, this might cause trouble in the future:\n",
      "If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n",
      "  warn(msg)\n",
      "/share/apps/python/3.8.6/intel/lib/python3.8/site-packages/setuptools/distutils_patch.py:25: UserWarning: Distutils was imported before Setuptools. This usage is discouraged and may exhibit undesirable behaviors or errors. Please use Setuptools' objects directly or at least import Setuptools first.\n",
      "  warnings.warn(\n",
      "2023-05-01 20:22:03,603 - INFO - <module>:85 - Is Flash Attention Enabled: True\n",
      "2023-05-01 20:22:03,604 - INFO - <module>:86 - Is Mem Efficient SDP Enabled: False\n",
      "2023-05-01 20:22:03,604 - INFO - <module>:87 - Is Math SDP Enabled: False\n",
      "2023-05-01 20:22:03,616 - INFO - print_gpu_utilization:106 - GPU memory occupied: 526 MB.\n",
      "2023-05-01 20:22:03,790 - INFO - run_data_pipeline:222 - Baseline: RAM used: 423.14 MB\n",
      "2023-05-01 20:22:03,791 - ERROR - run_data_pipeline:235 - \n",
      "2023-05-01 20:22:03,915 - WARNING - download_and_prepare:817 - Found cached dataset json (/scratch/vgn2004/fine_tuning/datasets/json/default-f34239bbfb70cca2/0.0.0/fe5dd6ea2639a6df622901539cb550cf8797e5a6b2dd7af1cf934bed8e233e6e)\n",
      "2023-05-01 20:22:03,930 - INFO - run_data_pipeline:244 - RAM used: 430.45 MB\n",
      "2023-05-01 20:22:03,931 - INFO - run_data_pipeline:245 - Dataset sample: {'meta': {'APPLICATION_ID': 100075}, 'text': \"ACF's Office of Refugee Resettlement (ORR) administers a variety of social service programs intended to connect newly resettled refugees with critical resources, help them become economically self-sufficient, and help them integrate into American society. One such program is the Refugee Cash Assistance (RCA) program, which provides both financial support and social services to newly resettled refugees. Refugee Cash Assistance is similar to TANF in that both are cash assistance programs that provide services aimed at promoting self-sufficiency; however the content, mode of delivery and rules surrounding these services vary significantly by state and locality. Some counties and states have reportedly integrated the delivery of TANF and RCA in a purposeful way to better serve refugees. However, there is little documented information on the extent to which refugees access benefits and services through TANF and RCA, differences in refugee characteristics between the two programs, how outcomes compare for refugees served under these two programs, whether integration of these programs holds promise for refugee self-sufficiency, and whether data is available to answer these questions. The Understanding the Intersection Between TANF and Refugee Cash Assistance Services project aims to improve understanding of how RCA and TANF serve refugee populations, how these programs intersect, and how these programs may be related to refugee self-sufficiency and employment outcomes. In fall 2014 ACF launched this descriptive study to document the similarities and differences between cash assistance and associated social services offered under RCA and TANF across different selected jurisdictions. The study aims to better understand the population of refugees served by TANF and RCA, and the major differences in programmatic services associated with these two programs. The study will also explore how states and localities have coordinated TANF and RCA programs to deliver social services to refugees and whether these approaches hold promise for long-term job stability and economic self-sufficiency among refugees. This field study will provide a deeper understanding of current social service delivery systems serving refugees and will help to identify gaps in existing knowledge and data around these systems. By improving knowledge of these programs and participant experiences, ACF hopes to move toward better serving this population. The project is being conducted by Abt Associates and MEF Associates.\"}\n",
      "2023-05-01 20:22:03,931 - INFO - run_data_pipeline:247 - Dataset size (cache file) : 2.01 GB\n",
      "2023-05-01 20:22:03,931 - INFO - preprocess_data:141 - Loading dataset from disk...\n",
      "2023-05-01 20:22:03,947 - INFO - preprocess_data:146 - Time taken to load dataset from : 0.02 seconds\n",
      "2023-05-01 20:22:03,947 - INFO - create_dataloaders:190 - Creating data loaders...\n",
      "2023-05-01 20:22:03,948 - INFO - create_dataloaders:205 - Time taken to create data loaders: 0.00 seconds\n",
      "2023-05-01 20:22:06,965 - INFO - create_or_load_model:370 - Model: facebook/opt-125m\n",
      "2023-05-01 20:22:06,966 - INFO - print_trainable_parameters:121 - Parameters: Trainable- 125.24M|| All- 125.24M || Trainable%- 100.0\n",
      "2023-05-01 20:22:06,968 - INFO - create_or_load_model:372 - Memory Memory Footprint: 500.957184 MB\n",
      "2023-05-01 20:22:06,968 - INFO - create_or_load_model:373 - Model is on device: cuda:0\n",
      "2023-05-01 20:22:11,317 - INFO - <module>:590 - Initial Text:\n",
      "This is probably the second or third time you have mentioned being a little awkward, but how did you get involved in doing a sketch so late in the life time?It was just a project I did that wasn't exactly long, so I was just like \"Hey, how about the next time I've been doing a sketch with a lot of people from my school that were in my class.\" The person I talked to was a guy, I knew the guy, and then the people I talked to did more sketching. I didn't know who it was. I'd just come to learn from the people I'd told the girl about, so it was pretty easy to get into.Good to know. And I remember you guys getting an answer pretty quick! :) So thank you :DYeah, we really didn't know until after I started to do a bunch of sketchs and then one of those people I had just met called me to let me know. She was very nice, though I didn't realize that until after I'd gotten a few more good calls out.Sounds great!  It sounds a little over my head, but I'll admit I've never really done it before. I'm a pretty good sketch artist, and sketchbooks are quite easy to learn, but it could be a bit of a mess. If you ever have someone suggest a good sketchbook to sketch like this, let me know!Thanks, you've been very helpful.  I was just curious if there's a place to download a sketch book if you had been on a really sketchy school that you're unfamiliar with and don't know how to access it. :)  Also, when you were getting your degree, did you make sketches or other \"real\" work for people?  Thanks for the reply!!I got my degree in biology. A few years ago I took a class with people who I'd worked for. I was talking to one of my colleagues the next day and I asked about what they were doing, and then they asked if they could do any \"real\" work for me. So when I got my degree, there were other people I was chatting with that didn't know much about me or knew me better than me, so I asked if they could sketch my work as well. Of course, I knew they'd let me know, and I was really happy with that.How's your current sketchbook setup? If you want to try this out in other classes do you already have your book or other library, are you currently working for one or both? :)  Thanks again! :)\n",
      "2023-05-01 20:22:11,322 - INFO - fetch_optimizer:315 - Using fused AdamW: True\n",
      "  0%|          | 0/30 [00:00<?, ?it/s]2023-05-01 20:22:11,325 - INFO - train:481 - Epoch: 1/30\n",
      "\n",
      "  0%|          | 0/13861 [00:00<?, ?it/s]\u001b[A2023-05-01 20:22:12,370 - INFO - print_gpu_utilization:106 - GPU memory occupied: 1930 MB.\n",
      "2023-05-01 20:22:12,815 - INFO - train:535 - Batch 1/13861, Loss: 3.2670, Learning Rate: 4.00962309542903e-07\n",
      "\n",
      "  0%|          | 1/13861 [00:01<6:29:06,  1.68s/it]\u001b[A2023-05-01 20:22:14,016 - INFO - print_gpu_utilization:106 - GPU memory occupied: 27256 MB.\n",
      "2023-05-01 20:22:14,448 - INFO - train:535 - Batch 2/13861, Loss: 3.3146, Learning Rate: 8.01924619085806e-07\n",
      "\n",
      "  0%|          | 2/13861 [00:03<6:21:56,  1.65s/it]\u001b[A2023-05-01 20:22:15,649 - INFO - print_gpu_utilization:106 - GPU memory occupied: 33528 MB.\n",
      "2023-05-01 20:22:16,080 - INFO - train:535 - Batch 3/13861, Loss: 3.3264, Learning Rate: 1.202886928628709e-06\n",
      "\n",
      "  0%|          | 3/13861 [00:04<6:19:07,  1.64s/it]\u001b[A2023-05-01 20:22:17,275 - INFO - print_gpu_utilization:106 - GPU memory occupied: 33528 MB.\n",
      "2023-05-01 20:22:17,706 - INFO - train:535 - Batch 4/13861, Loss: 3.3302, Learning Rate: 1.603849238171612e-06\n",
      "\n",
      "  0%|          | 4/13861 [00:06<6:18:50,  1.64s/it]\u001b[A2023-05-01 20:22:18,914 - INFO - print_gpu_utilization:106 - GPU memory occupied: 33528 MB.\n",
      "2023-05-01 20:22:19,402 - INFO - train:535 - Batch 5/13861, Loss: 3.3346, Learning Rate: 2.004811547714515e-06\n",
      "\n",
      "  0%|          | 5/13861 [00:08<6:22:22,  1.66s/it]\u001b[A2023-05-01 20:22:21,027 - INFO - train:535 - Batch 6/13861, Loss: 3.3243, Learning Rate: 2.405773857257418e-06\n",
      "\n",
      "  0%|          | 6/13861 [00:09<6:19:53,  1.65s/it]\u001b[A2023-05-01 20:22:22,652 - INFO - train:535 - Batch 7/13861, Loss: 3.3257, Learning Rate: 2.8067361668003206e-06\n",
      "\n",
      "  0%|          | 7/13861 [00:11<6:18:19,  1.64s/it]\u001b[A2023-05-01 20:22:24,276 - INFO - train:535 - Batch 8/13861, Loss: 3.3332, Learning Rate: 3.207698476343224e-06\n",
      "\n",
      "  0%|          | 8/13861 [00:13<6:17:48,  1.64s/it]\u001b[A2023-05-01 20:22:25,967 - INFO - train:535 - Batch 9/13861, Loss: 3.3356, Learning Rate: 3.608660785886127e-06\n",
      "\n",
      "  0%|          | 9/13861 [00:14<6:21:14,  1.65s/it]\u001b[A2023-05-01 20:22:27,592 - INFO - train:535 - Batch 10/13861, Loss: 3.3337, Learning Rate: 4.00962309542903e-06\n",
      "\n",
      "  0%|          | 10/13861 [00:16<6:19:22,  1.64s/it]\u001b[A2023-05-01 20:22:29,219 - INFO - train:535 - Batch 11/13861, Loss: 3.3328, Learning Rate: 4.410585404971933e-06\n",
      "\n",
      "  0%|          | 11/13861 [00:18<6:18:09,  1.64s/it]\u001b[A2023-05-01 20:22:30,845 - INFO - train:535 - Batch 12/13861, Loss: 3.3309, Learning Rate: 4.811547714514836e-06\n",
      "\n",
      "  0%|          | 12/13861 [00:19<6:17:47,  1.64s/it]\u001b[A2023-05-01 20:22:32,538 - INFO - train:535 - Batch 13/13861, Loss: 3.3374, Learning Rate: 5.212510024057739e-06\n",
      "\n",
      "  0%|          | 13/13861 [00:21<6:21:08,  1.65s/it]\u001b[A2023-05-01 20:22:34,163 - INFO - train:535 - Batch 14/13861, Loss: 3.3363, Learning Rate: 5.613472333600641e-06\n",
      "\n",
      "  0%|          | 14/13861 [00:23<6:19:24,  1.64s/it]\u001b[A2023-05-01 20:22:35,790 - INFO - train:535 - Batch 15/13861, Loss: 3.3331, Learning Rate: 6.014434643143544e-06\n",
      "\n",
      "  0%|          | 15/13861 [00:24<6:18:13,  1.64s/it]\u001b[A2023-05-01 20:22:37,419 - INFO - train:535 - Batch 16/13861, Loss: 3.3370, Learning Rate: 6.415396952686448e-06\n",
      "\n",
      "  0%|          | 16/13861 [00:26<6:17:52,  1.64s/it]\u001b[A2023-05-01 20:22:39,111 - INFO - train:535 - Batch 17/13861, Loss: 3.3360, Learning Rate: 6.816359262229351e-06\n",
      "\n",
      "  0%|          | 17/13861 [00:27<6:21:09,  1.65s/it]\u001b[A2023-05-01 20:22:40,738 - INFO - train:535 - Batch 18/13861, Loss: 3.3323, Learning Rate: 7.217321571772254e-06\n",
      "\n",
      "  0%|          | 18/13861 [00:29<6:19:21,  1.64s/it]\u001b[A2023-05-01 20:22:42,364 - INFO - train:535 - Batch 19/13861, Loss: 3.3340, Learning Rate: 7.618283881315156e-06\n",
      "\n",
      "  0%|          | 19/13861 [00:31<6:18:07,  1.64s/it]\u001b[A2023-05-01 20:22:48,937 - INFO - train:496 - Text:\n",
      "This is awesome! This really does not seem like a bug in my system. Thank you for taking this initiative. I was thinking of going to the store and opening it up to all. This could help me a bit. :)Yeah, I'll be looking at that. Just keep in mind that all the games are listed there, but it could be a one off sale. Thanks for the help, that makes it easier. :)Yeah, that would definitely be helpful.  :) It's kinda hard to make a difference in this way, but I'm definitely starting to get a hang of it.Well yeah, if you're buying the games through the store, you need to pay out of pocket to keep track of it in the end. I'd say you can get that from the free app, which probably costs more, but I think it'd save more time if you were buying games you didn't want to spend money on. It would be a very nice service, though :)Yeah, I'm definitely starting to feel more comfortable with it, so I might take the time to try and make it work.  I'm hoping my next project will help me a bit too. :)Awesome. Good luck!Thank you for this. The store is closed for now, but I'm going to post screenshots whenever it gets open. I've been thinking about buying some of the older games and making some more from the newer titles but that could do more with the game sales.  I'm going to put a little extra work in figuring out how to manage the numbers for the different accounts, but for now, I'm going to stick to just one thing until the store opens again. :)I have this problem in the Netherlands, it starts out to take hours to open the store after getting the games opened because the store closes for the day. (Not a country I want to visit anyway, but maybe I need some help).The store is closed.Nice, I would try to open it as soon as it is open in the Netherlands. :)Haha, I was thinking of maybe posting about that before the store opens, but after the store closes I will try and open it in the Netherlands, so hopefully I will find something better later :)Good luck on your projects! :)I know, it sucks a lot. Just keep trying and don't give up hope. :)Hey, thanks for your generosity! I'm really excited to finally be able to see things like this in real life. It is not something that I want to do. I really do love making and playing game modes, so hopefully it will be a big help!Thanks so much! I'll try and see if I can find something better! :)I've been going through some trouble getting into the web design and usability space. I've been reading up a lot on some pretty interesting stuff that I should be doing, but I've never really found the time to do all of it so that's a plus.  I've also found my interest in web sites is still really strong and I'm really interested in seeing what people do with their design.Thanks for this, really nice work on this. I think that is the one thing I would have to work on. As a new web user, it feels like I can't really put into words much in terms of how I am feeling about the whole thing. :) I am excited to see what the web design team can do for the project, and I have been working hard at getting more creative and getting my mind around the idea of \"how to make it happen.\" Thanks!\n",
      "2023-05-01 20:22:49,377 - INFO - train:535 - Batch 20/13861, Loss: 3.3331, Learning Rate: 8.01924619085806e-06\n",
      "\n",
      "  0%|          | 20/13861 [00:38<12:30:49,  3.25s/it]\u001b[A2023-05-01 20:22:51,070 - INFO - train:535 - Batch 21/13861, Loss: 3.3316, Learning Rate: 8.420208500400963e-06\n",
      "\n",
      "  0%|          | 21/13861 [00:39<10:42:06,  2.78s/it]\u001b[A2023-05-01 20:22:52,697 - INFO - train:535 - Batch 22/13861, Loss: 3.3323, Learning Rate: 8.821170809943866e-06\n",
      "\n",
      "  0%|          | 22/13861 [00:41<9:22:00,  2.44s/it] \u001b[A2023-05-01 20:22:54,324 - INFO - train:535 - Batch 23/13861, Loss: 3.3313, Learning Rate: 9.222133119486768e-06\n",
      "\n",
      "  0%|          | 23/13861 [00:43<8:25:59,  2.19s/it]\u001b[A2023-05-01 20:22:55,952 - INFO - train:535 - Batch 24/13861, Loss: 3.3327, Learning Rate: 9.623095429029671e-06\n",
      "\n",
      "  0%|          | 24/13861 [00:44<7:47:57,  2.03s/it]\u001b[A2023-05-01 20:22:57,655 - INFO - train:535 - Batch 25/13861, Loss: 3.3324, Learning Rate: 1.0024057738572574e-05\n",
      "\n",
      "  0%|          | 25/13861 [00:46<7:24:06,  1.93s/it]\u001b[A2023-05-01 20:22:59,281 - INFO - train:535 - Batch 26/13861, Loss: 3.3337, Learning Rate: 1.0425020048115479e-05\n",
      "\n",
      "  0%|          | 26/13861 [00:48<7:03:23,  1.84s/it]\u001b[A2023-05-01 20:23:00,909 - INFO - train:535 - Batch 27/13861, Loss: 3.3311, Learning Rate: 1.082598235765838e-05\n",
      "\n",
      "  0%|          | 27/13861 [00:49<6:48:54,  1.77s/it]\u001b[A2023-05-01 20:23:02,536 - INFO - train:535 - Batch 28/13861, Loss: 3.3304, Learning Rate: 1.1226944667201283e-05\n",
      "\n",
      "  0%|          | 28/13861 [00:51<6:39:15,  1.73s/it]\u001b[A2023-05-01 20:23:04,229 - INFO - train:535 - Batch 29/13861, Loss: 3.3287, Learning Rate: 1.1627906976744187e-05\n",
      "\n",
      "  0%|          | 29/13861 [00:53<6:36:01,  1.72s/it]\u001b[A2023-05-01 20:23:05,857 - INFO - train:535 - Batch 30/13861, Loss: 3.3256, Learning Rate: 1.2028869286287088e-05\n",
      "\n",
      "  0%|          | 30/13861 [00:54<6:29:52,  1.69s/it]\u001b[A2023-05-01 20:23:07,485 - INFO - train:535 - Batch 31/13861, Loss: 3.3249, Learning Rate: 1.2429831595829993e-05\n",
      "\n",
      "  0%|          | 31/13861 [00:56<6:25:27,  1.67s/it]\u001b[A2023-05-01 20:23:09,113 - INFO - train:535 - Batch 32/13861, Loss: 3.3249, Learning Rate: 1.2830793905372896e-05\n",
      "\n",
      "  0%|          | 32/13861 [00:57<6:22:52,  1.66s/it]\u001b[A2023-05-01 20:23:10,807 - INFO - train:535 - Batch 33/13861, Loss: 3.3238, Learning Rate: 1.3231756214915798e-05\n",
      "\n",
      "  0%|          | 33/13861 [00:59<6:24:34,  1.67s/it]\u001b[A2023-05-01 20:23:12,434 - INFO - train:535 - Batch 34/13861, Loss: 3.3240, Learning Rate: 1.3632718524458701e-05\n",
      "\n",
      "  0%|          | 34/13861 [01:01<6:21:44,  1.66s/it]\u001b[A2023-05-01 20:23:14,062 - INFO - train:535 - Batch 35/13861, Loss: 3.3241, Learning Rate: 1.4033680834001606e-05\n",
      "\n",
      "  0%|          | 35/13861 [01:02<6:19:41,  1.65s/it]\u001b[A2023-05-01 20:23:15,690 - INFO - train:535 - Batch 36/13861, Loss: 3.3229, Learning Rate: 1.4434643143544509e-05\n",
      "\n",
      "  0%|          | 36/13861 [01:04<6:18:48,  1.64s/it]\u001b[A2023-05-01 20:23:17,384 - INFO - train:535 - Batch 37/13861, Loss: 3.3217, Learning Rate: 1.483560545308741e-05\n",
      "\n",
      "  0%|          | 37/13861 [01:06<6:21:47,  1.66s/it]\u001b[A2023-05-01 20:23:19,013 - INFO - train:535 - Batch 38/13861, Loss: 3.3197, Learning Rate: 1.5236567762630312e-05\n",
      "\n",
      "  0%|          | 38/13861 [01:07<6:19:45,  1.65s/it]\u001b[A2023-05-01 20:23:20,641 - INFO - train:535 - Batch 39/13861, Loss: 3.3176, Learning Rate: 1.5637530072173217e-05\n",
      "\n",
      "  0%|          | 39/13861 [01:09<6:18:21,  1.64s/it]\u001b[A2023-05-01 20:23:29,259 - INFO - train:496 - Text:\n",
      "This is a very large scale, detailed, multi-pronged investigation into the possible role of trans-spermatic mechanisms in developing breast cancer, including a multidisciplinary investigation of the role of trans-spermatic mechanisms in developing breast cancer, and the potential role of trans-spermatic mechanisms in developing breast cancer in terms of prognosis, survival and prognosis. We will evaluate trans-spermatic mechanisms in a new, multi-pronged multi-pronged investigation, focusing on trans-spermatic mechanisms that are implicated in metastatic breast cancer development and progression. We will first evaluate the effectiveness of trans-spermatic mechanisms in developing breast cancer in terms of the mechanisms they interact with. This will focus on trans-spermatic mechanisms that are implicated in tumor progression and growth by the role of trans-spermatic mechanisms in breast cancer progression and development. We will then evaluate trans-spermatic mechanisms and the role of trans-spermatic mechanisms in developing cancer progression and development. In this effort, we will evaluate trans-spermatic mechanisms and trans-spermatic mechanisms to define and develop new, multidisciplinary investigations of the role of trans-spermatic mechanisms in developing cancer progression and development and the potential role of trans-spermatic mechanisms in developing cancer progression and development. We will also evaluate trans-spermatic mechanisms that are implicated in tumor progression and growth by the role of trans-spermatic mechanisms in development. We will also evaluate trans-spermatic mechanisms and trans-spermatic mechanisms in developing breast cancer. In this endeavor, we will establish and test new trans-spermatic mechanisms that will be the focal and leading investigators of this new multi-pronged multi-pronged investigation. We will also focus on trans-spermatic mechanisms that are implicated in tumor progression and growth by the role of trans-spermatic mechanisms in developing breast cancer. We will also investigate the role of trans-spermatic mechanisms and trans-spermatic mechanisms in developing breast cancer. We will also provide new research, new data and new data on trans-spermatic mechanisms. In this effort, we will establish and test new trans-spermatic mechanisms that will be the focal and leading investigators of this new multi-pronged multi-pronged multi-pronged investigation. We will also provide new research, new data and new data on trans-spermatic mechanisms in developing breast cancer. In this effort, we will establish and test new trans-spermatic mechanisms that will be the focal and leading investigators of this new multi-pronged multi-pronged investigation. We will also provide new research, new data and new data on trans-spermatic mechanisms in developing breast cancer. We will also provide new research, new data and new data on trans-spermatic mechanisms in developing breast cancer. We will also provide new data and new data on trans-spermatic mechanisms that are implicated in tumor progression and growth by the role of trans-spermatic mechanisms in developing breast cancer. This will be a multi-pronged multi-pronged multi-pronged multi-pronged investigation centered on trans-spermatic mechanisms in developing breast cancer development and progression and development in terms of trans-spermatic mechanisms and trans-spermatic mechanisms that are implicated in cancer progression and development. In this effort, we will establish and test new trans-spermatic mechanisms that will be the focal and leading investigators of this new multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pronged multi-pr\n",
      "2023-05-01 20:23:29,707 - INFO - train:535 - Batch 40/13861, Loss: 3.3171, Learning Rate: 1.603849238171612e-05\n",
      "\n",
      "  0%|          | 40/13861 [01:18<14:51:45,  3.87s/it]\u001b[A2023-05-01 20:23:31,400 - INFO - train:535 - Batch 41/13861, Loss: 3.3172, Learning Rate: 1.6439454691259023e-05\n",
      "\n",
      "  0%|          | 41/13861 [01:20<12:20:42,  3.22s/it]\u001b[A2023-05-01 20:23:33,027 - INFO - train:535 - Batch 42/13861, Loss: 3.3160, Learning Rate: 1.6840417000801925e-05\n",
      "\n",
      "  0%|          | 42/13861 [01:21<10:30:52,  2.74s/it]\u001b[A2023-05-01 20:23:34,655 - INFO - train:535 - Batch 43/13861, Loss: 3.3151, Learning Rate: 1.7241379310344828e-05\n",
      "\n",
      "  0%|          | 43/13861 [01:23<9:14:08,  2.41s/it] \u001b[A2023-05-01 20:23:36,284 - INFO - train:535 - Batch 44/13861, Loss: 3.3128, Learning Rate: 1.764234161988773e-05\n",
      "\n",
      "  0%|          | 44/13861 [01:25<8:20:50,  2.17s/it]\u001b[A2023-05-01 20:23:37,978 - INFO - train:535 - Batch 45/13861, Loss: 3.3121, Learning Rate: 1.8043303929430634e-05\n",
      "\n",
      "  0%|          | 45/13861 [01:26<7:47:04,  2.03s/it]\u001b[A2023-05-01 20:23:39,607 - INFO - train:535 - Batch 46/13861, Loss: 3.3119, Learning Rate: 1.8444266238973537e-05\n",
      "\n",
      "  0%|          | 46/13861 [01:28<7:19:25,  1.91s/it]\u001b[A2023-05-01 20:23:41,235 - INFO - train:535 - Batch 47/13861, Loss: 3.3114, Learning Rate: 1.884522854851644e-05\n",
      "\n",
      "  0%|          | 47/13861 [01:30<7:00:05,  1.82s/it]\u001b[A2023-05-01 20:23:42,863 - INFO - train:535 - Batch 48/13861, Loss: 3.3102, Learning Rate: 1.9246190858059342e-05\n",
      "\n",
      "  0%|          | 48/13861 [01:31<6:46:58,  1.77s/it]\u001b[A2023-05-01 20:23:44,558 - INFO - train:535 - Batch 49/13861, Loss: 3.3094, Learning Rate: 1.9647153167602245e-05\n",
      "\n",
      "  0%|          | 49/13861 [01:33<6:41:28,  1.74s/it]\u001b[A2023-05-01 20:23:46,187 - INFO - train:535 - Batch 50/13861, Loss: 3.3073, Learning Rate: 2.0048115477145148e-05\n",
      "\n",
      "  0%|          | 50/13861 [01:35<6:33:27,  1.71s/it]\u001b[A2023-05-01 20:23:47,816 - INFO - train:535 - Batch 51/13861, Loss: 3.3070, Learning Rate: 2.0449077786688054e-05\n",
      "\n",
      "  0%|          | 51/13861 [01:36<6:27:53,  1.69s/it]\u001b[A2023-05-01 20:23:49,446 - INFO - train:535 - Batch 52/13861, Loss: 3.3063, Learning Rate: 2.0850040096230957e-05\n",
      "\n",
      "  0%|          | 52/13861 [01:38<6:24:29,  1.67s/it]\u001b[A2023-05-01 20:23:51,140 - INFO - train:535 - Batch 53/13861, Loss: 3.3067, Learning Rate: 2.1251002405773857e-05\n",
      "\n",
      "  0%|          | 53/13861 [01:40<6:25:37,  1.68s/it]\u001b[A2023-05-01 20:23:52,769 - INFO - train:535 - Batch 54/13861, Loss: 3.3060, Learning Rate: 2.165196471531676e-05\n",
      "\n",
      "  0%|          | 54/13861 [01:41<6:22:20,  1.66s/it]\u001b[A2023-05-01 20:23:54,398 - INFO - train:535 - Batch 55/13861, Loss: 3.3054, Learning Rate: 2.2052927024859662e-05\n",
      "\n",
      "  0%|          | 55/13861 [01:43<6:20:04,  1.65s/it]\u001b[A2023-05-01 20:23:56,027 - INFO - train:535 - Batch 56/13861, Loss: 3.3041, Learning Rate: 2.2453889334402565e-05\n",
      "\n",
      "  0%|          | 56/13861 [01:44<6:18:58,  1.65s/it]\u001b[A2023-05-01 20:23:57,722 - INFO - train:535 - Batch 57/13861, Loss: 3.3026, Learning Rate: 2.285485164394547e-05\n",
      "\n",
      "  0%|          | 57/13861 [01:46<6:21:48,  1.66s/it]\u001b[A2023-05-01 20:23:59,351 - INFO - train:535 - Batch 58/13861, Loss: 3.3023, Learning Rate: 2.3255813953488374e-05\n",
      "\n",
      "  0%|          | 58/13861 [01:48<6:19:39,  1.65s/it]\u001b[A2023-05-01 20:24:00,979 - INFO - train:535 - Batch 59/13861, Loss: 3.3020, Learning Rate: 2.3656776263031277e-05\n",
      "\n",
      "  0%|          | 59/13861 [01:49<6:18:07,  1.64s/it]\u001b[A2023-05-01 20:24:09,608 - INFO - train:496 - Text:\n",
      "This is an all-vol, state-of-the-art device for providing real-time and detailed clinical assessment of endocrinological and metabolic syndrome-alleviated stress and other metabolic syndrome. The device is designed to determine the metabolic metabolic syndrome component of this disorder. It can also be used to determine the effects of glucocorticoids, corticosteroids, and other therapeutic agents on the metabolic syndrome. This project is funded through National Research Foundation (NIH), the National Institute for Human Genetics (NIDH), and the Institute of Diabetes and Digestive and Human Nutrition (IDNAN). This project was supported in part by grants of NIH and by the U.S. Department of Health and Human Services. The objectives of this project are to examine metabolic syndrome-alleviated stress and insulin sensitivity for diagnosis and treatment of diabetes. The primary aim of the project is to identify the mechanisms underlying and to design the metabolic syndrome component of this disorder. The endocrinology and metabolic syndrome component is a heterogeneous subtype. This disorder is characterized by a distinct genetic component. The main objective of this project is to identify the genetic components of metabolic syndrome (srs) and find out the mechanisms associated with these events. For this project, the primary goal of this project is to determine the biochemical component of Srs with Srs (or Srs syndrome A) and to determine the biological mechanisms of Srs with Srs (or Srs syndrome B). The project has three main aims: 1. The first aims are to determine the extent to which Srs and Srs A and/or B can cause systemic or systemic metabolic syndrome symptoms. Second aims are to determine the functionalities of Srs and Srs A and/or B and to determine the role of Srs in Srs A and/or B and the role of Srs in Srs B. Third goals are to determine if Srs with Srs A and/or B can cause chronic diabetes or obesity. The first two goals are to determine if Srs with Srs A and/or B can be associated with systemic and systemic metabolic syndrome symptoms. The third goal is to determine whether Srs and Srs A and/or B and Srs A and/or B can cause chronic kidney disease. The second goal is to determine whether Srs and Srs A and/or B can cause chronic diabetes or obesity. The third goal is to determine if Srs with Srs A and/or B can cause systemic or systemic metabolic syndrome symptoms. The research objectives are: 1. To determine the molecular structure and activity of the Srs and Srs A and/or B components. 2. To determine the molecular mechanisms of Srs and Srs A and/or B and their associations with systemic and systemic metabolic syndrome symptoms. 3. To determine whether Srs with Srs A and/or B can cause chronic kidney disease. 4. To determine if Srs with Srs A and/or B can have a metabolic syndrome or metabolic syndrome-alleviated stress and that Srs A and/or B have metabolic syndrome-alleviated stress and that Srs A and/or B can have metabolic syndrome-alleviated stress and that Srs A and/or B can have metabolic syndrome-alleviated stress. The research objectives include: 1) To determine the molecular structures and activity of the Srs and Srs A and/or B components. 2. To determine whether Srs with Srs A and/or B can cause systemic and systemic metabolic syndrome symptoms. 3. To determine if Srs with Srs A and/or B can cause chronic kidney disease. The objectives include: 1) Find out the mechanisms underlying and to design the metabolic syndrome component of this disorder. 2. To establish if Srs A and/or B and Srs A and/or B and Srs A and/or B and Srs A and/or B can have metabolic syndrome-alleviated stress and that Srs A and/or B can have metabolic syndrome-alleviated stress and that Srs A and/or B can have metabolic syndrome-alleviated stress. 2. To determine if Srs with Srs A and/or B can cause chronic kidney disease. The third goal is to determine if Srs with Srs A and/or B and Srs A and/or B and Srs A and/or B and Srs A and/or B can cause chronic kidney disease. The research objectives will be: 1) to determine if Srs and Srs A and/or B can be associated with systemic and systemic metabolic syndrome symptoms. 2) To determine whether Srs with Srs A and/or B can cause chronic kidney disease. 3. To determine whether Srs with Srs A and/or B can cause diabetes and obesity. 4. To determine if Srs with Srs A and/or B can cause cardiovascular disease. The research objectives will be: 1\n",
      "2023-05-01 20:24:10,046 - INFO - train:535 - Batch 60/13861, Loss: 3.3021, Learning Rate: 2.4057738572574176e-05\n",
      "\n",
      "  0%|          | 60/13861 [01:58<14:50:45,  3.87s/it]\u001b[A2023-05-01 20:24:11,740 - INFO - train:535 - Batch 61/13861, Loss: 3.3028, Learning Rate: 2.445870088211708e-05\n",
      "\n",
      "  0%|          | 61/13861 [02:00<12:19:51,  3.22s/it]\u001b[A2023-05-01 20:24:13,368 - INFO - train:535 - Batch 62/13861, Loss: 3.3017, Learning Rate: 2.4859663191659985e-05\n",
      "  0%|          | 61/13861 [02:02<7:40:53,  2.00s/it] \n",
      "  0%|          | 0/30 [02:03<?, ?it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-a24edaee5a5c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    590\u001b[0m     \u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Initial Text:\\n{generated_text}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    591\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 592\u001b[0;31m     \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-3-a24edaee5a5c>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, device, data_dict, start_epoch, start_iteration_number)\u001b[0m\n\u001b[1;32m    536\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    537\u001b[0m             \u001b[0;31m# Backward pass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 538\u001b[0;31m             \u001b[0mscaler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscale\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    539\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mindex\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"GRADIENT_ACCUMULATION_STEPS\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    485\u001b[0m                 \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    486\u001b[0m             )\n\u001b[0;32m--> 487\u001b[0;31m         torch.autograd.backward(\n\u001b[0m\u001b[1;32m    488\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    489\u001b[0m         )\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    198\u001b[0m     \u001b[0;31m# some Python versions print out the first line of a multi-line function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m     \u001b[0;31m# calls in the traceback and some print out the last line\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m     Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n\u001b[0m\u001b[1;32m    201\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m         allow_unreachable=True, accumulate_grad=True)  # Calls into the C++ engine to run the backward pass\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Change net ID here to use your scratch folder\n",
    "ENV = \"dev\"\n",
    "NET_ID = \"vgn2004\"\n",
    "DATA_PATH =  f\"/scratch/{NET_ID}/fine_tuning\" \n",
    "ROOT_PATH = f\"/scratch/{NET_ID}/fine_tuning/{ENV}\"\n",
    "\n",
    "# Global configurations\n",
    "config = {\n",
    "    \"DATASET_URL\": \"https://the-eye.eu/public/AI/pile_v2/data\",\n",
    "    \"DATASET_NAME\": \"NIH_ExPORTER_awarded_grant_text\",\n",
    "    \"NUM_WORKERS\": 8,\n",
    "    \"DATASET_SPLIT_RATIO\": 0.9,\n",
    "    \"PADDING_STRATEGY\": \"max_length\",\n",
    "    \"MAX_TOKENS\": 512,\n",
    "    \"MIN_GENERATION\": 512,\n",
    "    \"MODEL_NAME\": \"facebook/opt-125m\",\n",
    "    \"TOKENIZED_NAME\": \"opt_2700m_512\",\n",
    "    \"BATCH_SIZE\": 64,\n",
    "    \"NUM_EPOCHS\": 30,\n",
    "    \"LEARNING_RATE\": 5e-4,\n",
    "    \"MIN_LEARNING_RATE\": 5e-5,\n",
    "    \"EPSILON\": 1e-8,\n",
    "    \"BETAS\": (0.9,0.95),\n",
    "    \"GRADIENT_CLIP\": 1.0,\n",
    "    \"WEIGHT_DECAY\": 0.01,\n",
    "    \"DECAY_STYLE\": \"cosine\", #not used currently\n",
    "    \"WARMUP_RATIO\": 0.003,\n",
    "    \"SAMPLING_INTERVAL\": 20,\n",
    "    \"CHECKPOINTING_INTERVAL\": 100,\n",
    "    \"VALIDATION_INTERVAL\": 500,\n",
    "    \"GRADIENT_ACCUMULATION_STEPS\": 4, #TODO: need to bring this back\n",
    "    \n",
    "    \"DYNAMIC_LR\": True,\n",
    "    \"PEFT\": False,\n",
    "}\n",
    "\n",
    "from peft import LoraConfig, PeftConfig, get_peft_model \n",
    "lora_config = LoraConfig(\n",
    "    r=16,\n",
    "    lora_alpha=32,\n",
    "    target_modules=[\"q_proj\", \"v_proj\"],\n",
    "    lora_dropout=0.2,\n",
    "    bias=\"none\",\n",
    "    task_type=\"CAUSAL_LM\"\n",
    ")\n",
    "\n",
    "# Ensure that packages can be found\n",
    "import sys\n",
    "sys.path.insert(0, f\"/home/{NET_ID}/.local/lib/python3.8/site-packages\")\n",
    "\n",
    "# Ensure that GPU can be found\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "os.environ['TRANSFORMERS_NO_ADVISORY_WARNINGS'] = 'true'\n",
    "# os.environ[\"TORCHDYNAMO_DISABLE\"] = \"1\"\n",
    "\n",
    "# Setup logging\n",
    "import logging\n",
    "logger = logging.getLogger()\n",
    "logger.setLevel(logging.INFO)\n",
    "logging.basicConfig(format='%(asctime)s - %(levelname)s - %(funcName)s:%(lineno)d - %(message)s')\n",
    "\n",
    "# Packages for profiling\n",
    "import inspect\n",
    "import math\n",
    "import random\n",
    "import psutil\n",
    "from time import time\n",
    "from tqdm import tqdm\n",
    "import tqdm.notebook as tq\n",
    "from pynvml import *\n",
    "\n",
    "# Packages for data loading\n",
    "from datasets import load_dataset, load_from_disk, DatasetDict, Dataset\n",
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
    "from transformers import DataCollatorForLanguageModeling\n",
    "from transformers import Trainer, TrainingArguments\n",
    "\n",
    "# Core packages\n",
    "import torch\n",
    "torch.backends.cudnn.benchmark = True\n",
    "torch.cuda.empty_cache()\n",
    "torch.backends.cuda.enable_mem_efficient_sdp(False)\n",
    "torch.backends.cuda.enable_math_sdp(False)\n",
    "logging.info(f\"Is Flash Attention Enabled: {torch.backends.cuda.flash_sdp_enabled()}\")\n",
    "logging.info(f\"Is Mem Efficient SDP Enabled: {torch.backends.cuda.mem_efficient_sdp_enabled()}\")\n",
    "logging.info(f\"Is Math SDP Enabled: {torch.backends.cuda.math_sdp_enabled()}\")\n",
    "\n",
    "\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig\n",
    "from transformers import AdamW, get_linear_schedule_with_warmup, BitsAndBytesConfig\n",
    "quantization_config = BitsAndBytesConfig(\n",
    "    load_in_8bit=True,\n",
    "    llm_int8_skip_modules=[\"lm_head\"],\n",
    "    llm_int8_threshold=3.0\n",
    ")\n",
    "from transformers.optimization import Adafactor\n",
    "import bitsandbytes.optim as bnb_optim\n",
    "\n",
    "\n",
    "# Get GPU Utilization\n",
    "def print_gpu_utilization():\n",
    "    nvmlInit()\n",
    "    handle = nvmlDeviceGetHandleByIndex(0)\n",
    "    info = nvmlDeviceGetMemoryInfo(handle)\n",
    "    logging.info(f\"GPU memory occupied: {info.used//1024**2} MB.\")\n",
    "    \n",
    "\n",
    "# Returns RAM usage in MB\n",
    "def get_ram_usage():\n",
    "    return psutil.Process().memory_info().rss / (1024 * 1024)\n",
    "\n",
    "# Returns number of trainable parameters and percentage\n",
    "def print_trainable_parameters(model):\n",
    "        trainable_params = 0\n",
    "        all_param = 0\n",
    "        for _, param in model.named_parameters():\n",
    "            all_param += param.numel()\n",
    "            if param.requires_grad:\n",
    "                trainable_params += param.numel()\n",
    "        logger.info(\n",
    "            f\"Parameters: Trainable- {trainable_params/1e6:.2f}M|| All- {all_param/1e6:.2f}M || Trainable%- {100 * trainable_params / all_param}\"\n",
    "        )\n",
    "\n",
    "#Takes a batch of inputs and runs the tokenizer on them\n",
    "def tokenize_function(examples, tokenizer):\n",
    "    return tokenizer(\n",
    "        examples[\"text\"],\n",
    "        padding=config[\"PADDING_STRATEGY\"],\n",
    "        truncation=True,\n",
    "        max_length=config[\"MAX_TOKENS\"],\n",
    "        return_attention_mask=True\n",
    "    )\n",
    "\n",
    "# Tokenizes dataset and creates train and validation split\n",
    "def preprocess_data(dataset, tokenizer):\n",
    "    tokenized_dataset_path = f\"{DATA_PATH}/datasets/tokenized_{config['DATASET_NAME']}_{config['TOKENIZED_NAME']}\"\n",
    "    train_dataset_path = f\"{tokenized_dataset_path}_train\"\n",
    "    valid_dataset_path = f\"{tokenized_dataset_path}_valid\"\n",
    "    if os.path.exists(train_dataset_path) and os.path.exists(valid_dataset_path):\n",
    "        logger.info(f\"Loading dataset from disk...\")\n",
    "        start_time = time()\n",
    "        train_dataset = load_from_disk(train_dataset_path)\n",
    "        valid_dataset = load_from_disk(valid_dataset_path)\n",
    "        elapsed_time = time() - start_time\n",
    "        logger.info(f\"Time taken to load dataset from : {elapsed_time:.2f} seconds\")\n",
    "        return train_dataset, valid_dataset\n",
    "        \n",
    "    logger.info(f\"Tokenizing the dataset...\")\n",
    "    start_time = time()\n",
    "    try:\n",
    "        tokenized_dataset = load_from_disk(tokenized_dataset_path)\n",
    "    except Exception as e:\n",
    "        logging.error(e)\n",
    "        tokenized_dataset = dataset.map(\n",
    "            tokenize_function,\n",
    "            fn_kwargs={'tokenizer': tokenizer},\n",
    "            batched=True,\n",
    "            num_proc=8,\n",
    "            remove_columns=[\"text\", \"meta\"],\n",
    "        )\n",
    "        tokenized_dataset.save_to_disk(tokenized_dataset_path)\n",
    "\n",
    "    elapsed_time = time() - start_time\n",
    "    logger.info(f\"Time taken to tokenize the dataset: {elapsed_time:.2f} seconds\")\n",
    "\n",
    "    logger.info(f\"Splitting the dataset...\")\n",
    "    start_time = time()\n",
    "    \n",
    "    if os.path.exists(train_dataset_path) and os.path.exists(valid_dataset_path):\n",
    "        train_dataset = load_from_disk(train_dataset_path)\n",
    "        valid_dataset = load_from_disk(valid_dataset_path)\n",
    "    else:\n",
    "        train_size = int(config[\"DATASET_SPLIT_RATIO\"] * len(tokenized_dataset))\n",
    "        datasets = DatasetDict({\n",
    "            'train': Dataset.from_dict(tokenized_dataset[:train_size]),\n",
    "            'valid': Dataset.from_dict(tokenized_dataset[train_size:])\n",
    "        })\n",
    "        train_dataset = datasets['train']\n",
    "        valid_dataset = datasets['valid']\n",
    "        train_dataset.save_to_disk(train_dataset_path)\n",
    "        valid_dataset.save_to_disk(valid_dataset_path)\n",
    "    elapsed_time = time() - start_time\n",
    "    logger.info(f\"Time taken to split the datasets (or load pre-split datasets): {elapsed_time:.2f} seconds\")\n",
    "    \n",
    "    return train_dataset, valid_dataset\n",
    "\n",
    "# Creates data loaders\n",
    "def create_dataloaders(train_dataset, valid_dataset, data_collator):\n",
    "    logger.info(f\"Creating data loaders...\")\n",
    "    start_time = time()\n",
    "    train_dataloader = DataLoader(train_dataset,\n",
    "                                  sampler=RandomSampler(train_dataset),\n",
    "                                  batch_size=config[\"BATCH_SIZE\"],\n",
    "                                  num_workers=config[\"NUM_WORKERS\"],\n",
    "                                  collate_fn=data_collator,\n",
    "                                  pin_memory=True)\n",
    "    valid_dataloader = DataLoader(valid_dataset,\n",
    "                                  sampler=SequentialSampler(valid_dataset),\n",
    "                                  batch_size=config[\"BATCH_SIZE\"],\n",
    "                                  num_workers=config[\"NUM_WORKERS\"],\n",
    "                                  collate_fn=data_collator,\n",
    "                                  pin_memory=True)\n",
    "    elapsed_time = time() - start_time\n",
    "    logging.info(f\"Time taken to create data loaders: {elapsed_time:.2f} seconds\")\n",
    "    return train_dataloader, valid_dataloader\n",
    "\n",
    "# Fetches tokenizer relevant to the model\n",
    "def create_or_load_tokenizer(checkpointed_path=None):\n",
    "    if checkpointed_path:\n",
    "        tokenizer = AutoTokenizer.from_pretrained(checkpointed_path)\n",
    "    else:\n",
    "        tokenizer = AutoTokenizer.from_pretrained(config[\"MODEL_NAME\"], cache_dir=f\"{DATA_PATH}/datasets\")\n",
    "        tokenizer.pad_token = tokenizer.eos_token\n",
    "        tokenizer.padding_side = 'left'\n",
    "    return tokenizer\n",
    "\n",
    "# Data preparation\n",
    "def run_data_pipeline(tokenizer, load_from_file=False):\n",
    "    # Measure how much RAM is being used before anything runs\n",
    "    ram_usage = get_ram_usage()\n",
    "    logging.info(f\"Baseline: RAM used: {ram_usage:.2f} MB\")\n",
    "\n",
    "    # Load data, either from url or from datasets folder\n",
    "    data_file_url = f\"{config['DATASET_URL']}/{config['DATASET_NAME']}.jsonl.zst\"\n",
    "    try:\n",
    "        if load_from_file:\n",
    "            raise Exception\n",
    "        dataset = load_dataset(\"json\",\n",
    "                               data_files=data_file_url,\n",
    "                               num_proc=config[\"NUM_WORKERS\"],\n",
    "                               split=\"train\",\n",
    "                               cache_dir=f\"{DATA_PATH}/datasets\")\n",
    "    except Exception as e:\n",
    "        logging.error(e)\n",
    "        dataset = load_dataset(\"json\",\n",
    "                               data_files=f\"{DATA_PATH}/datasets/{config['DATASET_NAME']}.jsonl.zst\",\n",
    "                               num_proc=config[\"NUM_WORKERS\"],\n",
    "                               split=\"train\",\n",
    "                               cache_dir=f\"{DATA_PATH}/datasets\")\n",
    "\n",
    "    # Measurements relevant to the dataset\n",
    "    ram_usage = get_ram_usage()\n",
    "    logging.info(f\"RAM used: {ram_usage:.2f} MB\")\n",
    "    logging.info(f\"Dataset sample: {dataset[10]}\")\n",
    "    size_gb = dataset.dataset_size / (1024 ** 3)\n",
    "    logging.info(f\"Dataset size (cache file) : {size_gb:.2f} GB\")\n",
    "\n",
    "    # Fetch a tokenizer and tokenize + split the dataset\n",
    "    train_dataset, valid_dataset = preprocess_data(dataset, tokenizer)\n",
    "\n",
    "    # Create a data collator and use it to make data loaders\n",
    "    data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)\n",
    "    train_dataloader, valid_dataloader = create_dataloaders(train_dataset, valid_dataset, data_collator)\n",
    "\n",
    "    return {\n",
    "        \"TRAIN_DATASET\": train_dataset,\n",
    "        \"VALIDATION_DATASET\": valid_dataset,\n",
    "        \"TRAIN_DATALOADER\": train_dataloader,\n",
    "        \"VALIDATION_DATALOADER\": valid_dataloader,\n",
    "        \"TOKENIZER\": tokenizer\n",
    "    }\n",
    "\n",
    "#Get optimizer\n",
    "def fetch_optimizer(model):\n",
    "    # separate out all parameters to those that will and won't experience regularizing weight decay\n",
    "    decay = set()\n",
    "    no_decay = set()\n",
    "    whitelist_weight_modules = (torch.nn.Linear, )\n",
    "    blacklist_weight_modules = (torch.nn.LayerNorm, torch.nn.Embedding)\n",
    "    for mn, m in model.named_modules():\n",
    "        for pn, p in m.named_parameters():\n",
    "            fpn = '%s.%s' % (mn, pn) if mn else pn # full param name\n",
    "            if pn.endswith('bias'):\n",
    "                # all biases will not be decayed\n",
    "                no_decay.add(fpn)\n",
    "            elif pn.endswith('weight') and isinstance(m, whitelist_weight_modules):\n",
    "                # weights of whitelist modules will be weight decayed\n",
    "                decay.add(fpn)\n",
    "            elif pn.endswith('weight') and isinstance(m, blacklist_weight_modules):\n",
    "                # weights of blacklist modules will NOT be weight decayed\n",
    "                no_decay.add(fpn)\n",
    "    \n",
    "    head_layers = set(['lm_head.weight', '_orig_mod.lm_head.weight', 'base_model.model.lm_head.0.weight'])\n",
    "    decay = set([d for d in decay if d not in head_layers])\n",
    "\n",
    "    # validate that we considered every parameter\n",
    "    param_dict = {pn: p for pn, p in model.named_parameters()}\n",
    "    inter_params = decay & no_decay\n",
    "    union_params = decay | no_decay\n",
    "    assert len(inter_params) == 0, \"parameters %s made it into both decay/no_decay sets!\" % (str(inter_params), )\n",
    "    assert len(param_dict.keys() - union_params) == 0, \"parameters %s were not separated into either decay/no_decay set!\" \\\n",
    "                                                % (str(param_dict.keys() - union_params), )\n",
    "\n",
    "    # create the pytorch optimizer object\n",
    "    optim_groups = [\n",
    "        {\"params\": [param_dict[pn] for pn in sorted(list(decay))], \"weight_decay\": config[\"WEIGHT_DECAY\"]},\n",
    "        {\"params\": [param_dict[pn] for pn in sorted(list(no_decay))], \"weight_decay\": 0.0},\n",
    "    ]\n",
    "    \n",
    "    if(config[\"PEFT\"]):\n",
    "        optimizer = bnb_optim.AdamW(optim_groups, lr=config[\"LEARNING_RATE\"], betas=config[\"BETAS\"], weight_decay=config[\"WEIGHT_DECAY\"], optim_bits=8)\n",
    "        manager = bnb_optim.GlobalOptimManager.get_instance()\n",
    "\n",
    "        skipped = 0\n",
    "        for module in model.modules():\n",
    "            if isinstance(module, torch.nn.Embedding):\n",
    "                skipped += sum({p.data_ptr(): p.numel() for p in module.parameters()}.values())\n",
    "                manager.register_module_override(module, \"weight\", {\"optim_bits\": 32})\n",
    "                logger.info(f\"bitsandbytes: will optimize {module} in fp32\")\n",
    "        logger.info(f\"Quantizing: Skipped: {skipped/2**20}M params\")\n",
    "    else:\n",
    "        # new PyTorch nightly has a new 'fused' option for AdamW that is much faster, only works for floating point values\n",
    "        use_fused = 'fused' in inspect.signature(torch.optim.AdamW).parameters\n",
    "        logger.info(f\"Using fused AdamW: {use_fused}\")\n",
    "        fused_arg_dict = dict(fused=True) if use_fused else dict()\n",
    "        optimizer = torch.optim.AdamW(optim_groups, lr=config[\"LEARNING_RATE\"], betas=config[\"BETAS\"], weight_decay=config[\"WEIGHT_DECAY\"], **fused_arg_dict)\n",
    "\n",
    "    return optimizer\n",
    "\n",
    "# Get learning rate per iteration\n",
    "def get_lr(it, max_iters):\n",
    "    warmup_iters = int(config[\"WARMUP_RATIO\"]*max_iters)\n",
    "    if it < warmup_iters:\n",
    "        return config[\"LEARNING_RATE\"] * it / warmup_iters\n",
    "    if it > max_iters:\n",
    "        return config[\"MIN_LEARNING_RATE\"]\n",
    "    \n",
    "    #Cosine decay after warmup phase is over\n",
    "    decay_ratio = (it - warmup_iters) / (max_iters - warmup_iters)\n",
    "    coeff = 0.5 * (1.0 + math.cos(math.pi * decay_ratio))\n",
    "    return config[\"MIN_LEARNING_RATE\"] + coeff * (config[\"LEARNING_RATE\"] - config[\"MIN_LEARNING_RATE\"])\n",
    "\n",
    "\n",
    "# Create model\n",
    "def create_or_load_model(checkpointed_path=None, quantized=config[\"PEFT\"], frozen=False, cast_layer_norm_to_fp32=False, cast_output_to_fp32=False):\n",
    "    class CastOutputToFloat(torch.nn.Sequential):\n",
    "        def forward(self, x): return super().forward(x).to(torch.float32)\n",
    "    \n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "    if checkpointed_path:\n",
    "        model = AutoModelForCausalLM.from_pretrained(checkpointed_path)\n",
    "        model.to(device)\n",
    "    else:\n",
    "        configuration = AutoConfig.from_pretrained(config[\"MODEL_NAME\"])\n",
    "        \n",
    "        if quantized:\n",
    "             model = AutoModelForCausalLM.from_pretrained(config[\"MODEL_NAME\"], config=configuration, load_in_8bit=True, device_map='auto', quantization_config=quantization_config)\n",
    "        else:\n",
    "            model = AutoModelForCausalLM.from_pretrained(config[\"MODEL_NAME\"], config=configuration)\n",
    "            model.to(device)\n",
    "            \n",
    "        if frozen:\n",
    "            for param in model.parameters():\n",
    "                param.requires_grad = False\n",
    "                \n",
    "        if cast_layer_norm_to_fp32:\n",
    "            for param in model.parameters():\n",
    "                if param.ndim == 1:\n",
    "                    param.data = param.data.to(torch.float32)\n",
    "                \n",
    "    #Enable gradient checkpointing\n",
    "    model.gradient_checkpointing_enable()\n",
    "    model.enable_input_require_grads()\n",
    "    \n",
    "    if cast_output_to_fp32:\n",
    "        model.lm_head = CastOutputToFloat(model.lm_head)\n",
    "    \n",
    "    # Log details\n",
    "    logger.info(f\"Model: {config['MODEL_NAME']}\")\n",
    "    print_trainable_parameters(model)\n",
    "    logger.info(f\"Memory Memory Footprint: {model.get_memory_footprint() / 1e6:,} MB\")\n",
    "    logger.info(f\"Model is on device: {model.device}\")\n",
    "    \n",
    "    model.config.use_cache = False\n",
    "    return model, device\n",
    "\n",
    "# Use the model to generate text\n",
    "def generate(model, inputs):\n",
    "    output_sequence = model.generate(\n",
    "        **inputs,\n",
    "        bos_token_id=tokenizer.bos_token_id,\n",
    "        pad_token_id=tokenizer.eos_token_id,\n",
    "        do_sample=True,\n",
    "        top_k=50,\n",
    "        min_length=config[\"MIN_GENERATION\"],\n",
    "        max_length=2*config[\"MIN_GENERATION\"],\n",
    "        top_p=0.95,\n",
    "        num_return_sequences=1\n",
    "    )\n",
    "    return output_sequence\n",
    "    \n",
    "def inference(model, tokenizer, device, quantized=config[\"PEFT\"]):\n",
    "    # Put the model in eval mode and enable caching\n",
    "    model.config.use_cache = True\n",
    "    model.eval()\n",
    "    \n",
    "    inputs = tokenizer(tokenizer.eos_token+\"This is\", return_tensors=\"pt\").to(device)\n",
    "    # Generate a sequence of text tokens\n",
    "    with torch.no_grad():\n",
    "        if quantized:\n",
    "            with torch.cuda.amp.autocast():\n",
    "                output_sequence = generate(model, inputs)\n",
    "        else:\n",
    "            output_sequence = generate(model, inputs)\n",
    "        \n",
    "\n",
    "    # Decode the tokens to text\n",
    "    generated_text = tokenizer.decode(output_sequence[0], \n",
    "                                      skip_special_tokens=True).replace('\\n', '').replace('\\t', ' ')\n",
    "\n",
    "    # Put the model back into train mode and disable caching\n",
    "    model.train()\n",
    "    model.config.use_cache = False\n",
    "    \n",
    "    return generated_text\n",
    "\n",
    "# Evaluate the model on a data loader\n",
    "def validate(model, device, valid_dataloader):\n",
    "    model.eval()\n",
    "    total_eval_loss = 0.0\n",
    "    counter = 0\n",
    "    for index, batch in tqdm(enumerate(valid_dataloader,1)):\n",
    "        if counter<5:\n",
    "                print_gpu_utilization()\n",
    "                counter+=1\n",
    "        batch = {k: v.pin_memory().to(device, non_blocking=True) for k, v in batch.items()}\n",
    "        with torch.no_grad():\n",
    "            outputs = model(**batch)\n",
    "            loss = outputs.loss\n",
    "        total_eval_loss += loss.item()\n",
    "        avg_eval_loss = total_eval_loss / index\n",
    "        logging.info(f\"Validation: Batch {index}/{len(valid_dataloader)}, Loss: {avg_eval_loss:.4f}\")\n",
    "\n",
    "    perplexity = torch.exp(torch.as_tensor(avg_eval_loss)).item()\n",
    "    model.train()\n",
    "    return avg_eval_loss, perplexity\n",
    "\n",
    "# Train the model\n",
    "def train(model, device, data_dict, start_epoch=1, start_iteration_number=0):\n",
    "    folder = f\"fine_tuned_{config['MODEL_NAME']}_{config['DATASET_NAME']}_{config['TOKENIZED_NAME']}\"\n",
    "    model_save_path = f\"{ROOT_PATH}/models/{folder}\"\n",
    "    \n",
    "    # Setup logging\n",
    "    log_save_path = f\"{ROOT_PATH}/logs/{folder}\"\n",
    "    if not os.path.exists(log_save_path):\n",
    "        os.makedirs(log_save_path)\n",
    "    with open(f\"{log_save_path}/training.log\",\"w+\") as f:\n",
    "        f.write(\"epoch\\tbatch\\ttrain\\tloss\\tgenerated_text\\n\")\n",
    "    with open(f\"{log_save_path}/validation.log\",\"w+\") as f:\n",
    "        f.write(\"epoch\\tbatch\\tvalidation_loss\\tperplexity\\n\")\n",
    "\n",
    "    train_dataloader = data_dict[\"TRAIN_DATALOADER\"]\n",
    "    valid_dataloader = data_dict[\"VALIDATION_DATALOADER\"]\n",
    "    tokenizer = data_dict[\"TOKENIZER\"]\n",
    "\n",
    "    # Scaler for mixed precision training\n",
    "    scaler = torch.cuda.amp.GradScaler(enabled=True)\n",
    "\n",
    "    # Early stopping\n",
    "    patience = 5\n",
    "    min_loss = float(\"inf\")\n",
    "    epochs_since_min_loss = 0\n",
    "\n",
    "    \n",
    "    if config[\"DYNAMIC_LR\"]:\n",
    "        with torch.cuda.amp.autocast(dtype=torch.float16):\n",
    "            optimizer = fetch_optimizer(model)\n",
    "    else:\n",
    "        optimizer = Adafactor(model.parameters(), lr=config[\"LEARNING_RATE\"], scale_parameter=False, relative_step=False, warmup_init=False)\n",
    "    model.train()\n",
    "    \n",
    "    max_iters = len(train_dataloader)*config[\"NUM_EPOCHS\"] \n",
    "    learning_rate = config[\"LEARNING_RATE\"]\n",
    "    \n",
    "    # Go through each epoch\n",
    "    iteration_number = start_iteration_number\n",
    "    for epoch in tqdm(range(start_epoch,config[\"NUM_EPOCHS\"]+1)):\n",
    "        iteration_number_per_epoch = 0\n",
    "        running_loss = 0.0\n",
    "        logging.info(f\"Epoch: {epoch}/{config['NUM_EPOCHS']}\")\n",
    "\n",
    "        #Go through each batch in the data loader\n",
    "        for index, batch in tqdm(enumerate(train_dataloader, 1), total=len(train_dataloader)):\n",
    "            iteration_number+=1\n",
    "            optimizer.zero_grad(set_to_none=True)\n",
    "            \n",
    "            # For the initial warmup phase, keep an eye on the GPU utilization\n",
    "            if iteration_number_per_epoch<5:\n",
    "                print_gpu_utilization()\n",
    "                iteration_number_per_epoch+=1\n",
    "\n",
    "            # Sample an output from the model, at each sampling interval\n",
    "            if index%config[\"SAMPLING_INTERVAL\"]==0:\n",
    "                generated_text = inference(model, tokenizer, device)\n",
    "                logging.info(f\"Text:\\n{generated_text}\")\n",
    "                \n",
    "                with open(f\"{log_save_path}/training.log\", \"a\") as f:\n",
    "                    f.write(f\"{epoch}\\t{index}\\t{avg_loss}\\t{generated_text}\\n\")\n",
    "\n",
    "            #Save the model at each checkpointing interval\n",
    "            if index%config[\"CHECKPOINTING_INTERVAL\"]==0:\n",
    "                logging.info(f\"Checkpointing model at epoch={epoch} and batch={index}\\n\")\n",
    "\n",
    "                checkpointing_path = f\"{model_save_path}_{epoch}_{index}\"\n",
    "                model.save_pretrained(checkpointing_path)\n",
    "                tokenizer.save_pretrained(checkpointing_path)\n",
    "\n",
    "            #Validate the model at each validation interval\n",
    "            if index%config[\"VALIDATION_INTERVAL\"]==0:\n",
    "                logging.info(\"Running Validation...\")\n",
    "                avg_eval_loss, perplexity = validate(model, device, valid_dataloader)\n",
    "                logging.info(f\"Batch {index}/{len(train_dataloader)}, Validation Loss: {avg_eval_loss:.4f}, Perplexity: {perplexity:.2f}\")\n",
    "                with open(f\"{log_save_path}/validation.log\", \"a\") as f:\n",
    "                    f.write(f\"{epoch}\\t{index}\\t{avg_eval_loss}\\t{perplexity}\\n\")\n",
    "\n",
    "            #Load batches in a non-blocking manner\n",
    "            batch = {k: v.pin_memory().to(device, non_blocking=True) for k, v in batch.items()}\n",
    "            \n",
    "            #Get dynamic learning rate using cosine decay and warmup\n",
    "            if config[\"DYNAMIC_LR\"]:                \n",
    "                learning_rate = get_lr(iteration_number, max_iters)\n",
    "                for param_group in optimizer.param_groups:\n",
    "                    param_group['lr'] = learning_rate\n",
    "                    \n",
    "            #Forward pass using mixed precision training\n",
    "            with torch.cuda.amp.autocast(dtype=torch.float16):\n",
    "                outputs = model(**batch)\n",
    "                loss = outputs.loss\n",
    "                loss = loss / config[\"GRADIENT_ACCUMULATION_STEPS\"]\n",
    "\n",
    "            # Log the loss\n",
    "            running_loss += (loss.item()*config[\"GRADIENT_ACCUMULATION_STEPS\"])\n",
    "            avg_loss = running_loss / index\n",
    "            logging.info(f\"Batch {index}/{len(train_dataloader)}, Loss: {avg_loss:.4f}, Learning Rate: {learning_rate}\")\n",
    "\n",
    "            # Backward pass\n",
    "            scaler.scale(loss).backward()\n",
    "\n",
    "            if index % config[\"GRADIENT_ACCUMULATION_STEPS\"] == 0:\n",
    "                # Gradient clipping mechanism\n",
    "                if \"GRADIENT_CLIP\" in config:\n",
    "                    scaler.unscale_(optimizer)\n",
    "                    torch.nn.utils.clip_grad_norm_(model.parameters(), config[\"GRADIENT_CLIP\"])\n",
    "                    scaler.step(optimizer)\n",
    "                scaler.update()\n",
    "\n",
    "                \n",
    "        # After each epoch, check if the training loss has improved\n",
    "        if avg_loss < min_loss:\n",
    "            min_loss = avg_loss\n",
    "            epochs_since_min_loss = 0\n",
    "        else:\n",
    "            epochs_since_min_loss += 1\n",
    "\n",
    "        # Early stopping mechanism\n",
    "        if epochs_since_min_loss >= patience:\n",
    "            logger.info(\"Early stopping triggered. No improvement in training loss for {} epochs.\".format(patience))\n",
    "            break\n",
    "\n",
    "    #After all epochs are completed, save the final model and tokenier\n",
    "    model.save_pretrained(model_save_path)\n",
    "    tokenizer.save_pretrained(model_save_path)\n",
    "\n",
    "if __name__==\"__main__\":\n",
    "    print_gpu_utilization()\n",
    "    \n",
    "    checkpointed_path = None\n",
    "    tokenizer = create_or_load_tokenizer(checkpointed_path=checkpointed_path)\n",
    "    data_dict = run_data_pipeline(tokenizer, load_from_file=True)\n",
    "    \n",
    "    if config[\"PEFT\"]:\n",
    "        model, device = create_or_load_model(checkpointed_path=checkpointed_path, \n",
    "                                             frozen=True,\n",
    "                                             cast_layer_norm_to_fp32=True,\n",
    "                                             cast_output_to_fp32=True)\n",
    "        model = get_peft_model(model, lora_config)\n",
    "        model.to(device)\n",
    "        logger.info(f\"Peft Model: {config['MODEL_NAME']}\")\n",
    "        print_trainable_parameters(model)\n",
    "        print_trainable_parameters(model)\n",
    "        logger.info(f\"Memory Memory Footprint: {model.get_memory_footprint() / 1e6:,} MB\")\n",
    "        logger.info(f\"Model is on device: {model.device}\")\n",
    "    else:\n",
    "        model, device = create_or_load_model(checkpointed_path=checkpointed_path)\n",
    "#         torch._dynamo.config.verbose=True \n",
    "#         model = torch.compile(model)\n",
    "\n",
    "    generated_text = inference(model, tokenizer, device)\n",
    "    logging.info(f\"Initial Text:\\n{generated_text}\")\n",
    "\n",
    "    train(model, device, data_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}