LoRA_ Low-Rank Adaptation of Large Language Models.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyOehO9IRBBCbLvvNGT8ZmjA"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "[LoRA: Low-Rank Adaptation of Large Language Models](https://github.com/microsoft/LoRA)"
      ],
      "metadata": {
        "id": "oxtxvoZjQIBG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/microsoft/LoRA.git"
      ],
      "metadata": {
        "id": "ewXG32QtQOPE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pwd"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UX_s-P4mQv2b",
        "outputId": "033e01ff-6c7c-4efa-8d9a-ad4ce36e1f34"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install loralib"
      ],
      "metadata": {
        "id": "zMtAhSP7Qyr8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://github.com/microsoft/LoRA/releases/download/DeBERTa/deberta_v2_xxlarge_lora_mnli.bin\n",
        "!wget https://github.com/microsoft/LoRA/releases/download/RoBERTa-base/roberta_base_lora_mnli.bin"
      ],
      "metadata": {
        "id": "6ey4XbZxQVhO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import loralib as lora\n",
        "\n",
        "from transformers import RobertaTokenizer, RobertaModel\n",
        "tokenizer = RobertaTokenizer.from_pretrained('roberta-base')\n",
        "model = RobertaModel.from_pretrained('roberta-base')"
      ],
      "metadata": {
        "id": "_pf-ebpxOLmD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text = \"Replace me by any text you'd like.\"\n",
        "encoded_input = tokenizer(text, return_tensors='pt')\n",
        "output = model(**encoded_input)\n",
        "output"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dhue_SatRb48",
        "outputId": "57b922cb-8f8b-47c8-c7ec-f7a1dfa10b1b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BaseModelOutputWithPoolingAndCrossAttentions(last_hidden_state=tensor([[[-0.1146,  0.1103, -0.0149,  ..., -0.0809, -0.0018, -0.0271],\n",
              "         [-0.0225,  0.1612,  0.0556,  ...,  0.5366,  0.1196,  0.1576],\n",
              "         [ 0.0532, -0.0020,  0.0370,  ..., -0.4887,  0.1641,  0.2736],\n",
              "         ...,\n",
              "         [-0.1586,  0.0837,  0.1302,  ...,  0.3970,  0.1715, -0.0848],\n",
              "         [-0.1065,  0.1044, -0.0383,  ..., -0.1068, -0.0015, -0.0517],\n",
              "         [ 0.0059,  0.0758,  0.1228,  ...,  0.1037,  0.0075,  0.0976]]],\n",
              "       grad_fn=<NativeLayerNormBackward0>), pooler_output=tensor([[ 5.5806e-03, -1.1143e-01,  3.9387e-02, -2.9700e-01,  2.6594e-01,\n",
              "          2.9573e-02, -1.4606e-01,  3.4658e-01, -2.5751e-02,  3.8939e-01,\n",
              "         -3.6055e-01,  2.1485e-01,  2.7523e-04,  3.1691e-02, -7.7268e-02,\n",
              "         -5.5888e-02,  1.1882e-01, -9.8919e-02, -2.0882e-01,  1.4858e-01,\n",
              "          2.6107e-01, -2.0239e-01, -1.2397e-01, -2.4258e-01,  5.5094e-03,\n",
              "         -2.4797e-01,  7.9875e-02,  3.6167e-01,  9.7099e-02, -2.3128e-01,\n",
              "         -3.6288e-01,  2.7853e-01, -1.0869e-01,  2.5151e-01,  1.0310e-01,\n",
              "         -8.9418e-02, -6.7107e-01, -1.8805e-01,  1.1129e-01, -1.1215e-01,\n",
              "          6.7212e-02, -4.6451e-01, -3.4114e-01,  1.0889e-01, -9.4227e-02,\n",
              "         -2.0575e-01, -2.2120e-01,  7.5009e-02, -2.2444e-01, -2.0264e-01,\n",
              "         -3.0600e-01,  2.4042e-01,  4.5319e-01, -2.1942e-01, -2.1357e-01,\n",
              "          4.4326e-01,  5.4001e-02,  5.2278e-02,  7.8251e-02,  1.5719e-01,\n",
              "          1.5549e-01,  8.8703e-02,  8.7616e-02,  2.5781e-01,  1.1631e-01,\n",
              "         -2.2753e-01,  1.2469e-01,  4.9459e-02,  1.3167e-02, -1.0755e-01,\n",
              "          9.2199e-02, -2.0150e-01, -4.6757e-01, -2.7973e-02, -4.5476e-02,\n",
              "         -8.5236e-02,  2.0205e-01,  2.4524e-02, -5.0333e-02, -9.0207e-02,\n",
              "         -4.7765e-01,  2.4406e-02,  3.2937e-02,  2.2701e-02,  1.2266e-01,\n",
              "          4.4220e-01, -1.9880e-01, -3.6723e-01, -2.2939e-02, -2.3164e-01,\n",
              "         -2.8682e-01,  2.5198e-01, -1.0205e-01, -4.8803e-03,  2.5378e-01,\n",
              "         -2.7735e-01, -1.8684e-02,  5.4359e-01, -1.2962e-01, -4.2893e-02,\n",
              "          5.6766e-02,  4.0875e-01,  3.1760e-02,  1.6654e-01, -3.4552e-03,\n",
              "          3.8182e-02, -3.8551e-02,  2.4058e-01,  1.8345e-01,  3.5509e-01,\n",
              "         -4.6108e-01,  2.4763e-02,  1.5650e-01,  1.1469e-01, -1.4753e-01,\n",
              "          1.4960e-01, -1.8691e-01, -1.3045e-01, -2.0681e-01,  1.4476e-01,\n",
              "         -4.6817e-02, -2.1925e-01,  2.2166e-01, -3.0632e-01,  4.6979e-01,\n",
              "          8.3430e-02, -3.6594e-02,  1.9104e-01, -3.3529e-01, -3.9709e-01,\n",
              "          1.0730e-01,  1.0796e-01,  4.1783e-02,  1.6728e-02, -3.2049e-02,\n",
              "          1.0368e-01,  2.7347e-01, -1.0417e-01,  3.8145e-01,  1.3797e-02,\n",
              "         -3.0860e-01,  1.3239e-01,  1.1548e-01, -2.2723e-01,  1.3807e-01,\n",
              "          1.2740e-01, -2.7027e-01,  1.7858e-02, -2.1657e-01, -2.3386e-02,\n",
              "         -1.3504e-01,  1.0305e-01, -2.9831e-03, -2.4936e-01,  2.8607e-01,\n",
              "          8.7338e-02, -2.9780e-01, -2.9897e-01,  2.0113e-01,  2.1712e-02,\n",
              "          3.4883e-01, -2.6070e-01,  3.1887e-01, -1.3992e-01, -6.3961e-03,\n",
              "         -3.4272e-01, -8.4653e-02,  2.0608e-01,  3.1747e-01,  1.0483e-02,\n",
              "          1.6047e-01,  1.9715e-01, -1.0814e-01,  4.3067e-01, -1.1318e-01,\n",
              "         -3.1822e-01,  5.3925e-01, -3.4390e-01,  3.6848e-02, -1.4028e-01,\n",
              "          5.1975e-02,  2.4326e-01,  1.7542e-01, -1.7289e-01, -4.3035e-01,\n",
              "         -2.8736e-01,  4.5705e-02, -1.0462e-01,  3.6911e-01,  1.8248e-01,\n",
              "         -2.2375e-01, -3.1289e-01, -2.3069e-01,  2.3305e-01,  5.8566e-02,\n",
              "          5.9574e-02, -1.8248e-01, -4.0859e-01, -1.1523e-01, -2.6787e-01,\n",
              "         -7.1488e-02, -1.4821e-01,  1.0919e-01,  4.4420e-02,  3.8325e-01,\n",
              "          1.1943e-01, -2.7543e-01, -2.5440e-02, -2.6118e-01,  2.6953e-01,\n",
              "         -3.5997e-01,  2.9982e-01,  1.5812e-01,  1.7514e-01, -5.4812e-01,\n",
              "         -1.5066e-01, -2.3366e-01, -4.5444e-02, -7.8351e-02,  2.8247e-01,\n",
              "          1.4664e-01,  1.0134e-02,  5.1923e-02,  4.7200e-03,  1.8480e-01,\n",
              "         -1.1750e-01, -1.1465e-01,  2.6069e-01, -3.6090e-01, -2.5707e-01,\n",
              "         -8.2324e-02,  2.9610e-01,  1.2576e-02, -5.9748e-02,  3.4219e-01,\n",
              "          2.9972e-01, -3.0768e-01, -1.7053e-01,  1.7219e-02,  2.0203e-01,\n",
              "          1.8816e-01, -5.2694e-02,  1.9782e-01, -4.3681e-01,  1.9428e-01,\n",
              "          6.4642e-02, -3.2399e-01,  2.7716e-02,  3.1406e-01, -2.9923e-01,\n",
              "          4.4623e-01,  4.2358e-02, -3.2236e-01, -7.7911e-02, -4.6552e-02,\n",
              "         -4.0286e-02, -7.7683e-02, -2.6406e-02,  1.5086e-01, -6.1046e-02,\n",
              "         -4.0954e-02,  2.3364e-01, -1.2442e-01,  9.8469e-02,  3.5780e-01,\n",
              "         -1.7226e-01, -6.2327e-02, -2.3195e-01,  6.1699e-02, -3.0990e-01,\n",
              "         -3.8724e-02, -1.8708e-01, -7.4185e-02,  7.6378e-02, -4.2099e-01,\n",
              "          1.5174e-01, -2.9218e-01,  6.6268e-02, -3.3502e-01,  7.3164e-02,\n",
              "         -1.0528e-01,  8.8129e-02,  2.6721e-01,  3.1220e-03,  4.4072e-01,\n",
              "         -8.9677e-02,  4.7832e-01, -5.9113e-02, -6.0650e-02, -1.3069e-01,\n",
              "         -2.4921e-01, -5.2887e-01,  6.6982e-02,  1.3563e-01,  2.6813e-01,\n",
              "          4.1435e-01,  2.5184e-02, -1.1963e-01, -5.5172e-02, -2.4039e-01,\n",
              "          2.4439e-01, -7.9360e-02,  4.3923e-02,  7.0326e-02,  1.1842e-01,\n",
              "         -4.6876e-01, -2.4053e-01, -4.6775e-01, -9.1252e-02, -2.1720e-01,\n",
              "         -2.3306e-01, -4.4074e-02,  5.8822e-02, -2.3345e-02,  4.5754e-01,\n",
              "          1.2114e-01, -8.9500e-02,  2.0978e-02,  5.9593e-01,  1.3763e-01,\n",
              "          1.1181e-01, -3.2402e-01,  2.8595e-01,  5.5430e-01,  9.0572e-02,\n",
              "          1.8534e-01, -2.6306e-01, -1.0352e-02,  4.6678e-02, -3.0984e-01,\n",
              "          1.1921e-01, -1.6511e-01, -3.6578e-01,  1.4210e-01,  1.8795e-01,\n",
              "         -3.2187e-01,  1.8033e-01, -1.6857e-02,  1.9535e-01,  3.2027e-01,\n",
              "         -2.9938e-01,  9.7572e-02, -1.5311e-01,  1.3937e-01,  6.7670e-02,\n",
              "         -2.4806e-01, -1.3256e-01, -5.8791e-03,  7.1215e-04, -1.7246e-03,\n",
              "          1.1157e-01,  9.6062e-02, -1.6265e-01, -3.5609e-01,  6.9895e-02,\n",
              "          1.2283e-01,  7.3133e-02, -5.4339e-02,  1.1410e-01,  6.5460e-03,\n",
              "         -1.9868e-01,  1.3057e-01,  2.7568e-01, -1.3955e-01, -7.1655e-02,\n",
              "          4.0340e-01,  2.1828e-01, -6.7357e-02,  2.2466e-01, -2.7883e-02,\n",
              "         -3.8311e-02,  1.5883e-01, -1.7461e-01, -7.8870e-02, -4.1686e-01,\n",
              "          1.4940e-01,  3.0292e-01, -1.7588e-01,  4.0788e-01,  9.2463e-02,\n",
              "         -2.1264e-01,  2.8084e-02,  1.6349e-01, -1.4109e-01,  2.3412e-01,\n",
              "          1.1370e-02, -6.1678e-03,  4.0573e-01, -2.0731e-02,  1.4916e-02,\n",
              "          2.3960e-01,  4.6518e-01,  2.1597e-01,  1.2306e-01, -1.7394e-02,\n",
              "          3.5239e-02,  2.1020e-02, -3.9117e-01,  2.1312e-02, -2.6024e-01,\n",
              "         -2.6745e-01,  1.9906e-01, -2.2340e-01, -8.9303e-03,  3.0010e-01,\n",
              "         -4.7627e-01,  3.0870e-01,  3.1442e-01, -1.6372e-01, -2.2796e-01,\n",
              "          5.6169e-01,  9.2761e-02, -2.1074e-01,  3.7388e-02,  2.6512e-01,\n",
              "          4.7131e-02,  2.1243e-01, -3.4580e-01, -2.1685e-02,  3.0381e-01,\n",
              "          2.4865e-01,  1.6143e-01, -7.9887e-02, -2.7476e-01, -1.5550e-01,\n",
              "         -1.2067e-01, -2.2242e-01,  2.8583e-01,  3.1701e-01,  3.0572e-01,\n",
              "         -7.4438e-03, -2.7862e-01,  9.2392e-02, -2.5021e-01, -1.1635e-01,\n",
              "          6.1055e-02, -2.1446e-01,  3.3016e-01, -2.1208e-01, -3.1097e-01,\n",
              "          2.0034e-01,  3.5503e-01, -2.1370e-01, -2.2545e-01, -1.2927e-01,\n",
              "         -2.7135e-01,  2.3396e-01, -6.0533e-02,  1.7591e-01,  8.3835e-02,\n",
              "          1.9203e-01,  2.4282e-01,  2.6439e-01,  7.2190e-02,  3.8208e-01,\n",
              "         -1.2763e-01,  2.0501e-01, -2.3160e-02, -5.2848e-02,  6.2037e-02,\n",
              "          7.4708e-02,  3.8244e-01, -5.2136e-03,  4.8411e-02, -8.0228e-02,\n",
              "         -2.0478e-01, -1.1940e-01,  2.8672e-02, -2.0677e-01,  8.7602e-02,\n",
              "         -1.1419e-01, -9.1339e-02, -3.9270e-01,  5.0946e-02, -3.6544e-01,\n",
              "          2.9951e-02, -1.7517e-03,  1.9060e-01, -6.2651e-02, -5.1859e-02,\n",
              "         -4.3782e-01,  4.7512e-01,  7.9484e-02, -1.7423e-02, -1.4335e-01,\n",
              "          8.1543e-02,  2.7280e-01,  2.0166e-01, -3.7177e-02,  7.0962e-02,\n",
              "         -4.7489e-02,  2.3146e-01, -2.8277e-01,  2.1462e-01,  4.0377e-03,\n",
              "          1.8919e-01,  4.2626e-02, -4.8317e-02,  2.5290e-01, -6.2143e-02,\n",
              "          2.1180e-01, -3.9400e-01,  3.5955e-01,  1.0372e-01, -1.7015e-01,\n",
              "         -1.3549e-01, -2.7065e-02,  7.9670e-03,  2.4956e-01,  2.1101e-01,\n",
              "         -3.3666e-01, -7.8809e-02, -7.2519e-02,  2.1024e-01, -3.5474e-02,\n",
              "          7.7231e-02, -4.0446e-02,  1.2515e-02, -3.2028e-01, -3.0176e-01,\n",
              "          1.1056e-01,  9.7770e-03, -2.3192e-01, -9.6736e-02, -1.8835e-01,\n",
              "         -8.3520e-02,  1.1298e-01, -3.4648e-02, -2.3102e-01,  2.7931e-02,\n",
              "         -3.6112e-01, -1.6620e-01, -7.4219e-02,  2.5882e-01, -5.6247e-01,\n",
              "         -2.2906e-02, -1.2776e-01, -4.4504e-01,  2.1557e-01,  2.4155e-01,\n",
              "         -1.7156e-01,  3.6531e-01, -2.0399e-01, -1.9080e-01,  1.6099e-03,\n",
              "          1.9557e-01,  1.7001e-01, -1.1718e-01,  1.5428e-01, -2.4999e-01,\n",
              "         -2.4877e-01,  1.8821e-02, -1.7590e-01, -2.8068e-01,  1.2952e-01,\n",
              "         -2.3168e-01,  1.6648e-01,  6.2899e-02, -2.7505e-01, -1.6033e-01,\n",
              "         -1.3223e-01, -2.4271e-01, -1.4411e-01,  3.1250e-01, -1.6398e-01,\n",
              "          2.2427e-01,  1.8800e-01, -1.5871e-01,  3.9190e-02,  4.8587e-02,\n",
              "          1.0665e-01,  3.2336e-01, -3.2659e-01, -1.7915e-01,  7.4404e-02,\n",
              "         -2.6602e-01, -1.8667e-01,  1.6385e-01,  1.0416e-01,  1.0396e-01,\n",
              "          2.2383e-02,  5.0197e-02,  2.5871e-01,  4.3621e-02,  3.9873e-01,\n",
              "          1.9428e-01, -6.8950e-02,  2.8438e-02, -3.2864e-01,  1.9577e-01,\n",
              "          1.2042e-01, -2.0417e-02,  3.8652e-01,  2.2201e-01, -8.3740e-03,\n",
              "          1.9647e-01, -2.7525e-01,  3.4296e-01, -7.9273e-03,  1.3787e-01,\n",
              "         -1.2691e-02,  2.2809e-01,  1.1130e-01,  4.0143e-02, -5.3090e-01,\n",
              "          3.9092e-03,  1.0425e-01,  2.5896e-01, -7.0231e-02,  1.0386e-01,\n",
              "          1.3721e-02,  2.3712e-01, -3.6937e-01, -2.4815e-01,  2.8843e-01,\n",
              "          2.3046e-01,  3.2812e-01, -2.5000e-01,  2.1286e-01, -1.6922e-01,\n",
              "          2.1451e-02, -3.0866e-02,  5.5052e-02, -2.4549e-01, -4.4286e-01,\n",
              "          4.4293e-01, -2.1648e-02,  1.2754e-01, -6.3632e-02,  3.8725e-01,\n",
              "          2.6442e-01,  2.2972e-01, -6.1603e-02, -4.1905e-01,  2.3930e-01,\n",
              "         -4.6044e-02, -1.9538e-01,  2.6934e-01,  4.0771e-01,  4.0791e-01,\n",
              "          1.7813e-01, -3.0799e-02,  1.0286e-01, -1.0471e-01,  4.2914e-01,\n",
              "          5.3437e-02, -4.9476e-02,  2.4975e-01, -3.7652e-01, -2.2129e-01,\n",
              "         -3.1179e-01, -1.2096e-02, -1.9490e-01,  1.2221e-01, -3.4089e-01,\n",
              "         -2.3982e-01, -1.5037e-01,  3.1218e-01, -4.2332e-01, -1.4028e-01,\n",
              "         -9.2540e-02, -9.3099e-02, -1.4119e-01, -1.2649e-01, -2.1639e-02,\n",
              "          1.0104e-01, -2.4781e-01,  7.8284e-02, -1.4635e-01, -1.4554e-01,\n",
              "          1.4242e-01,  2.5204e-01, -3.3286e-01, -6.9141e-02, -2.9722e-01,\n",
              "         -5.8165e-02,  5.0313e-02,  6.8621e-02,  2.6356e-01,  9.4174e-02,\n",
              "         -1.8744e-01,  8.6433e-02,  1.4853e-01, -3.4450e-01, -1.6510e-01,\n",
              "          2.2457e-01,  8.6255e-02,  1.1707e-01, -1.3846e-01, -1.0125e-01,\n",
              "          2.4152e-01,  5.8767e-03, -4.3681e-02, -1.2037e-01, -4.2842e-03,\n",
              "          1.4593e-01,  4.3438e-01,  1.3379e-01,  5.7964e-02,  2.1710e-01,\n",
              "          1.7881e-02, -1.7598e-01,  2.8741e-01, -3.1189e-01,  3.8104e-02,\n",
              "          3.1553e-01,  1.6381e-02,  1.4588e-01,  2.7625e-01,  3.8642e-01,\n",
              "          9.8975e-02,  1.4618e-01,  6.1458e-01,  2.4618e-01, -4.1255e-01,\n",
              "         -8.7381e-02,  4.2499e-02,  5.0521e-01,  4.8733e-01,  1.8327e-02,\n",
              "         -1.5984e-01, -9.7196e-02,  1.4893e-01, -2.1016e-01,  2.4164e-01,\n",
              "          6.9169e-03,  3.9222e-01, -5.3557e-01,  1.5739e-01,  2.5384e-02,\n",
              "         -1.0110e-01, -6.2432e-02,  1.8822e-01, -1.0046e-01,  2.3276e-01,\n",
              "         -1.5052e-01,  1.7892e-01, -2.2431e-01, -3.9733e-02, -3.3639e-01,\n",
              "         -4.6425e-02,  1.2229e-01, -2.0124e-01, -1.0472e-01, -7.0816e-02,\n",
              "          2.7413e-01,  5.3788e-01, -3.6903e-01,  1.6803e-01,  4.3409e-01,\n",
              "          2.2912e-01,  2.0518e-02, -7.5614e-02, -2.6787e-01, -3.0075e-01,\n",
              "         -2.5279e-01, -5.2009e-01, -5.5641e-02, -2.4164e-02,  2.2055e-01,\n",
              "          1.6942e-01,  2.1374e-01,  4.5189e-02,  2.1844e-01,  2.3853e-01,\n",
              "         -1.9705e-01, -1.9469e-01, -1.0089e-01]], grad_fn=<TanhBackward0>), hidden_states=None, past_key_values=None, attentions=None, cross_attentions=None)"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "HELLO"
      ],
      "metadata": {
        "id": "Xjmhq-KOIy2J"
      }
    }
  ]
}