transformer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPAShD+7suvTtsw5ntJRwir",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/galenzo17/AI-personal-test/blob/main/transformer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lCQxVUR8o_R0",
        "outputId": "4e06780d-5da9-42a9-ab67-61fdb6260965"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (1.26.4)\n",
            "(64, 38, 512)\n"
          ]
        }
      ],
      "source": [
        "# Instalación de dependencias (en este caso, solo numpy)\n",
        "!pip install numpy\n",
        "\n",
        "# Importar numpy\n",
        "import numpy as np\n",
        "\n",
        "# Definir funciones auxiliares\n",
        "def softmax(x):\n",
        "    exp_x = np.exp(x - np.max(x, axis=-1, keepdims=True))  # Evitar overflow\n",
        "    return exp_x / np.sum(exp_x, axis=-1, keepdims=True)\n",
        "\n",
        "def positional_encoding(seq_len, d_model):\n",
        "    pos = np.arange(seq_len)[:, np.newaxis]\n",
        "    i = np.arange(d_model)[np.newaxis, :]\n",
        "    angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n",
        "    angle_rads = pos * angle_rates\n",
        "    pos_encoding = np.zeros((seq_len, d_model))\n",
        "    pos_encoding[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
        "    pos_encoding[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
        "    return pos_encoding\n",
        "\n",
        "def layernorm(x, epsilon=1e-6):\n",
        "    mean = np.mean(x, axis=-1, keepdims=True)\n",
        "    std = np.std(x, axis=-1, keepdims=True)\n",
        "    return (x - mean) / (std + epsilon)\n",
        "\n",
        "class MultiHeadAttention:\n",
        "    def __init__(self, d_model, num_heads):\n",
        "        assert d_model % num_heads == 0\n",
        "        self.d_model = d_model\n",
        "        self.num_heads = num_heads\n",
        "        self.depth = d_model // num_heads\n",
        "        # Inicializar pesos\n",
        "        self.Wq = np.random.randn(d_model, d_model)\n",
        "        self.Wk = np.random.randn(d_model, d_model)\n",
        "        self.Wv = np.random.randn(d_model, d_model)\n",
        "        self.dense = np.random.randn(d_model, d_model)\n",
        "\n",
        "    def split_heads(self, x):\n",
        "        batch_size, seq_len, d_model = x.shape\n",
        "        x = x.reshape(batch_size, seq_len, self.num_heads, self.depth)\n",
        "        return np.transpose(x, (0,2,1,3))\n",
        "\n",
        "    def forward(self, v, k, q):\n",
        "        q = np.matmul(q, self.Wq)\n",
        "        k = np.matmul(k, self.Wk)\n",
        "        v = np.matmul(v, self.Wv)\n",
        "\n",
        "        q = self.split_heads(q)\n",
        "        k = self.split_heads(k)\n",
        "        v = self.split_heads(v)\n",
        "\n",
        "        matmul_qk = np.matmul(q, np.transpose(k, (0,1,3,2)))\n",
        "\n",
        "        dk = k.shape[-1]\n",
        "        scaled_attention_logits = matmul_qk / np.sqrt(dk)\n",
        "\n",
        "        attention_weights = softmax(scaled_attention_logits)\n",
        "\n",
        "        output = np.matmul(attention_weights, v)\n",
        "        output = np.transpose(output, (0,2,1,3))\n",
        "        batch_size, seq_len, num_heads, depth = output.shape\n",
        "        output = output.reshape(batch_size, seq_len, self.d_model)\n",
        "\n",
        "        output = np.matmul(output, self.dense)\n",
        "        return output\n",
        "\n",
        "class FeedForwardNetwork:\n",
        "    def __init__(self, d_model, dff):\n",
        "        self.W1 = np.random.randn(d_model, dff)\n",
        "        self.b1 = np.zeros((dff,))\n",
        "        self.W2 = np.random.randn(dff, d_model)\n",
        "        self.b2 = np.zeros((d_model,))\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = np.matmul(x, self.W1) + self.b1\n",
        "        x = np.maximum(0, x)  # ReLU\n",
        "        x = np.matmul(x, self.W2) + self.b2\n",
        "        return x\n",
        "\n",
        "class EncoderLayer:\n",
        "    def __init__(self, d_model, num_heads, dff):\n",
        "        self.mha = MultiHeadAttention(d_model, num_heads)\n",
        "        self.ffn = FeedForwardNetwork(d_model, dff)\n",
        "\n",
        "    def forward(self, x):\n",
        "        attn_output = self.mha.forward(x, x, x)\n",
        "        out1 = layernorm(x + attn_output)\n",
        "        ffn_output = self.ffn.forward(out1)\n",
        "        out2 = layernorm(out1 + ffn_output)\n",
        "        return out2\n",
        "\n",
        "class Encoder:\n",
        "    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, maximum_position_encoding):\n",
        "        self.d_model = d_model\n",
        "        self.num_layers = num_layers\n",
        "        self.embedding = np.random.randn(input_vocab_size, d_model)\n",
        "        self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n",
        "        self.enc_layers = [EncoderLayer(d_model, num_heads, dff) for _ in range(num_layers)]\n",
        "\n",
        "    def forward(self, x):\n",
        "        seq_len = x.shape[1]\n",
        "        x = self.embedding[x]  # Embedding lookup\n",
        "        x += self.pos_encoding[:seq_len, :]\n",
        "        for i in range(self.num_layers):\n",
        "            x = self.enc_layers[i].forward(x)\n",
        "        return x\n",
        "\n",
        "# Ejemplo de uso\n",
        "# Parámetros del modelo\n",
        "num_layers = 2\n",
        "d_model = 512\n",
        "num_heads = 8\n",
        "dff = 2048\n",
        "input_vocab_size = 8500\n",
        "maximum_position_encoding = 10000\n",
        "\n",
        "# Crear una instancia del codificador\n",
        "encoder = Encoder(num_layers, d_model, num_heads, dff, input_vocab_size, maximum_position_encoding)\n",
        "\n",
        "# Datos de entrada de ejemplo (batch_size, seq_len)\n",
        "batch_size = 64\n",
        "seq_len = 38\n",
        "sample_input = np.random.randint(0, input_vocab_size, (batch_size, seq_len))\n",
        "\n",
        "# Pasar los datos a través del codificador\n",
        "output = encoder.forward(sample_input)\n",
        "print(output.shape)  # Debería ser (batch_size, seq_len, d_model)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Nuevos parámetros para una prueba más pequeña\n",
        "num_layers = 1\n",
        "d_model = 128\n",
        "num_heads = 4\n",
        "dff = 512\n",
        "input_vocab_size = 50\n",
        "maximum_position_encoding = 20\n",
        "\n",
        "# Crear una nueva instancia del codificador con los nuevos parámetros\n",
        "encoder = Encoder(num_layers, d_model, num_heads, dff, input_vocab_size, maximum_position_encoding)\n",
        "\n",
        "# Datos de entrada de ejemplo (batch_size, seq_len)\n",
        "batch_size = 2\n",
        "seq_len = 5\n",
        "sample_input = np.array([\n",
        "    [10, 20, 30, 40, 0],\n",
        "    [0, 5, 15, 25, 35]\n",
        "])\n",
        "\n",
        "# Pasar los datos a través del codificador\n",
        "output = encoder.forward(sample_input)\n",
        "print(\"Forma de la salida:\", output.shape)\n",
        "print(\"Salida del codificador:\\n\", output)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q_0-x4IBq5u8",
        "outputId": "fe807be2-051e-458c-c9c2-197afe8de2c0"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Forma de la salida: (2, 5, 128)\n",
            "Salida del codificador:\n",
            " [[[-0.20356023 -1.74646801 -0.23580338 ... -1.56039245  1.36150773\n",
            "   -0.05923526]\n",
            "  [ 0.35612147 -1.0269305  -1.98589326 ... -1.01288484  0.94754422\n",
            "    1.47255032]\n",
            "  [ 0.78655874 -0.99664301 -0.30494571 ... -1.61646212  1.4039252\n",
            "   -0.47010044]\n",
            "  [-0.74792019 -1.95186556 -0.78940036 ... -2.08953442  0.98988852\n",
            "    0.34372049]\n",
            "  [ 0.67129336 -0.84356291 -0.95835354 ... -1.12661271  1.31033178\n",
            "    0.98101986]]\n",
            "\n",
            " [[ 0.2372466  -1.09898301  0.6749556  ... -0.70470468  1.24446999\n",
            "   -0.23548193]\n",
            "  [ 0.19765895 -0.91429862  0.56461194 ... -0.57561299 -0.49741797\n",
            "   -1.59695953]\n",
            "  [ 1.6760802   0.80498116  1.99575376 ... -0.32452507  0.61311849\n",
            "    0.2642539 ]\n",
            "  [ 0.44636392 -0.4083487   0.2963626  ... -1.69958446  0.27080227\n",
            "   -0.11927479]\n",
            "  [ 0.11304957 -2.1875785   0.88300197 ... -0.86553552  0.75816208\n",
            "   -0.51778891]]]\n"
          ]
        }
      ]
    }
  ]
}