cookbook/gigachat_vision/gigachat_vision.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# Работа с GigaChat Vision\n",
    "В этом ноутбуке мы рассмотрим работу с GigaChat Vision\n",
    "\n",
    "Наши действия:\n",
    "1. Загружаем фото на S3 хранилище GigaChat\n",
    "2. Просим GigaChat проанализировать фотографии, выделить сущности на них и написать описание к фото\n",
    "3. Выдать ответ в виде JSON, после чего преобразуем этот JSON в Pydantic модель\n",
    "\n",
    "Тестировать будем на этих фото\n",
    "![фото 1](cat.jpg)\n",
    "![фото 2](sea.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install langchain_gigachat langchain -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import getpass\n",
    "import os\n",
    "\n",
    "if \"GIGACHAT_CREDENTIALS\" not in os.environ:\n",
    "    os.environ[\"GIGACHAT_CREDENTIALS\"] = getpass.getpass(\"Credentials от GigaChat\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from langchain_gigachat.chat_models import GigaChat\n",
    "\n",
    "llm = GigaChat(\n",
    "    base_url=\"https://gigachat-preview.devices.sberbank.ru/api/v1\",\n",
    "    temperature=0.1,\n",
    "    verify_ssl_certs=False,\n",
    "    timeout=6000,\n",
    "    model=\"GigaChat-Pro-preview\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from typing import List, Optional\n",
    "\n",
    "from langchain.output_parsers import PydanticOutputParser\n",
    "from langchain_core.messages import HumanMessage\n",
    "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
    "from pydantic import BaseModel, Field, validator\n",
    "from langchain_core.runnables import RunnableLambda, RunnableParallel\n",
    "\n",
    "\n",
    "class Photo(BaseModel):\n",
    "    \"\"\"Информация о фото\"\"\"\n",
    "\n",
    "    content: str = Field(..., description=\"Что изображено на фото? 1-3 слова\")\n",
    "    description: str = Field(..., description=\"Опиши детальнее фото\")\n",
    "\n",
    "\n",
    "# Set up a parser\n",
    "parser = PydanticOutputParser(pydantic_object=Photo)\n",
    "\n",
    "\n",
    "def _get_messages_from_url(url: str):\n",
    "    return {\n",
    "        \"history\": [\n",
    "            HumanMessage(content=\"\", additional_kwargs={\"attachments\": [url]}),\n",
    "        ]\n",
    "    }\n",
    "\n",
    "\n",
    "# Prompt\n",
    "prompt = ChatPromptTemplate.from_messages(\n",
    "    [\n",
    "        (\n",
    "            \"system\",\n",
    "            \"Определи объект на фото. Ответь на запрос пользователя в формате JSON. Schema Information: \\n{format_instructions}\",\n",
    "        ),\n",
    "        MessagesPlaceholder(\"history\"),\n",
    "    ]\n",
    ").partial(format_instructions=parser.get_format_instructions())\n",
    "\n",
    "chain = RunnableLambda(_get_messages_from_url) | prompt | llm | parser"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "В итоге мы создали LCEL цепочку, в которую мы можем передавать id загруженных файлов и в ответ получать Pydantic модели"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Photo(content='котенок', description='белый котенок с черно-серыми пятнами на спине и хвосте'),\n",
       " Photo(content='пляж', description='На фотографии изображен каменистый пляж у подножия холма с домами и маяком. Море волнуется, создавая белые гребни волн.')]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Пример работы\n",
    "file = llm.upload_file(open(\"cat.jpg\", \"rb\"))\n",
    "file2 = llm.upload_file(open(\"sea.jpg\", \"rb\"))\n",
    "chain.batch([file.id_, file2.id_])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}