5 Gradio App for video search.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9ebb0cb4",
   "metadata": {},
   "source": [
    "# Video Search with Azure Computer Vision 4 (Florence)\n",
    "## 5 Gradio App for video search\n",
    "\n",
    "![image](logo.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4bb13529",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import glob\n",
    "import gradio as gr\n",
    "import json\n",
    "import os\n",
    "import pandas as pd\n",
    "import sys\n",
    "import time\n",
    "\n",
    "from azure import (\n",
    "    get_cosine_similarity,\n",
    "    image_embedding,\n",
    "    remove_background,\n",
    "    text_embedding,\n",
    ")\n",
    "\n",
    "from dotenv import load_dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5cac7df5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting Azure CV endpoint and key from the azure.env file\n",
    "\n",
    "load_dotenv(\"azure.env\")\n",
    "key = os.getenv(\"azure_cv_key\")\n",
    "endpoint = os.getenv(\"azure_cv_endpoint\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10fd6a08",
   "metadata": {},
   "source": [
    "## 1. Informations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "87e9c307",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.10.10 (main, Mar 21 2023, 18:45:11) [GCC 11.2.0]'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sys.version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "78e6b097",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Today is 2023-09-26 08:01:06.166806\n"
     ]
    }
   ],
   "source": [
    "print(\"Today is\", datetime.datetime.today())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "395e3c5c",
   "metadata": {},
   "source": [
    "## 2. Frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a9e90893",
   "metadata": {},
   "outputs": [],
   "source": [
    "IMAGES_DIR = \"frames\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "77de30d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Directory of images: frames\n",
      "Total number of catalog images = 1,448\n"
     ]
    }
   ],
   "source": [
    "image_files = glob.glob(IMAGES_DIR + \"/*\")\n",
    "\n",
    "print(\"Directory of images:\", IMAGES_DIR)\n",
    "print(\"Total number of catalog images =\", \"{:,}\".format(len(image_files)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec0c3ece",
   "metadata": {},
   "source": [
    "## 3. Loading vector embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e45aa4e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['json/img_embed_25Sep2023_133041.json']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "JSON_DIR = \"json\"\n",
    "\n",
    "glob.glob(JSON_DIR + \"/*.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6dddea6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing vectors embeddings...\n",
      "Loading the most recent file of the vector embeddings: json/img_embed_25Sep2023_133041.json\n",
      "\n",
      "Done: number of imported vector embeddings = 1,448\n"
     ]
    }
   ],
   "source": [
    "print(\"Importing vectors embeddings...\")\n",
    "\n",
    "jsonfiles = [entry.name for entry in os.scandir(JSON_DIR) if entry.is_file()]\n",
    "jsonfiles = [f for f in jsonfiles if os.path.isfile(os.path.join(JSON_DIR, f))]\n",
    "\n",
    "# Get the most recent file\n",
    "modification_times = [\n",
    "    (f, os.path.getmtime(os.path.join(JSON_DIR, f))) for f in jsonfiles\n",
    "]\n",
    "modification_times.sort(key=lambda x: x[1], reverse=True)\n",
    "most_recent_file = JSON_DIR + \"/\" + modification_times[0][0]\n",
    "\n",
    "# Loading the most recent file\n",
    "print(f\"Loading the most recent file of the vector embeddings: {most_recent_file}\")\n",
    "\n",
    "with open(most_recent_file) as f:\n",
    "    list_emb = json.load(f)\n",
    "\n",
    "print(f\"\\nDone: number of imported vector embeddings = {len(list_emb):,}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9de180a",
   "metadata": {},
   "source": [
    "## 4. Gradio webapp for visual search using an image"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e52b482d",
   "metadata": {},
   "source": [
    "### Generic gradio elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "023f159e",
   "metadata": {},
   "outputs": [],
   "source": [
    "footnote = \"Powered by Azure Computer Vision 4 (Florence)\"\n",
    "\n",
    "top_n = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c6deafc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "logo = \"https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/media/image-retrieval.png\"\n",
    "image = \"<center> <img src= {} width=2000px></center>\".format(logo)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f003e83",
   "metadata": {},
   "source": [
    "### Visual Search using an image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "232c9133",
   "metadata": {},
   "outputs": [],
   "source": [
    "def visual_search_from_image_app(image, list_emb=list_emb, topn=top_n):\n",
    "    \"\"\"\n",
    "    Function for visual search using an image for the gradio app\n",
    "    \"\"\"\n",
    "    # Reference image embeddding\n",
    "    nobackground_image = remove_background(image)\n",
    "    image_emb = image_embedding(nobackground_image)\n",
    "\n",
    "    # Comparing with all the images embeddings\n",
    "    results_list = [\n",
    "        get_cosine_similarity(image_emb, emb_image) for emb_image in list_emb\n",
    "    ]\n",
    "\n",
    "    # Topn results\n",
    "    df = pd.DataFrame(\n",
    "        list(zip(image_files, results_list)), columns=[\"image_file\", \"similarity\"]\n",
    "    )\n",
    "    df = df.sort_values(\"similarity\", ascending=False)\n",
    "    topn_list = df.nlargest(topn, \"similarity\")[\"image_file\"].tolist()\n",
    "    similarity_list = df.nlargest(topn, \"similarity\")[\"similarity\"].tolist()\n",
    "    \n",
    "    print(topn_list, similarity_list)\n",
    "    \n",
    "    return topn_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a9249c6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Gradio Interface for: visual_search_from_image_app\n",
       "--------------------------------------------------\n",
       "inputs:\n",
       "|-image\n",
       "outputs:\n",
       "|-image\n",
       "|-image\n",
       "|-image"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "header_image = \"Video visual search with Azure AI Vision using an image\"\n",
    "\n",
    "images_examples = [\n",
    "    \"images/image1.jpg\",\n",
    "    \"images/image2.jpg\",\n",
    "    \"images/image3.jpg\",\n",
    "    \"images/image4.jpg\",\n",
    "    \"images/image5.jpg\",\n",
    "    \"images/image6.jpg\",\n",
    "]\n",
    "\n",
    "topn_list_images = [\"\"] * top_n\n",
    "refimage = gr.components.Image(label=\"Your image:\", type=\"filepath\", shape=((200, 200)))\n",
    "\n",
    "list_img_results_prompt = [\n",
    "    gr.components.Image(\n",
    "        label=f\"Top {i+1} {topn_list_images[i]}\", type=\"filepath\", shape=((200, 200))\n",
    "    )\n",
    "    for i in range(top_n)\n",
    "]\n",
    "\n",
    "webapp_image = gr.Interface(\n",
    "    visual_search_from_image_app,\n",
    "    refimage,\n",
    "    list_img_results_prompt,\n",
    "    title=header_image,\n",
    "    examples=images_examples,\n",
    "    theme=\"gstaff/sketch\",\n",
    "    article=footnote,\n",
    "    description=image,\n",
    ")\n",
    "\n",
    "webapp_image.queue()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0519f592",
   "metadata": {},
   "source": [
    "### We can run this app"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "76eb2123",
   "metadata": {},
   "outputs": [],
   "source": [
    "# webapp_image.launch(share=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f665474",
   "metadata": {},
   "source": [
    "## 5. Gradio webapp for visual search using a prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6c1ca665",
   "metadata": {},
   "outputs": [],
   "source": [
    "def visual_search_from_prompt_app(query, list_emb=list_emb, topn=top_n):\n",
    "    \"\"\"\n",
    "    Function for visual search using a prompt for the gradio app\n",
    "    \"\"\"\n",
    "    # Text Embedding of the prompt\n",
    "    text_emb = text_embedding(query)\n",
    "\n",
    "    # Comparing the Text embedding with all the images embeddings\n",
    "    results_list = [\n",
    "        get_cosine_similarity(text_emb, emb_image) for emb_image in list_emb\n",
    "    ]\n",
    "\n",
    "    # Top5 results\n",
    "    df = pd.DataFrame(\n",
    "        list(zip(image_files, results_list)), columns=[\"image_file\", \"similarity\"]\n",
    "    )\n",
    "    df = df.sort_values(\"similarity\", ascending=False)\n",
    "    topn_list = df.nlargest(topn, \"similarity\")[\"image_file\"].tolist()\n",
    "\n",
    "    print(topn_list)\n",
    "\n",
    "    return topn_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c20c31c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Gradio Interface for: visual_search_from_prompt_app\n",
       "---------------------------------------------------\n",
       "inputs:\n",
       "|-textbox\n",
       "outputs:\n",
       "|-image\n",
       "|-image\n",
       "|-image"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "header_prompt = \"Video visual search with Azure AI Vision using a prompt\"\n",
    "\n",
    "prompt_examples = [\n",
    "    \"Dior\",\n",
    "    \"Eiffel Tower\",\n",
    "    \"Empty street\",\n",
    "    \"Love brings love exhibition\",\n",
    "    \"Métro\",\n",
    "    \"Mercedes\",\n",
    "    \"Monument with a flame\",\n",
    "    \"Padlocks\",\n",
    "    \"Palais de Tokyo\",\n",
    "    \"Palais Galliera\",\n",
    "    \"Paris Pont d'Iéna\",\n",
    "    \"Person wearing a mask\",\n",
    "    \"Person wearing an orange dress\",\n",
    "    \"Person with a bagpack\",\n",
    "    \"Pink clothes\",\n",
    "    \"Pink lines on the street\",\n",
    "]\n",
    "\n",
    "topn_list_prompt = [\"\"] * top_n\n",
    "\n",
    "prompt = gr.components.Textbox(\n",
    "    lines=1,\n",
    "    label=\"What do you want to search?\",\n",
    "    placeholder=\"Enter your prompt for the visual search and press the Submit button\",\n",
    ")\n",
    "\n",
    "labelfile = topn_list_prompt\n",
    "list_img_results_image = [\n",
    "    gr.components.Image(\n",
    "        label=f\"Top {i+1} {str(labelfile[i])} {topn_list_prompt[i]}\", type=\"filepath\"\n",
    "    )\n",
    "    for i in range(top_n)\n",
    "]\n",
    "\n",
    "webapp_prompt = gr.Interface(\n",
    "    visual_search_from_prompt_app,\n",
    "    prompt,\n",
    "    list_img_results_image,\n",
    "    title=header_prompt,\n",
    "    examples=prompt_examples,\n",
    "    theme=\"gstaff/sketch\",\n",
    "    article=footnote,\n",
    "    description=image,\n",
    ")\n",
    "\n",
    "webapp_prompt.queue()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "916f10f9",
   "metadata": {},
   "source": [
    "### We can run this app"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "99315f96",
   "metadata": {},
   "outputs": [],
   "source": [
    "# webapp_prompt.launch(share=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc466fb4",
   "metadata": {},
   "source": [
    "## 6. Gradio webapp for visual search using an image or a prompt\n",
    "### We can combine the webapps into a single one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bf851f35",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages/gradio/blocks.py:897: UserWarning: api_name predict already exists, using predict_1\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on local URL:  http://127.0.0.1:7860\n",
      "Running on public URL: https://0740e82b33304ae634.gradio.live\n",
      "\n",
      "This share link expires in 72 hours. For free permanent hosting and GPU upgrades (NEW!), check out Spaces: https://huggingface.co/spaces\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"https://0740e82b33304ae634.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['frames/frame_00_08_08.jpg', 'frames/frame_00_07_39.jpg', 'frames/frame_00_08_19.jpg']\n",
      "['frames/frame_00_08_08.jpg', 'frames/frame_00_07_39.jpg', 'frames/frame_00_08_19.jpg']\n",
      "['frames/frame_00_03_26.jpg', 'frames/frame_00_03_25.jpg', 'frames/frame_00_03_23.jpg']\n",
      "['frames/frame_00_11_09.jpg', 'frames/frame_00_11_08.jpg', 'frames/frame_00_11_10.jpg']\n",
      "['frames/frame_00_01_40.jpg', 'frames/frame_00_11_04.jpg', 'frames/frame_00_12_17.jpg']\n",
      "['frames/frame_00_14_52.jpg', 'frames/frame_00_14_51.jpg', 'frames/frame_00_14_53.jpg']\n",
      "Removing background from the image using Azure Computer Vision 4.0...\n",
      "Done\n",
      "['frames/frame_00_19_37.jpg', 'frames/frame_00_19_31.jpg', 'frames/frame_00_19_32.jpg'] [0.7566015977601326, 0.7472995177289738, 0.7273794629082129]\n",
      "Removing background from the image using Azure Computer Vision 4.0...\n",
      "Done\n",
      "['frames/frame_00_19_40.jpg', 'frames/frame_00_19_38.jpg', 'frames/frame_00_19_39.jpg'] [0.4951255185897939, 0.4717915953059321, 0.4445119369043772]\n",
      "Removing background from the image using Azure Computer Vision 4.0...\n",
      "Done\n",
      "['frames/frame_00_04_58.jpg', 'frames/frame_00_04_57.jpg', 'frames/frame_00_04_56.jpg'] [0.49518215686129813, 0.42651006787809687, 0.4157587151587754]\n",
      "Removing background from the image using Azure Computer Vision 4.0...\n",
      "Done\n",
      "['frames/frame_00_11_52.jpg', 'frames/frame_00_11_18.jpg', 'frames/frame_00_11_20.jpg'] [0.621969421107347, 0.5883592786288199, 0.5870357003930446]\n"
     ]
    }
   ],
   "source": [
    "visualsearch_webapp = gr.TabbedInterface(\n",
    "    [webapp_prompt, webapp_image],\n",
    "    [\"1 Video visual search using a prompt\", \"2 Video visual search using an image\"],\n",
    "    css=\"body {background-color: black}\",\n",
    "    theme=\"rottenlittlecreature/Moon_Goblin\",\n",
    "    # Themes: https://huggingface.co/spaces/gradio/theme-gallery\n",
    ")\n",
    "\n",
    "visualsearch_webapp.queue()\n",
    "visualsearch_webapp.launch(share=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "883113d3",
   "metadata": {},
   "source": [
    "> End"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "551e10e4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10 - SDK v2",
   "language": "python",
   "name": "python310-sdkv2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}