pinecone_sandbox.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %conda info\n",
    "# !pip install -qU openai pinecone-client datasets\n",
    "#!pip install markdown\n",
    "#!pip install --upgrade openai "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "\n",
    "# get API key from top-right dropdown on OpenAI website\n",
    "openai.api_key = \"***\"\n",
    "embed_model = \"text-embedding-ada-002\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def complete(prompt):\n",
    "    # query text-davinci-003\n",
    "    res = openai.Completion.create(\n",
    "        engine='gpt-3.5-turbo', #'text-davinci-003',\n",
    "        prompt=prompt,\n",
    "        temperature=0,\n",
    "        max_tokens=400,\n",
    "        top_p=1,\n",
    "        frequency_penalty=0,\n",
    "        presence_penalty=0,\n",
    "        stop=None\n",
    "    )\n",
    "    return res['choices'][0]['text'].strip()\n",
    "\n",
    "def complete_gpt3_5(prompt):\n",
    "    res = openai.ChatCompletion.create(\n",
    "    model=\"gpt-3.5-turbo\",\n",
    "    messages=[\n",
    "            {\"role\": \"system\", \"content\": \"You are a helpful and clairvoyant savant and oracle who tries to best answer the users query using just the context they provide.\"},\n",
    "            {\"role\": \"user\", \"content\": prompt}\n",
    "        ]\n",
    "    )\n",
    "    return res['choices'][0]['message']['content']\n",
    "\n",
    "def complete_gpt_4(prompt):\n",
    "    res = openai.ChatCompletion.create(\n",
    "    model=\"gpt-4\",\n",
    "    messages=[\n",
    "            {\"role\": \"system\", \"content\": \"You are a helpful assistant that elaborates on the users query using only the context they provide. If the context does not provide sufficient details for you to formulate an answer you politely let them know.\"},\n",
    "            {\"role\": \"user\", \"content\": prompt}\n",
    "        ]\n",
    "    )\n",
    "    return res['choices'][0]['message']['content']\n",
    "\n",
    "\n",
    "query = \"Who was the first man to walk on the moon?\"\n",
    "import time\n",
    "start = time.time()\n",
    "complete_gpt3_5(query)\n",
    "end = time.time()\n",
    "print(start - end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re, os, hashlib, fnmatch\n",
    "\n",
    "def search_image_files(filename, directory):\n",
    "    for dirpath, dirnames, filenames in os.walk(directory):\n",
    "        for f in filenames:\n",
    "            if f == filename:\n",
    "                ext = os.path.splitext(f)[1].lower()\n",
    "                if ext in ('.png', '.jpg', '.jpeg', '.gif'):\n",
    "                    return os.path.join(dirpath, f)\n",
    "    return None\n",
    "\n",
    "def get_all_attachments_in_text(note_string):\n",
    "    regex_pattern = r\"\\[\\[.*?\\]\\]|!\\[\\[.*?\\]\\]\"\n",
    "    strip_char = r\"\\[|\\]|!\"\n",
    "    matches = [re.sub(strip_char,'',match) for match in re.findall(regex_pattern,  note_string) ]\n",
    "    return matches\n",
    "\n",
    "def remove_urls(text):\n",
    "    # Regular expression pattern for matching URLs\n",
    "    url_pattern = re.compile(r'https?://\\S+|www\\.\\S+')\n",
    "    # Remove URLs from the text\n",
    "    without_urls = re.sub(url_pattern, '', text)\n",
    "    return without_urls\n",
    "\n",
    "def remove_obsidian_links(text):\n",
    "    clean = re.compile('\\[\\[.*?\\]\\]|!\\[\\[.*?\\]\\]')\n",
    "    return re.sub(clean, '', text)\n",
    "\n",
    "def parse_markdown_file(file_path):\n",
    "    with open(file_path, 'r') as f:\n",
    "        lines = f.readlines()\n",
    "    _, filename = os.path.split(file_path)\n",
    "    # Initialize a counter to keep track of how many '---'s we've seen\n",
    "    count = 0\n",
    "    for i, line in enumerate(lines):\n",
    "        # If we've seen one '---' already and we've just seen another, return the rest of the lines\n",
    "        if line == '---\\n' and count == 1:\n",
    "            count += 1\n",
    "            lines = lines[i:]\n",
    "            break\n",
    "        # If we've just seen our first 'b', increment the count\n",
    "        elif line == '---\\n' and count == 0:\n",
    "            count += 1\n",
    "\n",
    "    print(count)\n",
    "    # insert the filename as first element\n",
    "    lines.insert(0, filename + \"\\n\")\n",
    "    return lines\n",
    "\n",
    "\n",
    "def get_markdown_files(directory):\n",
    "    markdown_files = []\n",
    "    for item in os.listdir(directory):\n",
    "        if item == \".obsidian\" or item == \"Daily\":\n",
    "            continue\n",
    "        item_path = os.path.join(directory, item)\n",
    "        if os.path.isdir(item_path):\n",
    "            markdown_files.extend(get_markdown_files(item_path))\n",
    "        elif item.endswith('.md'):\n",
    "            markdown_files.append(item_path)\n",
    "    return markdown_files\n",
    "\n",
    "def split_list(note_lines_list, word_threshold):\n",
    "    sublists = []\n",
    "    sublist = []\n",
    "    subtotal = 0\n",
    "    for sentence in note_lines_list:\n",
    "        sublist.append(sentence)\n",
    "        word_count = len(sentence.split())\n",
    "        if subtotal + word_count > word_threshold:\n",
    "            sublists.append(sublist)\n",
    "            sublist = []\n",
    "            subtotal = 0\n",
    "        subtotal += word_count\n",
    "    if sublist:\n",
    "        sublists.append(sublist)\n",
    "    return sublists\n",
    "\n",
    "\n",
    "def get_files_to_index(rootdir, searchstr='publish: true'):\n",
    "    files_to_index = []\n",
    "    for subdir, dirs, files in os.walk(rootdir):\n",
    "        for file in files:\n",
    "            if fnmatch.fnmatch(file, '*.md') == True:\n",
    "                filepath = os.path.join(subdir, file)\n",
    "                if fnmatch.fnmatch(filepath,'*/.trash/*') == False:\n",
    "                    with open(filepath, 'r') as f:\n",
    "                        if searchstr in f.read():\n",
    "                            files_to_index.append(filepath)\n",
    "    return files_to_index\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#parse_markdown_file(\"/Users/ammarh/Documents/second-brain/command-center.md\")\n",
    "parse_markdown_file(\"/Users/ammarh/Documents/second-brain/3 - Resources/ai-ml/reinforcement learning.md\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fo = [1,2,3,4,5,6,7]\n",
    "fo = fo[2:]\n",
    "print(fo)\n",
    "fo.insert(0,0)\n",
    "print(fo)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_list = get_files_to_index(\"/Users/ammarh/Documents/second-brain/\")\n",
    "len(files_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_list = get_files_to_index(\"/Users/ammarh/Documents/second-brain/\")\n",
    "# files_list = get_markdown_files(\"/Users/ammarh/Documents/test-vault/.\")\n",
    "\n",
    "VECTOR_WORD_LIMIT = 800\n",
    "\n",
    "notes_data = []\n",
    "\n",
    "for file in files_list:\n",
    "    note_lines = parse_markdown_file(file)        \n",
    "    note_text_split = split_list(note_lines, VECTOR_WORD_LIMIT)\n",
    "    for i,note_text_split_snippet in enumerate(note_text_split):\n",
    "        note_snippet = \"\".join(note_text_split_snippet)\n",
    "        notes_data.append({\n",
    "            'uuid': hashlib.sha256((file + \"_^_\" + str(i)).encode()).hexdigest(),\n",
    "            'file': file,\n",
    "            'section': i,\n",
    "            'note': note_snippet\n",
    "        })\n",
    "\n",
    "print(f\"Adding {len(notes_data)} chunks from {len(files_list)} files\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = set([n['file'] for n in notes_data])\n",
    "a = [f for f in files if \"learning\" in f]\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import pinecone\n",
    "\n",
    "index_name = 'obsidian-second-brain'\n",
    "\n",
    "# initialize connection to pinecone (get API key at app.pinecone.io)\n",
    "pinecone.init(\n",
    "    api_key=\"***\",\n",
    "    environment=\"us-east1-gcp\"\n",
    ")\n",
    "\n",
    "# check if index already exists (it shouldn't if this is first time)\n",
    "if index_name not in pinecone.list_indexes():\n",
    "    print(\"creating new index\")\n",
    "    # if does not exist, create index\n",
    "    pinecone.create_index(\n",
    "        index_name,\n",
    "        dimension=1536,\n",
    "        metric='cosine',\n",
    "        metadata_config={'indexed': ['file']}\n",
    "    )\n",
    "# connect to index\n",
    "index = pinecone.Index(index_name)\n",
    "# view index stats\n",
    "# index.describe_index_stats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "from time import sleep\n",
    "\n",
    "batch_size = 100  # how many embeddings we create and insert at once\n",
    "\n",
    "for i in tqdm(range(0, len(notes_data), batch_size)):\n",
    "    #print(f\"i - {i}\")\n",
    "    # find end of batch\n",
    "    i_end = min(len(notes_data), i+batch_size)\n",
    "    meta_batch = notes_data[i:i_end]\n",
    "    # get ids\n",
    "    ids_batch = [x['uuid'] for x in meta_batch]\n",
    "    # get notes to encode\n",
    "    notes = [x['note'] for x in meta_batch]\n",
    "    # create embeddings (try-except added to avoid RateLimitError)\n",
    "    try:\n",
    "        res = openai.Embedding.create(input=notes, engine=embed_model)\n",
    "    except Exception as e:\n",
    "        # handle the exception by printing a message\n",
    "        print(f\"An exception occurred: \")#{repr(e)}\")\n",
    "        done = False\n",
    "        while not done:\n",
    "            #sleep(5)\n",
    "            try:\n",
    "                res = openai.Embedding.create(input=notes, engine=embed_model)\n",
    "                done = True\n",
    "            except Exception as e:\n",
    "                print(f\"Still getting an exception: {e} ... Passing\")\n",
    "                print(notes)\n",
    "                pass\n",
    "    embeds = [record['embedding'] for record in res['data']]\n",
    "    # cleanup metadata\n",
    "    meta_batch = [{\n",
    "        'uuid': x['uuid'],\n",
    "        'file': x['file'],\n",
    "        'note': x['note']\n",
    "    } for x in meta_batch]\n",
    "\n",
    "    to_upsert = list(zip(ids_batch, embeds, meta_batch))\n",
    "    # upsert to Pinecone\n",
    "    index.upsert(vectors=to_upsert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = openai.Embedding.create(\n",
    "    input=[\"how do I say hello in arabic?\"],\n",
    "    engine=embed_model\n",
    ")\n",
    "\n",
    "\n",
    "# retrieve from Pinecone\n",
    "xq = res['data'][0]['embedding']\n",
    "\n",
    "# get relevant contexts (including the questions)\n",
    "res = index.query(xq, top_k=2, include_metadata=True)\n",
    "#print(res)\n",
    "filtered_matches = [x for x in res['matches'] if x['score'] > 0]\n",
    "#print(filtered_matches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "limit = 8750\n",
    "\n",
    "def retrieve(query, confidence=0.78):\n",
    "    res = openai.Embedding.create(\n",
    "        input=[query],\n",
    "        engine=embed_model\n",
    "    )\n",
    "    # retrieve from Pinecone\n",
    "    xq = res['data'][0]['embedding']\n",
    "    # get relevant contexts\n",
    "    res = index.query(xq, top_k=20, include_metadata=True)\n",
    "    filtered_matches = [x for x in res['matches'] if x['score'] > confidence]\n",
    "    return filtered_matches\n",
    "\n",
    "def retrieve_context(query, confidence=0.78):\n",
    "    res = openai.Embedding.create(\n",
    "        input=[query],\n",
    "        engine=embed_model\n",
    "    )\n",
    "    # retrieve from Pinecone\n",
    "    xq = res['data'][0]['embedding']\n",
    "    # get relevant contexts\n",
    "    res = index.query(xq, top_k=10, include_metadata=True)\n",
    "    filtered_matches = [x for x in res['matches'] if x['score'] > confidence]\n",
    "    contexts = [\n",
    "        \"File: \" + x['metadata']['file'] + \"\\nNote: \" + x['metadata']['note'] for x in filtered_matches\n",
    "    ] # the score is a variable that should be exposed to the user \n",
    "\n",
    "    if len(contexts) == 0:\n",
    "        prompt = \"I got nothing\"\n",
    "\n",
    "    # build our prompt with the retrieved contexts included\n",
    "    prompt_start = (\n",
    "        \"Answer the question based on the context below.\\n\\n\"+\n",
    "        \"Context:\\n\"\n",
    "    )\n",
    "    prompt_end = (\n",
    "        f\"\\n\\nQuestion: {query}\\nAnswer:\"\n",
    "    )\n",
    "    # append contexts until hitting limit\n",
    "    for i in range(0, len(contexts)):\n",
    "        if len(\"\\n\\n---\\n\\n\".join(contexts[:i])) >= limit:\n",
    "            prompt = (\n",
    "                prompt_start +\n",
    "                \"\\n\\n---\\n\\n\".join(contexts[:i-1]) +\n",
    "                prompt_end\n",
    "            )\n",
    "            break\n",
    "        elif i == len(contexts)-1:\n",
    "            prompt = (\n",
    "                prompt_start +\n",
    "                \"\\n\\n---\\n\\n\".join(contexts) +\n",
    "                prompt_end\n",
    "            )\n",
    "    return (prompt, filtered_matches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fm = retrieve(\"xcs224\", 0.70)\n",
    "\n",
    "print(len(fm))\n",
    "for fms in fm:\n",
    "    print(fms['metadata']['note'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# first we retrieve relevant items from Pinecone\n",
    "query_with_contexts, filtered_matches = retrieve_context(\"what is machine learning?\", 0.7)\n",
    "# then we complete the context-infused query\n",
    "#ans = complete(query_with_contexts)\n",
    "for x in filtered_matches:\n",
    "    print(f\"{x['metadata']['file']} --- {x['metadata']['header']} --- {x['score']}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = \"---foobar\\n\\nammar\\nhusain\"\n",
    "\n",
    "my_list = [x for x in a.split(\"\\n\") if x != \"\"]\n",
    "print(my_list)\n",
    "\n",
    "if re.search(r'\\w', a):\n",
    "    print(\"The string starts is an alphanumeric character\")\n",
    "else:\n",
    "    print(\"The string does not start with an alphanumeric character\")\n",
    "\n",
    "print(re.search(r'\\w', a))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = 0\n",
    "\n",
    "if a:\n",
    "    print(\"foo\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}