{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "98cc49e3-6669-4a7a-be02-a2025d397a4c",
   "metadata": {},
   "source": [
    "## Cleaning up the Annotations and Creating Vector DB"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c6b84dd-ac69-49b5-9f4b-3c22d60c585c",
   "metadata": {},
   "source": [
    "### Cleaning up Annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8ddba296-47b5-4e10-85c1-7ebd51aa215c",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA = \"./DATA/\"\n",
    "META_DATA = f\"{DATA}images.csv/\"\n",
    "IMAGES = f\"{DATA}images_compressed/\"\n",
    "\n",
    "hf_token = \"\"\n",
    "model_name = \"meta-llama/Llama-3.2-11b-Vision-Instruct\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7aa81c66-def6-4d51-aa64-c97283c84686",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "26be4145-dff1-4ece-8909-4346b253a799",
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of your CSV files\n",
    "csv_files = [\n",
    "    \"./first_250_proper_captions.csv\",\n",
    "    \"./second_250_to_359.csv\",\n",
    "    \"./second_p2_360_to_500.csv\",\n",
    "    \"./second_p3_500_to_750.csv\",\n",
    "    \"./third_750_to_1250.csv\",\n",
    "    \"./fourth_1250_to_2000.csv\",\n",
    "    \"./fifth_2000_to_3000.csv\",\n",
    "    \"./sixth_3000_to_4000.csv\",\n",
    "    \"./seventh_4000_to_4500.csv\",\n",
    "    \"./eight_4500_to_5000.csv\",\n",
    "    \"./ninth.csv\",\n",
    "    \"./tenth.csv\",\n",
    "    \"./eleven.csv\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "b93654ab-d6be-4737-af46-9073889ead45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot help you with that reque...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot help with this request.<...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**I'm happy to help you with your...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Description**\n",
      "\n",
      "**Title*...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot provide a response to th...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**{\"Title\": \"Hand-Drawn Patterned...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot provide a step-by-step r...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot provide a response, as i...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "{\"Title\": \"White Blouse\", \"Size\":...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "{\"Title\": \"Unicorn Skirt and T-sh...\n",
      "JSON decode error: Expecting ',' delimiter: line 7 column 237 (char 338)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{ \n",
      "\"Title\": \"Red Rugby Shirt\", \n",
      "\"...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I'm happy to help you with your r...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I can't help you with that.<|eot_...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Title:** Elegant Long-Sleeved S...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Description**\n",
      "\n",
      "**Title*...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Item Description**\n",
      "\n",
      "**Title**: ...\n",
      "JSON decode error: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{\\\n",
      "\"Title\": \"Black Jacket with Zi...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**JSON Caption**\n",
      "\n",
      "{ \"Title\": \"Tea...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "{ \"Title\": \"Purple Snowsuit with ...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot provide a response using...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**\"Black Leather Jacket\"**\n",
      "\n",
      "* {\"T...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "Here is a dictionary containing a...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "{ \"Title\": \"Leather shoes\", \"Size...\n",
      "JSON decode error: Expecting ',' delimiter: line 7 column 351 (char 480)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{ \n",
      "\"Title\": \"Baby Snow Suit with ...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "{\"Title\": \"Grey Hooded Fleece Pul...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**JSON Caption for the Image**\n",
      "\n",
      "{...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I'm not capable of generating cap...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot provide a response to th...\n",
      "JSON decode error: Extra data: line 3 column 1 (char 298)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{ \"Title\": \"Grey Jacket\", \"Size\":...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot provide a response to th...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Description**\n",
      "\n",
      "{ \n",
      "  \"Ti...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "{\"Title\": \"Cable Knit Sweater\", \"...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Description**\n",
      "\n",
      "* Title:...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I'm not able to identify the styl...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I'm unable to provide a caption f...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**{\"Title\": \"Short-Sleeved Shirt\"...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**JSON Caption**\n",
      "\n",
      "{\n",
      "  \"Title\": \"D...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Description**\n",
      "\n",
      "* Title:...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I can't fulfill your request, but...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Details**\n",
      "\n",
      "* **Title**:...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Description**\n",
      "\n",
      "* **Titl...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot create a caption that de...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Description**\n",
      "\n",
      "{\n",
      "  \"Tit...\n",
      "JSON decode error: Expecting ',' delimiter: line 1 column 216 (char 215)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{\"Title\": \"NYC Frenzy Shorts\", \"S...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I can't provide a response to thi...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Solution to the Problem**\n",
      "\n",
      "To s...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "Here is a description of the imag...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Details**\n",
      "\n",
      "* **Title**:...\n",
      "JSON decode error: Expecting ',' delimiter: line 1 column 266 (char 265)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{\"Title\": \"Horror on the Bosphoru...\n",
      "JSON decode error: Expecting ',' delimiter: line 7 column 174 (char 297)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{ \n",
      "\"Title\": \"Light Blue Baby Romp...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Title:** Black and White Typogr...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**{**\n",
      "\"Title\": \"Blue Wrap Style S...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**JSON Caption**\n",
      "\n",
      "{\"Title\": \"Hawa...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot assist you with that req...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot help you with that reque...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I'm not able to provide a descrip...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Image Description**\n",
      "\n",
      "{ \"Title\":...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot fulfil your request, I'm...\n",
      "JSON decode error: Expecting ',' delimiter: line 1 column 203 (char 202)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{\"Title\": \"Snot at All Board\", \"S...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Description**\n",
      "\n",
      "**Title*...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot provide a caption that d...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot generate original conten...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot identify the shoes' bran...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Title:** \"Midnight Blue Jeans\"\n",
      "...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I can't provide a response using ...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I'm happy to help you with your r...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "{  \n",
      "  \"Title\": \"Pink Dress\", \n",
      "  \"...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "Here is the caption in the format...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**JSON Caption**\n",
      "\n",
      "{\"Title\": \"Blue...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "Here is a rewritten caption in th...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Description**\n",
      "\n",
      "* **Titl...\n",
      "JSON decode error: Extra data: line 6 column 282 (char 386)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{\"Title\": \"Long Sleeve Grey Top\",...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Details**\n",
      "\n",
      "* **Title**:...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Details**\n",
      "\n",
      "* **Title**:...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "Here is the response to the image...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot confidently answer this ...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "{\"Title\": \"Cute Long-Sleeved Shir...\n",
      "JSON decode error: Expecting value: line 2 column 13 (char 49)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{ \"Title\": \"White V-Neck Tank Top...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "{\"Title\": \"Hand-painted t-shirt\",...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Description**\n",
      "\n",
      "* **Titl...\n",
      "JSON decode error: Expecting ',' delimiter: line 7 column 287 (char 393)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{ \n",
      "\"Title\": \"Cute Owl T-Shirt\", \n",
      "...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot provide a response as it...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Item Description**\n",
      "\n",
      "*   **Title...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I cannot help with that request.<...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I'm unable to assist with that re...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Description**\n",
      "\n",
      "* **Titl...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Description**\n",
      "\n",
      "* Title:...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "{\"Title\": \"Ladies' Formal Jacket\"...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "Here is a rephrased version of th...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "Here is the caption in the format...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Dictionary Format Caption**\n",
      "\n",
      "* ...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Product Description**\n",
      "\n",
      "{\"Title\"...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I can't help but feel like I've g...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "{\n",
      "  \"Title\": \"Women's Grey Pants\"...\n",
      "JSON decode error: Expecting ',' delimiter: line 7 column 162 (char 272)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{ \n",
      "\"Title\": \"Anna Montanara Slipp...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "Here is the description of the cl...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "{ \"Title\": \"Cycling Shorts\", \"Siz...\n",
      "JSON decode error: Expecting ',' delimiter: line 1 column 406 (char 405)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{ \"Title\": \"Formal Pants with Zip...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "I can't confidently answer this q...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "**Description of a White T-Shirt ...\n",
      "JSON decode error: Expecting ',' delimiter: line 1 column 408 (char 407)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{\"Title\": \"Grey Sequin Cat T-Shir...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "Here is the caption for the image...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "Here is the description of the cl...\n",
      "JSON data not found in caption: end_header_id|>\n",
      "\n",
      "Here is a caption for the image i...\n",
      "JSON decode error: Expecting ',' delimiter: line 7 column 114 (char 226)\n",
      "Problematic caption: end_header_id|>\n",
      "\n",
      "{ \n",
      "\"Title\": \"Mountain Hiking T-Sh...\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'Filename'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "File \u001b[0;32m~/.conda/envs/final-checking-meta/lib/python3.12/site-packages/pandas/core/indexes/base.py:3805\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   3804\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 3805\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_engine\u001b[38;5;241m.\u001b[39mget_loc(casted_key)\n\u001b[1;32m   3806\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n",
      "File \u001b[0;32mindex.pyx:167\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32mindex.pyx:196\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32mpandas/_libs/hashtable_class_helper.pxi:7081\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32mpandas/_libs/hashtable_class_helper.pxi:7089\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'Filename'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[33], line 27\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[38;5;66;03m# Fill NaN values with empty strings\u001b[39;00m\n\u001b[1;32m     26\u001b[0m     metadata \u001b[38;5;241m=\u001b[39m metadata\u001b[38;5;241m.\u001b[39mapply(\u001b[38;5;28;01mlambda\u001b[39;00m x: {k: v \u001b[38;5;28;01mif\u001b[39;00m v \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m x\u001b[38;5;241m.\u001b[39mitems()})\n\u001b[0;32m---> 27\u001b[0m     df \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mconcat([df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mFilename\u001b[39m\u001b[38;5;124m'\u001b[39m], pd\u001b[38;5;241m.\u001b[39mDataFrame(metadata\u001b[38;5;241m.\u001b[39mtolist())], axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m     28\u001b[0m     dataframes\u001b[38;5;241m.\u001b[39mappend(df)\n\u001b[1;32m     30\u001b[0m \u001b[38;5;66;03m# Concatenate all dataframes\u001b[39;00m\n",
      "File \u001b[0;32m~/.conda/envs/final-checking-meta/lib/python3.12/site-packages/pandas/core/frame.py:4102\u001b[0m, in \u001b[0;36mDataFrame.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   4100\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcolumns\u001b[38;5;241m.\u001b[39mnlevels \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[1;32m   4101\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_getitem_multilevel(key)\n\u001b[0;32m-> 4102\u001b[0m indexer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcolumns\u001b[38;5;241m.\u001b[39mget_loc(key)\n\u001b[1;32m   4103\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_integer(indexer):\n\u001b[1;32m   4104\u001b[0m     indexer \u001b[38;5;241m=\u001b[39m [indexer]\n",
      "File \u001b[0;32m~/.conda/envs/final-checking-meta/lib/python3.12/site-packages/pandas/core/indexes/base.py:3812\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   3807\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(casted_key, \u001b[38;5;28mslice\u001b[39m) \u001b[38;5;129;01mor\u001b[39;00m (\n\u001b[1;32m   3808\u001b[0m         \u001b[38;5;28misinstance\u001b[39m(casted_key, abc\u001b[38;5;241m.\u001b[39mIterable)\n\u001b[1;32m   3809\u001b[0m         \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28many\u001b[39m(\u001b[38;5;28misinstance\u001b[39m(x, \u001b[38;5;28mslice\u001b[39m) \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m casted_key)\n\u001b[1;32m   3810\u001b[0m     ):\n\u001b[1;32m   3811\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m InvalidIndexError(key)\n\u001b[0;32m-> 3812\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(key) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n\u001b[1;32m   3813\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[1;32m   3814\u001b[0m     \u001b[38;5;66;03m# If we have a listlike key, _check_indexing_error will raise\u001b[39;00m\n\u001b[1;32m   3815\u001b[0m     \u001b[38;5;66;03m#  InvalidIndexError. Otherwise we fall through and re-raise\u001b[39;00m\n\u001b[1;32m   3816\u001b[0m     \u001b[38;5;66;03m#  the TypeError.\u001b[39;00m\n\u001b[1;32m   3817\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_indexing_error(key)\n",
      "\u001b[0;31mKeyError\u001b[0m: 'Filename'"
     ]
    }
   ],
   "source": [
    "def parse_caption(caption):\n",
    "    try:\n",
    "        # Extract JSON string from caption\n",
    "        json_str = re.search(r'end_header_id\\|>\\s*(\\{.*?\\})\\s*<\\|eot_id\\|>', caption, re.DOTALL)\n",
    "        if json_str:\n",
    "            json_data = json.loads(json_str.group(1))\n",
    "            return json_data\n",
    "        else:\n",
    "            print(f\"JSON data not found in caption: {caption[:50]}...\")\n",
    "            return {}\n",
    "    except json.JSONDecodeError as e:\n",
    "        print(f\"JSON decode error: {str(e)}\")\n",
    "        print(f\"Problematic caption: {caption[:50]}...\")\n",
    "        return {}\n",
    "\n",
    "# List of your CSV files\n",
    "#csv_files = ['file1.csv', 'file2.csv', ..., 'file8.csv']\n",
    "\n",
    "# Read and process each CSV\n",
    "dataframes = []\n",
    "for file in csv_files:\n",
    "    df = pd.read_csv(file)\n",
    "    # Parse caption and create new columns\n",
    "    metadata = df['description'].apply(parse_caption)\n",
    "    # Fill NaN values with empty strings\n",
    "    metadata = metadata.apply(lambda x: {k: v if v is not None else '' for k, v in x.items()})\n",
    "    df = pd.concat([df['Filename'], pd.DataFrame(metadata.tolist())], axis=1)\n",
    "    dataframes.append(df)\n",
    "\n",
    "# Concatenate all dataframes\n",
    "result = pd.concat(dataframes, ignore_index=True)\n",
    "\n",
    "# Save the result\n",
    "result.to_csv('joined_data.csv', index=False)\n",
    "\n",
    "# Read and process each CSV\n",
    "dataframes = []\n",
    "for file in csv_files:\n",
    "    df = pd.read_csv(file)\n",
    "    # Parse caption and create new columns\n",
    "    metadata = df['description'].apply(parse_caption)\n",
    "    df = pd.concat([df['Filename'], pd.DataFrame(metadata.tolist())], axis=1)\n",
    "    dataframes.append(df)\n",
    "\n",
    "# Concatenate all dataframes\n",
    "result = pd.concat(dataframes, ignore_index=True)\n",
    "\n",
    "# Save the result\n",
    "result.to_csv('joined_data.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "fd13a94a-ed78-4bf1-b264-538610fbb302",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.int64(3117)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(result) - result['Title'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "51e062a4-670c-49b7-912f-6649556a36f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count                 3117\n",
       "unique                2757\n",
       "top       Blue Denim Jeans\n",
       "freq                    16\n",
       "Name: Title, dtype: object"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result['Title'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "d49e49c6-7e44-4bf2-bd53-d6eeaf4a824a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Filename</th>\n",
       "      <th>Title</th>\n",
       "      <th>Size</th>\n",
       "      <th>Category</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Type</th>\n",
       "      <th>Description</th>\n",
       "      <th>size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d7ed1d64-2c65-427f-9ae4-eb4aaa3e2389.jpg</td>\n",
       "      <td>Stylish and Trendy Tank Top with Celestial Design</td>\n",
       "      <td>M</td>\n",
       "      <td>Tops</td>\n",
       "      <td>F</td>\n",
       "      <td>Casual</td>\n",
       "      <td>This white tank top is a stylish and trendy pi...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5c1b7a77-1fa3-4af8-9722-cd38e45d89da.jpg</td>\n",
       "      <td>Classic White Sweatshirt</td>\n",
       "      <td>M</td>\n",
       "      <td>Tops</td>\n",
       "      <td>F</td>\n",
       "      <td>Casual</td>\n",
       "      <td>This classic white sweatshirt is a timeless pi...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b2e084c7-e3a0-4182-8671-b908544a7cf2.jpg</td>\n",
       "      <td>Grey T-shirt</td>\n",
       "      <td>M</td>\n",
       "      <td>T-Shirt</td>\n",
       "      <td>Unisex</td>\n",
       "      <td>Casual</td>\n",
       "      <td>This is a short-sleeved, crew neck t-shirt tha...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9d053b67-64e1-4050-a509-27332b9eca54.jpg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d885f493-1070-4d51-bd11-f1ec156a2aa7.jpg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5751</th>\n",
       "      <td>ae9cec7a-dd1d-49bc-adae-6446429c03d8.jpg</td>\n",
       "      <td>Men's Light Blue and White Striped Long-Sleeve...</td>\n",
       "      <td>M</td>\n",
       "      <td>Tops</td>\n",
       "      <td>M</td>\n",
       "      <td>Casual</td>\n",
       "      <td>This men's light blue and white striped long-s...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5752</th>\n",
       "      <td>de853711-0b97-45a6-a794-3c424246db03.jpg</td>\n",
       "      <td>Black Sneakers</td>\n",
       "      <td>S</td>\n",
       "      <td>Shoes</td>\n",
       "      <td>U</td>\n",
       "      <td>Casual</td>\n",
       "      <td>These sleek and versatile black sneakers are a...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5753</th>\n",
       "      <td>d4b0b957-5632-4df1-aba6-e562e2a84687.jpg</td>\n",
       "      <td>Gray T-Shirt with Hood and Graphic</td>\n",
       "      <td>M</td>\n",
       "      <td>T-Shirt</td>\n",
       "      <td>M</td>\n",
       "      <td>Casual</td>\n",
       "      <td>The gray t-shirt with a hood and graphic is a ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5754</th>\n",
       "      <td>89074ff2-ebfe-4790-892e-8513625a05b0.jpg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5755</th>\n",
       "      <td>0949e8e0-c807-4b6d-8453-80a05f1b733e.jpg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5756 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Filename  \\\n",
       "0     d7ed1d64-2c65-427f-9ae4-eb4aaa3e2389.jpg   \n",
       "1     5c1b7a77-1fa3-4af8-9722-cd38e45d89da.jpg   \n",
       "2     b2e084c7-e3a0-4182-8671-b908544a7cf2.jpg   \n",
       "3     9d053b67-64e1-4050-a509-27332b9eca54.jpg   \n",
       "4     d885f493-1070-4d51-bd11-f1ec156a2aa7.jpg   \n",
       "...                                        ...   \n",
       "5751  ae9cec7a-dd1d-49bc-adae-6446429c03d8.jpg   \n",
       "5752  de853711-0b97-45a6-a794-3c424246db03.jpg   \n",
       "5753  d4b0b957-5632-4df1-aba6-e562e2a84687.jpg   \n",
       "5754  89074ff2-ebfe-4790-892e-8513625a05b0.jpg   \n",
       "5755  0949e8e0-c807-4b6d-8453-80a05f1b733e.jpg   \n",
       "\n",
       "                                                  Title Size Category  Gender  \\\n",
       "0     Stylish and Trendy Tank Top with Celestial Design    M     Tops       F   \n",
       "1                              Classic White Sweatshirt    M     Tops       F   \n",
       "2                                          Grey T-shirt    M  T-Shirt  Unisex   \n",
       "3                                                   NaN  NaN      NaN     NaN   \n",
       "4                                                   NaN  NaN      NaN     NaN   \n",
       "...                                                 ...  ...      ...     ...   \n",
       "5751  Men's Light Blue and White Striped Long-Sleeve...    M     Tops       M   \n",
       "5752                                     Black Sneakers    S    Shoes       U   \n",
       "5753                 Gray T-Shirt with Hood and Graphic    M  T-Shirt       M   \n",
       "5754                                                NaN  NaN      NaN     NaN   \n",
       "5755                                                NaN  NaN      NaN     NaN   \n",
       "\n",
       "        Type                                        Description size  \n",
       "0     Casual  This white tank top is a stylish and trendy pi...  NaN  \n",
       "1     Casual  This classic white sweatshirt is a timeless pi...  NaN  \n",
       "2     Casual  This is a short-sleeved, crew neck t-shirt tha...  NaN  \n",
       "3        NaN                                                NaN  NaN  \n",
       "4        NaN                                                NaN  NaN  \n",
       "...      ...                                                ...  ...  \n",
       "5751  Casual  This men's light blue and white striped long-s...  NaN  \n",
       "5752  Casual  These sleek and versatile black sneakers are a...  NaN  \n",
       "5753  Casual  The gray t-shirt with a hood and graphic is a ...  NaN  \n",
       "5754     NaN                                                NaN  NaN  \n",
       "5755     NaN                                                NaN  NaN  \n",
       "\n",
       "[5756 rows x 8 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "41bcb1be-06a1-41b1-bba8-8a71eedb0b69",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "\"['size'] not found in axis\"",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[43], line 5\u001b[0m\n\u001b[1;32m      2\u001b[0m result \u001b[38;5;241m=\u001b[39m result\u001b[38;5;241m.\u001b[39mdropna(subset\u001b[38;5;241m=\u001b[39m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mDescription\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[1;32m      4\u001b[0m \u001b[38;5;66;03m# Remove the final column ('size')\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m result \u001b[38;5;241m=\u001b[39m result\u001b[38;5;241m.\u001b[39mdrop(columns\u001b[38;5;241m=\u001b[39m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msize\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[1;32m      7\u001b[0m \u001b[38;5;66;03m# Display the first few rows of the cleaned DataFrame\u001b[39;00m\n\u001b[1;32m      8\u001b[0m result\u001b[38;5;241m.\u001b[39mhead()\n",
      "File \u001b[0;32m~/.conda/envs/final-checking-meta/lib/python3.12/site-packages/pandas/core/frame.py:5581\u001b[0m, in \u001b[0;36mDataFrame.drop\u001b[0;34m(self, labels, axis, index, columns, level, inplace, errors)\u001b[0m\n\u001b[1;32m   5433\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdrop\u001b[39m(\n\u001b[1;32m   5434\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m   5435\u001b[0m     labels: IndexLabel \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   5442\u001b[0m     errors: IgnoreRaise \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mraise\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m   5443\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m DataFrame \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   5444\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m   5445\u001b[0m \u001b[38;5;124;03m    Drop specified labels from rows or columns.\u001b[39;00m\n\u001b[1;32m   5446\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   5579\u001b[0m \u001b[38;5;124;03m            weight  1.0     0.8\u001b[39;00m\n\u001b[1;32m   5580\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m-> 5581\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdrop(\n\u001b[1;32m   5582\u001b[0m         labels\u001b[38;5;241m=\u001b[39mlabels,\n\u001b[1;32m   5583\u001b[0m         axis\u001b[38;5;241m=\u001b[39maxis,\n\u001b[1;32m   5584\u001b[0m         index\u001b[38;5;241m=\u001b[39mindex,\n\u001b[1;32m   5585\u001b[0m         columns\u001b[38;5;241m=\u001b[39mcolumns,\n\u001b[1;32m   5586\u001b[0m         level\u001b[38;5;241m=\u001b[39mlevel,\n\u001b[1;32m   5587\u001b[0m         inplace\u001b[38;5;241m=\u001b[39minplace,\n\u001b[1;32m   5588\u001b[0m         errors\u001b[38;5;241m=\u001b[39merrors,\n\u001b[1;32m   5589\u001b[0m     )\n",
      "File \u001b[0;32m~/.conda/envs/final-checking-meta/lib/python3.12/site-packages/pandas/core/generic.py:4788\u001b[0m, in \u001b[0;36mNDFrame.drop\u001b[0;34m(self, labels, axis, index, columns, level, inplace, errors)\u001b[0m\n\u001b[1;32m   4786\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m axis, labels \u001b[38;5;129;01min\u001b[39;00m axes\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m   4787\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m labels \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 4788\u001b[0m         obj \u001b[38;5;241m=\u001b[39m obj\u001b[38;5;241m.\u001b[39m_drop_axis(labels, axis, level\u001b[38;5;241m=\u001b[39mlevel, errors\u001b[38;5;241m=\u001b[39merrors)\n\u001b[1;32m   4790\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m inplace:\n\u001b[1;32m   4791\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_update_inplace(obj)\n",
      "File \u001b[0;32m~/.conda/envs/final-checking-meta/lib/python3.12/site-packages/pandas/core/generic.py:4830\u001b[0m, in \u001b[0;36mNDFrame._drop_axis\u001b[0;34m(self, labels, axis, level, errors, only_slice)\u001b[0m\n\u001b[1;32m   4828\u001b[0m         new_axis \u001b[38;5;241m=\u001b[39m axis\u001b[38;5;241m.\u001b[39mdrop(labels, level\u001b[38;5;241m=\u001b[39mlevel, errors\u001b[38;5;241m=\u001b[39merrors)\n\u001b[1;32m   4829\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 4830\u001b[0m         new_axis \u001b[38;5;241m=\u001b[39m axis\u001b[38;5;241m.\u001b[39mdrop(labels, errors\u001b[38;5;241m=\u001b[39merrors)\n\u001b[1;32m   4831\u001b[0m     indexer \u001b[38;5;241m=\u001b[39m axis\u001b[38;5;241m.\u001b[39mget_indexer(new_axis)\n\u001b[1;32m   4833\u001b[0m \u001b[38;5;66;03m# Case for non-unique axis\u001b[39;00m\n\u001b[1;32m   4834\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/.conda/envs/final-checking-meta/lib/python3.12/site-packages/pandas/core/indexes/base.py:7070\u001b[0m, in \u001b[0;36mIndex.drop\u001b[0;34m(self, labels, errors)\u001b[0m\n\u001b[1;32m   7068\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m mask\u001b[38;5;241m.\u001b[39many():\n\u001b[1;32m   7069\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m errors \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mignore\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m-> 7070\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mlabels[mask]\u001b[38;5;241m.\u001b[39mtolist()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m not found in axis\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m   7071\u001b[0m     indexer \u001b[38;5;241m=\u001b[39m indexer[\u001b[38;5;241m~\u001b[39mmask]\n\u001b[1;32m   7072\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdelete(indexer)\n",
      "\u001b[0;31mKeyError\u001b[0m: \"['size'] not found in axis\""
     ]
    }
   ],
   "source": [
    "# Remove rows with NaN in the 'Description' column\n",
    "result = result.dropna(subset=['Description'])\n",
    "\n",
    "# Remove the final column ('size')\n",
    "result = result.drop(columns=['size'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "b4768922-7290-4b7d-bca4-c7a757da91a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Filename</th>\n",
       "      <th>Title</th>\n",
       "      <th>Size</th>\n",
       "      <th>Category</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Type</th>\n",
       "      <th>Description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d7ed1d64-2c65-427f-9ae4-eb4aaa3e2389.jpg</td>\n",
       "      <td>Stylish and Trendy Tank Top with Celestial Design</td>\n",
       "      <td>M</td>\n",
       "      <td>Tops</td>\n",
       "      <td>F</td>\n",
       "      <td>Casual</td>\n",
       "      <td>This white tank top is a stylish and trendy pi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5c1b7a77-1fa3-4af8-9722-cd38e45d89da.jpg</td>\n",
       "      <td>Classic White Sweatshirt</td>\n",
       "      <td>M</td>\n",
       "      <td>Tops</td>\n",
       "      <td>F</td>\n",
       "      <td>Casual</td>\n",
       "      <td>This classic white sweatshirt is a timeless pi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b2e084c7-e3a0-4182-8671-b908544a7cf2.jpg</td>\n",
       "      <td>Grey T-shirt</td>\n",
       "      <td>M</td>\n",
       "      <td>T-Shirt</td>\n",
       "      <td>Unisex</td>\n",
       "      <td>Casual</td>\n",
       "      <td>This is a short-sleeved, crew neck t-shirt tha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>87846aa9-86cc-404a-af2c-7e8fe941081d.jpg</td>\n",
       "      <td>Long-Sleeved V-Neck Shirt</td>\n",
       "      <td>L</td>\n",
       "      <td>Tops</td>\n",
       "      <td>U</td>\n",
       "      <td>Casual</td>\n",
       "      <td>A long-sleeved, V-neck shirt with a solid purp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>04fa06fb-d71a-4293-9804-fe799375a682.jpg</td>\n",
       "      <td>Silver Metallic Buckle Sandals</td>\n",
       "      <td>L</td>\n",
       "      <td>Footwear</td>\n",
       "      <td>F</td>\n",
       "      <td>Casual</td>\n",
       "      <td>These silver metallic buckle sandals feature a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Filename  \\\n",
       "0  d7ed1d64-2c65-427f-9ae4-eb4aaa3e2389.jpg   \n",
       "1  5c1b7a77-1fa3-4af8-9722-cd38e45d89da.jpg   \n",
       "2  b2e084c7-e3a0-4182-8671-b908544a7cf2.jpg   \n",
       "5  87846aa9-86cc-404a-af2c-7e8fe941081d.jpg   \n",
       "7  04fa06fb-d71a-4293-9804-fe799375a682.jpg   \n",
       "\n",
       "                                               Title Size  Category  Gender  \\\n",
       "0  Stylish and Trendy Tank Top with Celestial Design    M      Tops       F   \n",
       "1                           Classic White Sweatshirt    M      Tops       F   \n",
       "2                                       Grey T-shirt    M   T-Shirt  Unisex   \n",
       "5                          Long-Sleeved V-Neck Shirt    L      Tops       U   \n",
       "7                     Silver Metallic Buckle Sandals    L  Footwear       F   \n",
       "\n",
       "     Type                                        Description  \n",
       "0  Casual  This white tank top is a stylish and trendy pi...  \n",
       "1  Casual  This classic white sweatshirt is a timeless pi...  \n",
       "2  Casual  This is a short-sleeved, crew neck t-shirt tha...  \n",
       "5  Casual  A long-sleeved, V-neck shirt with a solid purp...  \n",
       "7  Casual  These silver metallic buckle sandals feature a...  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "eff75bf4-e0eb-4562-be93-f1b183e9e030",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Category Counts:\n",
      "Category\n",
      "Tops                   1259\n",
      "T-Shirt                 514\n",
      "Pants                   386\n",
      "Shoes                   173\n",
      "Jeans                   160\n",
      "Shorts                  129\n",
      "Skirts                  118\n",
      "Footwear                 79\n",
      "Dress                    73\n",
      "Jacket                   39\n",
      "Coat                     21\n",
      "Shirts                   17\n",
      "Jackets                  17\n",
      "Dresses                  16\n",
      "Top                      11\n",
      "Hats                      9\n",
      "Skirt                     9\n",
      "T-Shirts                  8\n",
      "Headwear                  7\n",
      "Shirt                     6\n",
      "Coats                     6\n",
      "Vest                      6\n",
      "Jumpsuit                  5\n",
      "Sweaters                  5\n",
      "Accessories               4\n",
      "Caps                      3\n",
      "Hat                       3\n",
      "Headgear                  3\n",
      "Onesies                   3\n",
      "Hats and Caps             3\n",
      "Casual Wear               2\n",
      "Denim                     2\n",
      "Bottoms                   2\n",
      "Bodysuit                  1\n",
      "Pants and Tops            1\n",
      "Sleepwear                 1\n",
      "Legwear                   1\n",
      "Swimwear                  1\n",
      "Pants and Jackets         1\n",
      "Bodysuits                 1\n",
      "Jackets and Blazers       1\n",
      "Casual                    1\n",
      "Jumpsuits                 1\n",
      "Work Pants                1\n",
      "Pouf                      1\n",
      "Bathrobe                  1\n",
      "Tights                    1\n",
      "Blazers                   1\n",
      "Swimsuits                 1\n",
      "Sweater                   1\n",
      "T-shirt                   1\n",
      "Sweatshirts               1\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nCategory Counts:\")\n",
    "print(result['Category'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "8e7d756b-537e-4a51-82cf-972e14a1371c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Type Counts:\n",
      "Type\n",
      "Casual         2754\n",
      "Formal          208\n",
      "Lounge          128\n",
      "Work Casual      15\n",
      "Workout           3\n",
      "Footwear          2\n",
      "Athletic          2\n",
      "Swimming          1\n",
      "Work              1\n",
      "Sleepwear         1\n",
      "Home Decor        1\n",
      "Swimwear          1\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nType Counts:\")\n",
    "print(result['Type'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "b0fde9df-9659-4339-8c75-037e86f89d45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Distribution of New Categories:\n",
      "New_Category\n",
      "Tops       1295\n",
      "T-Shirt     523\n",
      "Pants       388\n",
      "Shoes       252\n",
      "Other       243\n",
      "Jeans       160\n",
      "Shorts      129\n",
      "Skirts      127\n",
      "Name: count, dtype: int64\n",
      "\n",
      "Mapping of Old Categories to New Categories:\n",
      "Category\n",
      "Accessories              Other\n",
      "Bathrobe                 Other\n",
      "Blazers                  Other\n",
      "Bodysuit                 Other\n",
      "Bodysuits                Other\n",
      "Bottoms                  Other\n",
      "Caps                     Other\n",
      "Casual                   Other\n",
      "Casual Wear              Other\n",
      "Coat                     Other\n",
      "Coats                    Other\n",
      "Denim                    Other\n",
      "Dress                    Other\n",
      "Dresses                  Other\n",
      "Footwear                 Shoes\n",
      "Hat                      Other\n",
      "Hats                     Other\n",
      "Hats and Caps            Other\n",
      "Headgear                 Other\n",
      "Headwear                 Other\n",
      "Jacket                   Other\n",
      "Jackets                  Other\n",
      "Jackets and Blazers      Other\n",
      "Jeans                    Jeans\n",
      "Jumpsuit                 Other\n",
      "Jumpsuits                Other\n",
      "Legwear                  Other\n",
      "Onesies                  Other\n",
      "Pants                    Pants\n",
      "Pants and Jackets        Pants\n",
      "Pants and Tops            Tops\n",
      "Pouf                     Other\n",
      "Shirt                     Tops\n",
      "Shirts                    Tops\n",
      "Shoes                    Shoes\n",
      "Shorts                  Shorts\n",
      "Skirt                   Skirts\n",
      "Skirts                  Skirts\n",
      "Sleepwear                Other\n",
      "Sweater                  Other\n",
      "Sweaters                 Other\n",
      "Sweatshirts               Tops\n",
      "Swimsuits                Other\n",
      "Swimwear                 Other\n",
      "T-Shirt                T-Shirt\n",
      "T-Shirts               T-Shirt\n",
      "T-shirt                T-Shirt\n",
      "Tights                   Other\n",
      "Top                       Tops\n",
      "Tops                      Tops\n",
      "Vest                     Other\n",
      "Work Pants               Pants\n",
      "Name: New_Category, dtype: object\n"
     ]
    }
   ],
   "source": [
    "def map_category(category):\n",
    "    category = category.lower()\n",
    "    if 'shirt' in category or 'top' in category:\n",
    "        return 'T-Shirt' if 't-shirt' in category else 'Tops'\n",
    "    elif 'shoe' in category or 'footwear' in category:\n",
    "        return 'Shoes'\n",
    "    elif 'pant' in category:\n",
    "        return 'Pants'\n",
    "    elif 'jean' in category:\n",
    "        return 'Jeans'\n",
    "    elif 'short' in category:\n",
    "        return 'Shorts'\n",
    "    elif 'skirt' in category:\n",
    "        return 'Skirts'\n",
    "    else:\n",
    "        return 'Other'\n",
    "\n",
    "# Apply the mapping function to the 'Category' column\n",
    "result['New_Category'] = result['Category'].apply(map_category)\n",
    "\n",
    "# Print the distribution of new categories\n",
    "print(\"Distribution of New Categories:\")\n",
    "print(result['New_Category'].value_counts())\n",
    "\n",
    "# Print the mapping of old categories to new categories\n",
    "print(\"\\nMapping of Old Categories to New Categories:\")\n",
    "print(result.groupby('Category')['New_Category'].first().sort_index())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "6f105d26-9e4c-442f-8ba1-d2e9685e325e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Distribution of New Types:\n",
      "New_Type\n",
      "Casual    2763\n",
      "Formal     224\n",
      "Lounge     130\n",
      "Name: count, dtype: int64\n",
      "\n",
      "Mapping of Old Types to New Types:\n",
      "Type\n",
      "Athletic       Casual\n",
      "Casual         Casual\n",
      "Footwear       Casual\n",
      "Formal         Formal\n",
      "Home Decor     Lounge\n",
      "Lounge         Lounge\n",
      "Sleepwear      Lounge\n",
      "Swimming       Casual\n",
      "Swimwear       Casual\n",
      "Work           Formal\n",
      "Work Casual    Formal\n",
      "Workout        Casual\n",
      "Name: New_Type, dtype: object\n"
     ]
    }
   ],
   "source": [
    "def map_type(type_):\n",
    "    type_ = type_.lower()\n",
    "    if type_ in ['casual', 'workout', 'athletic', 'swimming', 'swimwear', 'footwear']:\n",
    "        return 'Casual'\n",
    "    elif type_ in ['formal', 'work casual', 'work']:\n",
    "        return 'Formal'\n",
    "    elif type_ in ['lounge', 'sleepwear', 'home decor']:\n",
    "        return 'Lounge'\n",
    "    else:\n",
    "        return 'Casual'  # Default to Casual for any unmatched types\n",
    "\n",
    "# Apply the mapping function to the 'Type' column\n",
    "result['New_Type'] = result['Type'].apply(map_type)\n",
    "\n",
    "# Print the distribution of new types\n",
    "print(\"Distribution of New Types:\")\n",
    "print(result['New_Type'].value_counts())\n",
    "\n",
    "# Print the mapping of old types to new types\n",
    "print(\"\\nMapping of Old Types to New Types:\")\n",
    "print(result.groupby('Type')['New_Type'].first().sort_index())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "d3837323-a815-4337-b5e2-24322fec6b08",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "f8476f83-a0ec-408d-a471-5bab4e4e330b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 59864 characters]",
      "text/plain": [
       "<Figure size 1200x1600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 5 Categories:\n",
      "Category\n",
      "Tops       1259\n",
      "T-Shirt     514\n",
      "Pants       386\n",
      "Shoes       173\n",
      "Jeans       160\n",
      "Name: count, dtype: int64\n",
      "\n",
      "Top 5 Types:\n",
      "Type\n",
      "Casual         2754\n",
      "Formal          208\n",
      "Lounge          128\n",
      "Work Casual      15\n",
      "Workout           3\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "plt.style.use('ggplot')\n",
    "\n",
    "# Create a figure with two subplots\n",
    "fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1, figsize=(12, 16))\n",
    "\n",
    "# Plot distribution of Categories\n",
    "sns.countplot(data=result, y='Category', ax=ax1, order=result['New_Category'].value_counts().index)\n",
    "ax1.set_title('Distribution of Categories', fontsize=16)\n",
    "ax1.set_xlabel('Count', fontsize=12)\n",
    "ax1.set_ylabel('Category', fontsize=12)\n",
    "\n",
    "# Plot distribution of Types\n",
    "sns.countplot(data=result, y='Type', ax=ax2, order=result['New_Type'].value_counts().index)\n",
    "ax2.set_title('Distribution of Types', fontsize=16)\n",
    "ax2.set_xlabel('Count', fontsize=12)\n",
    "ax2.set_ylabel('Type', fontsize=12)\n",
    "\n",
    "# Adjust layout and display the plot\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "# Optional: Save the figure\n",
    "# plt.savefig('category_type_distribution.png', dpi=300, bbox_inches='tight')\n",
    "\n",
    "# Additional analysis: Print top 5 categories and types\n",
    "print(\"Top 5 Categories:\")\n",
    "print(result['Category'].value_counts().head())\n",
    "\n",
    "print(\"\\nTop 5 Types:\")\n",
    "print(result['Type'].value_counts().head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "1a1bbaff-da3c-40f3-bde1-21b9350d3900",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Distribution of Categories in Sampled Data:\n",
      "New_Category\n",
      "Jeans      100\n",
      "Other      100\n",
      "Pants      100\n",
      "Shoes      100\n",
      "Shorts     100\n",
      "Skirts     100\n",
      "T-Shirt    100\n",
      "Tops       100\n",
      "Name: count, dtype: int64\n",
      "\n",
      "Distribution of Types in Sampled Data:\n",
      "New_Type\n",
      "Casual    700\n",
      "Formal     64\n",
      "Lounge     36\n",
      "Name: count, dtype: int64\n",
      "\n",
      "Percentage Distribution of Categories:\n",
      "New_Category\n",
      "Jeans      12.5\n",
      "Other      12.5\n",
      "Pants      12.5\n",
      "Shoes      12.5\n",
      "Shorts     12.5\n",
      "Skirts     12.5\n",
      "T-Shirt    12.5\n",
      "Tops       12.5\n",
      "Name: count, dtype: float64\n",
      "\n",
      "Percentage Distribution of Types:\n",
      "New_Type\n",
      "Casual    87.5\n",
      "Formal     8.0\n",
      "Lounge     4.5\n",
      "Name: count, dtype: float64\n",
      "\n",
      "Total number of items in the sampled dataset: 800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_525083/1300003174.py:8: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
      "  sampled_data = result.groupby('New_Category').apply(sample_category).reset_index(drop=True)\n"
     ]
    }
   ],
   "source": [
    "def sample_category(group):\n",
    "    if len(group) > 100:\n",
    "        return group.sample(n=100, random_state=42)\n",
    "    else:\n",
    "        return group\n",
    "\n",
    "# Group by New_Category and apply the sampling function\n",
    "sampled_data = result.groupby('New_Category').apply(sample_category).reset_index(drop=True)\n",
    "\n",
    "# Print the distribution of categories in the sampled data\n",
    "print(\"Distribution of Categories in Sampled Data:\")\n",
    "print(sampled_data['New_Category'].value_counts())\n",
    "\n",
    "# Print the distribution of types in the sampled data\n",
    "print(\"\\nDistribution of Types in Sampled Data:\")\n",
    "print(sampled_data['New_Type'].value_counts())\n",
    "\n",
    "# Calculate and print percentages\n",
    "total = len(sampled_data)\n",
    "print(\"\\nPercentage Distribution of Categories:\")\n",
    "category_percentage = (sampled_data['New_Category'].value_counts() / total * 100).round(2)\n",
    "print(category_percentage)\n",
    "\n",
    "print(\"\\nPercentage Distribution of Types:\")\n",
    "type_percentage = (sampled_data['New_Type'].value_counts() / total * 100).round(2)\n",
    "print(type_percentage)\n",
    "\n",
    "# Print the total number of items in the sampled dataset\n",
    "print(f\"\\nTotal number of items in the sampled dataset: {len(sampled_data)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "6e09e47a-6bef-4259-b6b7-51b3d677b1ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_525083/3643476101.py:8: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
      "  sampled_data = result.groupby('New_Category').apply(sample_category).reset_index(drop=True)\n",
      "/tmp/ipykernel_525083/3643476101.py:16: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  axs[0, 0].set_xticklabels(axs[0, 0].get_xticklabels(), rotation=45, ha='right')\n",
      "/tmp/ipykernel_525083/3643476101.py:21: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  axs[0, 1].set_xticklabels(axs[0, 1].get_xticklabels(), rotation=45, ha='right')\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 157588 characters]",
      "text/plain": [
       "<Figure size 2000x1500 with 5 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of items in the sampled dataset: 800\n"
     ]
    }
   ],
   "source": [
    "def sample_category(group):\n",
    "    if len(group) > 100:\n",
    "        return group.sample(n=100, random_state=42)\n",
    "    else:\n",
    "        return group\n",
    "\n",
    "# Group by New_Category and apply the sampling function\n",
    "sampled_data = result.groupby('New_Category').apply(sample_category).reset_index(drop=True)\n",
    "\n",
    "# Set up the matplotlib figure\n",
    "fig, axs = plt.subplots(2, 2, figsize=(20, 15))\n",
    "\n",
    "# 1. Bar plot of Category distribution\n",
    "sns.countplot(data=sampled_data, x='New_Category', order=sampled_data['New_Category'].value_counts().index, ax=axs[0, 0])\n",
    "axs[0, 0].set_title('Distribution of Categories')\n",
    "axs[0, 0].set_xticklabels(axs[0, 0].get_xticklabels(), rotation=45, ha='right')\n",
    "\n",
    "# 2. Bar plot of Type distribution\n",
    "sns.countplot(data=sampled_data, x='New_Type', order=sampled_data['New_Type'].value_counts().index, ax=axs[0, 1])\n",
    "axs[0, 1].set_title('Distribution of Types')\n",
    "axs[0, 1].set_xticklabels(axs[0, 1].get_xticklabels(), rotation=45, ha='right')\n",
    "\n",
    "# 3. Heatmap of Category vs Type\n",
    "cross_tab = pd.crosstab(sampled_data['New_Category'], sampled_data['New_Type'])\n",
    "sns.heatmap(cross_tab, annot=True, fmt='d', cmap='YlGnBu', ax=axs[1, 0])\n",
    "axs[1, 0].set_title('Heatmap of Category vs Type')\n",
    "\n",
    "# 4. Grouped bar plot of Type distribution within each Category\n",
    "cross_tab_normalized = cross_tab.div(cross_tab.sum(axis=1), axis=0)\n",
    "cross_tab_normalized.plot(kind='bar', stacked=False, ax=axs[1, 1])\n",
    "axs[1, 1].set_title('Type Distribution within each Category')\n",
    "axs[1, 1].set_xlabel('Category')\n",
    "axs[1, 1].set_ylabel('Proportion')\n",
    "axs[1, 1].legend(title='Type', bbox_to_anchor=(1.05, 1), loc='upper left')\n",
    "axs[1, 1].set_xticklabels(axs[1, 1].get_xticklabels(), rotation=45, ha='right')\n",
    "\n",
    "# Adjust layout and display the plot\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "# Print the total number of items in the sampled dataset\n",
    "print(f\"Total number of items in the sampled dataset: {len(sampled_data)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "8232d4d8-6239-4fa8-a1c9-a6e7fa70243b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Filename</th>\n",
       "      <th>Title</th>\n",
       "      <th>Size</th>\n",
       "      <th>Category</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Type</th>\n",
       "      <th>Description</th>\n",
       "      <th>New_Category</th>\n",
       "      <th>New_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d7ed1d64-2c65-427f-9ae4-eb4aaa3e2389.jpg</td>\n",
       "      <td>Stylish and Trendy Tank Top with Celestial Design</td>\n",
       "      <td>M</td>\n",
       "      <td>Tops</td>\n",
       "      <td>F</td>\n",
       "      <td>Casual</td>\n",
       "      <td>This white tank top is a stylish and trendy pi...</td>\n",
       "      <td>Tops</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5c1b7a77-1fa3-4af8-9722-cd38e45d89da.jpg</td>\n",
       "      <td>Classic White Sweatshirt</td>\n",
       "      <td>M</td>\n",
       "      <td>Tops</td>\n",
       "      <td>F</td>\n",
       "      <td>Casual</td>\n",
       "      <td>This classic white sweatshirt is a timeless pi...</td>\n",
       "      <td>Tops</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b2e084c7-e3a0-4182-8671-b908544a7cf2.jpg</td>\n",
       "      <td>Grey T-shirt</td>\n",
       "      <td>M</td>\n",
       "      <td>T-Shirt</td>\n",
       "      <td>Unisex</td>\n",
       "      <td>Casual</td>\n",
       "      <td>This is a short-sleeved, crew neck t-shirt tha...</td>\n",
       "      <td>T-Shirt</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>87846aa9-86cc-404a-af2c-7e8fe941081d.jpg</td>\n",
       "      <td>Long-Sleeved V-Neck Shirt</td>\n",
       "      <td>L</td>\n",
       "      <td>Tops</td>\n",
       "      <td>U</td>\n",
       "      <td>Casual</td>\n",
       "      <td>A long-sleeved, V-neck shirt with a solid purp...</td>\n",
       "      <td>Tops</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>04fa06fb-d71a-4293-9804-fe799375a682.jpg</td>\n",
       "      <td>Silver Metallic Buckle Sandals</td>\n",
       "      <td>L</td>\n",
       "      <td>Footwear</td>\n",
       "      <td>F</td>\n",
       "      <td>Casual</td>\n",
       "      <td>These silver metallic buckle sandals feature a...</td>\n",
       "      <td>Shoes</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Filename  \\\n",
       "0  d7ed1d64-2c65-427f-9ae4-eb4aaa3e2389.jpg   \n",
       "1  5c1b7a77-1fa3-4af8-9722-cd38e45d89da.jpg   \n",
       "2  b2e084c7-e3a0-4182-8671-b908544a7cf2.jpg   \n",
       "5  87846aa9-86cc-404a-af2c-7e8fe941081d.jpg   \n",
       "7  04fa06fb-d71a-4293-9804-fe799375a682.jpg   \n",
       "\n",
       "                                               Title Size  Category  Gender  \\\n",
       "0  Stylish and Trendy Tank Top with Celestial Design    M      Tops       F   \n",
       "1                           Classic White Sweatshirt    M      Tops       F   \n",
       "2                                       Grey T-shirt    M   T-Shirt  Unisex   \n",
       "5                          Long-Sleeved V-Neck Shirt    L      Tops       U   \n",
       "7                     Silver Metallic Buckle Sandals    L  Footwear       F   \n",
       "\n",
       "     Type                                        Description New_Category  \\\n",
       "0  Casual  This white tank top is a stylish and trendy pi...         Tops   \n",
       "1  Casual  This classic white sweatshirt is a timeless pi...         Tops   \n",
       "2  Casual  This is a short-sleeved, crew neck t-shirt tha...      T-Shirt   \n",
       "5  Casual  A long-sleeved, V-neck shirt with a solid purp...         Tops   \n",
       "7  Casual  These silver metallic buckle sandals feature a...        Shoes   \n",
       "\n",
       "  New_Type  \n",
       "0   Casual  \n",
       "1   Casual  \n",
       "2   Casual  \n",
       "5   Casual  \n",
       "7   Casual  "
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "354db8c0-b348-44df-9900-3560c9db136b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "First few rows of the final dataset:\n",
      "                                   Filename  \\\n",
      "0  d7ed1d64-2c65-427f-9ae4-eb4aaa3e2389.jpg   \n",
      "1  5c1b7a77-1fa3-4af8-9722-cd38e45d89da.jpg   \n",
      "2  b2e084c7-e3a0-4182-8671-b908544a7cf2.jpg   \n",
      "5  87846aa9-86cc-404a-af2c-7e8fe941081d.jpg   \n",
      "7  04fa06fb-d71a-4293-9804-fe799375a682.jpg   \n",
      "\n",
      "                                               Title Size  Gender  \\\n",
      "0  Stylish and Trendy Tank Top with Celestial Design    M       F   \n",
      "1                           Classic White Sweatshirt    M       F   \n",
      "2                                       Grey T-shirt    M  Unisex   \n",
      "5                          Long-Sleeved V-Neck Shirt    L       U   \n",
      "7                     Silver Metallic Buckle Sandals    L       F   \n",
      "\n",
      "                                         Description Category    Type  \n",
      "0  This white tank top is a stylish and trendy pi...     Tops  Casual  \n",
      "1  This classic white sweatshirt is a timeless pi...     Tops  Casual  \n",
      "2  This is a short-sleeved, crew neck t-shirt tha...  T-Shirt  Casual  \n",
      "5  A long-sleeved, V-neck shirt with a solid purp...     Tops  Casual  \n",
      "7  These silver metallic buckle sandals feature a...    Shoes  Casual  \n",
      "\n",
      "Columns in the final dataset:\n",
      "['Filename', 'Title', 'Size', 'Gender', 'Description', 'Category', 'Type']\n",
      "\n",
      "Final dataset saved as 'final_balanced_sample_dataset.csv'\n"
     ]
    }
   ],
   "source": [
    "final_data = result.drop(columns=['Type', 'Category'])\n",
    "\n",
    "# Rename 'New_Type' to 'Type' and 'New_Category' to 'Category'\n",
    "final_data = final_data.rename(columns={'New_Type': 'Type', 'New_Category': 'Category'})\n",
    "\n",
    "# Print the first few rows of the final dataset\n",
    "print(\"\\nFirst few rows of the final dataset:\")\n",
    "print(final_data.head())\n",
    "\n",
    "# Print the column names of the final dataset\n",
    "print(\"\\nColumns in the final dataset:\")\n",
    "print(final_data.columns.tolist())\n",
    "\n",
    "# Save the final DataFrame\n",
    "final_data.to_csv('final_balanced_sample_dataset.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9577f9f6-23e7-4fde-a162-2fa633265399",
   "metadata": {},
   "source": [
    "### Creating a Vector DB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "5e7d968d-bf1b-4a43-ad9f-7f2ca6736c1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install lancedb rerankers\n",
    "#!pip install sentence-transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a0db3c93-a0f2-4f49-908a-181b63b5847e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a176716866754071b3f28533a0b5db10",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "modules.json:   0%|          | 0.00/349 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac21a9a40c6f410e84bd46137de72cf2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config_sentence_transformers.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33a4b93a6e4d4ed2a6212b9bb3177f97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/94.8k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1884e2f091dc42e8bc0199f16542bfd9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "sentence_bert_config.json:   0%|          | 0.00/52.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9dfd32a6a964b33bfb7dcb2ff8a5770",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/743 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f16f57b26544e4d9c7aabfc9e6ec7e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/133M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50deb4679796404d8ea10c53c6da2d92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/366 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c8ccaf41b4e4fe8a8dbfa3f90fd760c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1bf4bcb6c2c5482d827f72131f6719ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/711k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "25402e266bf04a138f31c8dfea9e3d16",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/125 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae377bce030646a2a5823c9d1bbaf481",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "1_Pooling/config.json:   0%|          | 0.00/190 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2024-10-02T21:22:52Z WARN  lance::dataset] No existing dataset at /home/sanyambhutani/.lancedb/clothes.lance, it will be created\n"
     ]
    }
   ],
   "source": [
    "import lancedb\n",
    "from lancedb.pydantic import LanceModel, Vector\n",
    "from lancedb.embeddings import get_registry\n",
    "from lancedb.rerankers import ColbertReranker\n",
    "\n",
    "model = get_registry().get(\"sentence-transformers\").create(name=\"BAAI/bge-small-en-v1.5\", device=\"cuda\")\n",
    "\n",
    "\n",
    "class Schema(LanceModel):\n",
    "    Filename: str\n",
    "    Title: str\n",
    "    Size: str\n",
    "    Gender: str\n",
    "    Description: str = model.SourceField()\n",
    "    vector: Vector(model.ndims()) = model.VectorField()\n",
    "    Category: str\n",
    "    Type: str\n",
    "    \n",
    "db = lancedb.connect(\"~/.lancedb\")\n",
    "tbl = db.create_table(name=\"clothes\", schema=Schema, mode=\"overwrite\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a3341568-d835-4c80-8e90-de651657bcca",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./final_balanced_sample_dataset.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "47976e72-6093-4314-a7d9-fedba7316e56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Filename</th>\n",
       "      <th>Title</th>\n",
       "      <th>Size</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Description</th>\n",
       "      <th>Category</th>\n",
       "      <th>Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d7ed1d64-2c65-427f-9ae4-eb4aaa3e2389.jpg</td>\n",
       "      <td>Stylish and Trendy Tank Top with Celestial Design</td>\n",
       "      <td>M</td>\n",
       "      <td>F</td>\n",
       "      <td>This white tank top is a stylish and trendy pi...</td>\n",
       "      <td>Tops</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5c1b7a77-1fa3-4af8-9722-cd38e45d89da.jpg</td>\n",
       "      <td>Classic White Sweatshirt</td>\n",
       "      <td>M</td>\n",
       "      <td>F</td>\n",
       "      <td>This classic white sweatshirt is a timeless pi...</td>\n",
       "      <td>Tops</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b2e084c7-e3a0-4182-8671-b908544a7cf2.jpg</td>\n",
       "      <td>Grey T-shirt</td>\n",
       "      <td>M</td>\n",
       "      <td>Unisex</td>\n",
       "      <td>This is a short-sleeved, crew neck t-shirt tha...</td>\n",
       "      <td>T-Shirt</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>87846aa9-86cc-404a-af2c-7e8fe941081d.jpg</td>\n",
       "      <td>Long-Sleeved V-Neck Shirt</td>\n",
       "      <td>L</td>\n",
       "      <td>U</td>\n",
       "      <td>A long-sleeved, V-neck shirt with a solid purp...</td>\n",
       "      <td>Tops</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>04fa06fb-d71a-4293-9804-fe799375a682.jpg</td>\n",
       "      <td>Silver Metallic Buckle Sandals</td>\n",
       "      <td>L</td>\n",
       "      <td>F</td>\n",
       "      <td>These silver metallic buckle sandals feature a...</td>\n",
       "      <td>Shoes</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3112</th>\n",
       "      <td>c1fafe22-a65b-4ce4-9383-dbd470a205e6.jpg</td>\n",
       "      <td>Pink Bird Printed Long Sleeved T-Shirt</td>\n",
       "      <td>L</td>\n",
       "      <td>F</td>\n",
       "      <td>A long-sleeved t-shirt with a crew neck and pi...</td>\n",
       "      <td>Tops</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3113</th>\n",
       "      <td>4cc0a79e-aa26-4afc-aabc-5612f8515bf8.jpg</td>\n",
       "      <td>Blue and Gold Top</td>\n",
       "      <td>L</td>\n",
       "      <td>F</td>\n",
       "      <td>This sleeveless top features a beautiful blue ...</td>\n",
       "      <td>Tops</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3114</th>\n",
       "      <td>ae9cec7a-dd1d-49bc-adae-6446429c03d8.jpg</td>\n",
       "      <td>Men's Light Blue and White Striped Long-Sleeve...</td>\n",
       "      <td>M</td>\n",
       "      <td>M</td>\n",
       "      <td>This men's light blue and white striped long-s...</td>\n",
       "      <td>Tops</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3115</th>\n",
       "      <td>de853711-0b97-45a6-a794-3c424246db03.jpg</td>\n",
       "      <td>Black Sneakers</td>\n",
       "      <td>S</td>\n",
       "      <td>U</td>\n",
       "      <td>These sleek and versatile black sneakers are a...</td>\n",
       "      <td>Shoes</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3116</th>\n",
       "      <td>d4b0b957-5632-4df1-aba6-e562e2a84687.jpg</td>\n",
       "      <td>Gray T-Shirt with Hood and Graphic</td>\n",
       "      <td>M</td>\n",
       "      <td>M</td>\n",
       "      <td>The gray t-shirt with a hood and graphic is a ...</td>\n",
       "      <td>T-Shirt</td>\n",
       "      <td>Casual</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3117 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Filename  \\\n",
       "0     d7ed1d64-2c65-427f-9ae4-eb4aaa3e2389.jpg   \n",
       "1     5c1b7a77-1fa3-4af8-9722-cd38e45d89da.jpg   \n",
       "2     b2e084c7-e3a0-4182-8671-b908544a7cf2.jpg   \n",
       "3     87846aa9-86cc-404a-af2c-7e8fe941081d.jpg   \n",
       "4     04fa06fb-d71a-4293-9804-fe799375a682.jpg   \n",
       "...                                        ...   \n",
       "3112  c1fafe22-a65b-4ce4-9383-dbd470a205e6.jpg   \n",
       "3113  4cc0a79e-aa26-4afc-aabc-5612f8515bf8.jpg   \n",
       "3114  ae9cec7a-dd1d-49bc-adae-6446429c03d8.jpg   \n",
       "3115  de853711-0b97-45a6-a794-3c424246db03.jpg   \n",
       "3116  d4b0b957-5632-4df1-aba6-e562e2a84687.jpg   \n",
       "\n",
       "                                                  Title Size  Gender  \\\n",
       "0     Stylish and Trendy Tank Top with Celestial Design    M       F   \n",
       "1                              Classic White Sweatshirt    M       F   \n",
       "2                                          Grey T-shirt    M  Unisex   \n",
       "3                             Long-Sleeved V-Neck Shirt    L       U   \n",
       "4                        Silver Metallic Buckle Sandals    L       F   \n",
       "...                                                 ...  ...     ...   \n",
       "3112             Pink Bird Printed Long Sleeved T-Shirt    L       F   \n",
       "3113                                  Blue and Gold Top    L       F   \n",
       "3114  Men's Light Blue and White Striped Long-Sleeve...    M       M   \n",
       "3115                                     Black Sneakers    S       U   \n",
       "3116                 Gray T-Shirt with Hood and Graphic    M       M   \n",
       "\n",
       "                                            Description Category    Type  \n",
       "0     This white tank top is a stylish and trendy pi...     Tops  Casual  \n",
       "1     This classic white sweatshirt is a timeless pi...     Tops  Casual  \n",
       "2     This is a short-sleeved, crew neck t-shirt tha...  T-Shirt  Casual  \n",
       "3     A long-sleeved, V-neck shirt with a solid purp...     Tops  Casual  \n",
       "4     These silver metallic buckle sandals feature a...    Shoes  Casual  \n",
       "...                                                 ...      ...     ...  \n",
       "3112  A long-sleeved t-shirt with a crew neck and pi...     Tops  Casual  \n",
       "3113  This sleeveless top features a beautiful blue ...     Tops  Casual  \n",
       "3114  This men's light blue and white striped long-s...     Tops  Casual  \n",
       "3115  These sleek and versatile black sneakers are a...    Shoes  Casual  \n",
       "3116  The gray t-shirt with a hood and graphic is a ...  T-Shirt  Casual  \n",
       "\n",
       "[3117 rows x 7 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9470c102-781d-4888-a373-efc184115cc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_unescaped_quotes(json_string):\n",
    "    # Find the \"Description\" field and its content\n",
    "    pattern = r'\"Description\"\\s*:\\s*\"(.*?)\"(?=\\s*[,}])'\n",
    "    \n",
    "    def escape_quotes(match):\n",
    "        # Escape any unescaped quotes in the description content\n",
    "        content = match.group(1)\n",
    "        escaped_content = re.sub(r'(?<!\\\\)\"', r'\\\"', content)\n",
    "        return f'\"Description\":\"{escaped_content}\"'\n",
    "    \n",
    "    # Replace the Description field with properly escaped content\n",
    "    fixed_json = re.sub(pattern, escape_quotes, json_string)\n",
    "    # Now we can safely parse the JSON\n",
    "    try:\n",
    "        fixed_json = \"{\" + fixed_json.split(\"{\")[1].split(\"}\")[0] + \"}\"\n",
    "        return json.loads(fixed_json)\n",
    "    except:\n",
    "        return {\n",
    "        'Title': \"\", 'Size': \"\" , 'Category': \"\" , 'Gender': \"\" , 'Type': \"\" , 'Description': \"\"\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3c9ce7f4-9c8d-4631-88b8-796a9c97ffdd",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = [{\"Filename\": row[\"Filename\"],**fix_unescaped_quotes(row[\"Description\"])} for index, row in df.iterrows()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8b0402c2-ee14-4bb3-bc0b-1854a6aac72a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# automatically generate vectors\n",
    "tbl.add(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7a278c8f-17de-47a7-9f90-a8d68ea4e6aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Filename</th>\n",
       "      <th>Title</th>\n",
       "      <th>Size</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Description</th>\n",
       "      <th>vector</th>\n",
       "      <th>Category</th>\n",
       "      <th>Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d7ed1d64-2c65-427f-9ae4-eb4aaa3e2389.jpg</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[0.04846169, -0.0012961391, 0.016879003, -0.04...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5c1b7a77-1fa3-4af8-9722-cd38e45d89da.jpg</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[0.04846169, -0.0012961391, 0.016879003, -0.04...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b2e084c7-e3a0-4182-8671-b908544a7cf2.jpg</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[0.04846169, -0.0012961391, 0.016879003, -0.04...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>87846aa9-86cc-404a-af2c-7e8fe941081d.jpg</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[0.04846169, -0.0012961391, 0.016879003, -0.04...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>04fa06fb-d71a-4293-9804-fe799375a682.jpg</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[0.04846169, -0.0012961391, 0.016879003, -0.04...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>8f576f1a-839d-4fb2-a224-a4700b2d05da.jpg</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[0.04846169, -0.0012961391, 0.016879003, -0.04...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>e976a8f6-6731-485f-8a9a-2872a5208818.jpg</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[0.04846169, -0.0012961391, 0.016879003, -0.04...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>bbf0d9c7-663d-46d1-a9f8-66e8e5678541.jpg</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[0.04846169, -0.0012961391, 0.016879003, -0.04...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>e25a7faa-7a49-4e72-a7ef-e74427f77784.jpg</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[0.04846169, -0.0012961391, 0.016879003, -0.04...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>d995ac1f-fbd0-482c-a308-dafb6a93cfd0.jpg</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[0.04846169, -0.0012961391, 0.016879003, -0.04...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Filename Title Size Gender Description  \\\n",
       "0  d7ed1d64-2c65-427f-9ae4-eb4aaa3e2389.jpg                                 \n",
       "1  5c1b7a77-1fa3-4af8-9722-cd38e45d89da.jpg                                 \n",
       "2  b2e084c7-e3a0-4182-8671-b908544a7cf2.jpg                                 \n",
       "3  87846aa9-86cc-404a-af2c-7e8fe941081d.jpg                                 \n",
       "4  04fa06fb-d71a-4293-9804-fe799375a682.jpg                                 \n",
       "5  8f576f1a-839d-4fb2-a224-a4700b2d05da.jpg                                 \n",
       "6  e976a8f6-6731-485f-8a9a-2872a5208818.jpg                                 \n",
       "7  bbf0d9c7-663d-46d1-a9f8-66e8e5678541.jpg                                 \n",
       "8  e25a7faa-7a49-4e72-a7ef-e74427f77784.jpg                                 \n",
       "9  d995ac1f-fbd0-482c-a308-dafb6a93cfd0.jpg                                 \n",
       "\n",
       "                                              vector Category Type  \n",
       "0  [0.04846169, -0.0012961391, 0.016879003, -0.04...                \n",
       "1  [0.04846169, -0.0012961391, 0.016879003, -0.04...                \n",
       "2  [0.04846169, -0.0012961391, 0.016879003, -0.04...                \n",
       "3  [0.04846169, -0.0012961391, 0.016879003, -0.04...                \n",
       "4  [0.04846169, -0.0012961391, 0.016879003, -0.04...                \n",
       "5  [0.04846169, -0.0012961391, 0.016879003, -0.04...                \n",
       "6  [0.04846169, -0.0012961391, 0.016879003, -0.04...                \n",
       "7  [0.04846169, -0.0012961391, 0.016879003, -0.04...                \n",
       "8  [0.04846169, -0.0012961391, 0.016879003, -0.04...                \n",
       "9  [0.04846169, -0.0012961391, 0.016879003, -0.04...                "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tbl.search().to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee854540-3908-4428-a063-72c8997a2540",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
