|
|
@@ -1310,473 +1310,13 @@
|
|
|
},
|
|
|
{
|
|
|
"cell_type": "code",
|
|
|
- "execution_count": 52,
|
|
|
- "id": "e458bdda-4ac0-424a-8b04-a4f2261e0f57",
|
|
|
- "metadata": {},
|
|
|
- "outputs": [
|
|
|
- {
|
|
|
- "data": {
|
|
|
- "application/vnd.jupyter.widget-view+json": {
|
|
|
- "model_id": "6cf3b9eca0bc4120b6fc7303bf82dc2f",
|
|
|
- "version_major": 2,
|
|
|
- "version_minor": 0
|
|
|
- },
|
|
|
- "text/plain": [
|
|
|
- "VBox(children=(FileUpload(value=(), accept='image/*', description='Upload'), Text(value='', placeholder='Enter…"
|
|
|
- ]
|
|
|
- },
|
|
|
- "metadata": {},
|
|
|
- "output_type": "display_data"
|
|
|
- }
|
|
|
- ],
|
|
|
- "source": [
|
|
|
- "import pandas as pd\n",
|
|
|
- "import lancedb\n",
|
|
|
- "from lancedb.pydantic import LanceModel, Vector\n",
|
|
|
- "from lancedb.embeddings import get_registry\n",
|
|
|
- "from pathlib import Path\n",
|
|
|
- "from PIL import Image\n",
|
|
|
- "import io\n",
|
|
|
- "import base64\n",
|
|
|
- "from together import Together\n",
|
|
|
- "import os\n",
|
|
|
- "from ipywidgets import FileUpload, Output, VBox, HBox, Button, Text\n",
|
|
|
- "from IPython.display import display, clear_output\n",
|
|
|
- "\n",
|
|
|
- "# Set up the sentence transformer model\n",
|
|
|
- "model = get_registry().get(\"sentence-transformers\").create(name=\"BAAI/bge-small-en-v1.5\", device=\"cuda\")\n",
|
|
|
- "\n",
|
|
|
- "# Define the schema for our LanceDB table\n",
|
|
|
- "class Schema(LanceModel):\n",
|
|
|
- " Filename: str\n",
|
|
|
- " Title: str\n",
|
|
|
- " Size: str\n",
|
|
|
- " Gender: str\n",
|
|
|
- " Description: str = model.SourceField()\n",
|
|
|
- " Category: str\n",
|
|
|
- " Type: str\n",
|
|
|
- " vector: Vector(model.ndims()) = model.VectorField()\n",
|
|
|
- "\n",
|
|
|
- "# Connect to LanceDB and create/load the table\n",
|
|
|
- "db = lancedb.connect(\"~/.lancedb\")\n",
|
|
|
- "tbl = db.create_table(name=\"clothes\", schema=Schema, mode=\"overwrite\")\n",
|
|
|
- "\n",
|
|
|
- "# Load and clean data (assuming this part is already done)\n",
|
|
|
- "df = pd.read_csv(\"./final_balanced_sample_dataset.csv\")\n",
|
|
|
- "df = df.dropna().astype(str)\n",
|
|
|
- "tbl.add(df.to_dict('records'))\n",
|
|
|
- "\n",
|
|
|
- "# Set up the Together API client\n",
|
|
|
- "os.environ[\"TOGETHER_API_KEY\"] = \"714312b00529afd04be6c863e71d813a309f764ecf487c7538ab6da6a2addd5a\"\n",
|
|
|
- "client = Together(api_key=os.environ.get('TOGETHER_API_KEY'))\n",
|
|
|
- "\n",
|
|
|
- "def encode_image(image_content):\n",
|
|
|
- " return base64.b64encode(image_content).decode('utf-8')\n",
|
|
|
- "\n",
|
|
|
- "def generate_description(image_content):\n",
|
|
|
- " base64_image = encode_image(image_content)\n",
|
|
|
- " response = client.chat.completions.create(\n",
|
|
|
- " model=\"meta-llama/Llama-Vision-Free\",\n",
|
|
|
- " messages=[\n",
|
|
|
- " {\n",
|
|
|
- " \"role\": \"user\",\n",
|
|
|
- " \"content\": [\n",
|
|
|
- " {\n",
|
|
|
- " \"type\": \"image_url\",\n",
|
|
|
- " \"image_url\": {\n",
|
|
|
- " \"url\": f\"data:image/jpeg;base64,{base64_image}\"\n",
|
|
|
- " }\n",
|
|
|
- " },\n",
|
|
|
- " {\n",
|
|
|
- " \"type\": \"text\",\n",
|
|
|
- " \"text\": \"Describe this clothing item in detail.\"\n",
|
|
|
- " }\n",
|
|
|
- " ]\n",
|
|
|
- " }\n",
|
|
|
- " ],\n",
|
|
|
- " max_tokens=512,\n",
|
|
|
- " temperature=0.7,\n",
|
|
|
- " )\n",
|
|
|
- " return response.choices[0].message.content\n",
|
|
|
- "\n",
|
|
|
- "def retrieve_similar_items(description, n=10):\n",
|
|
|
- " results = tbl.search(description).limit(n).to_pandas()\n",
|
|
|
- " return results\n",
|
|
|
- "\n",
|
|
|
- "def show_images(results, output):\n",
|
|
|
- " with output:\n",
|
|
|
- " clear_output(wait=True)\n",
|
|
|
- " for _, row in results.iterrows():\n",
|
|
|
- " img_path = Path(\"archive\") / \"images_compressed\" / row['Filename']\n",
|
|
|
- " img = Image.open(img_path)\n",
|
|
|
- " display(img)\n",
|
|
|
- " print(f\"Title: {row['Title']}\")\n",
|
|
|
- " print(f\"Description: {row['Description']}\")\n",
|
|
|
- " print(\"---\")\n",
|
|
|
- "\n",
|
|
|
- "def create_fashion_assistant():\n",
|
|
|
- " upload = FileUpload(accept='image/*', multiple=False)\n",
|
|
|
- " output = Output()\n",
|
|
|
- " description_text = Text(placeholder='Enter a description or refinement prompt')\n",
|
|
|
- " search_button = Button(description='Search')\n",
|
|
|
- " \n",
|
|
|
- " def on_upload_change(change):\n",
|
|
|
- " if change['type'] == 'change' and change['name'] == 'value':\n",
|
|
|
- " with output:\n",
|
|
|
- " clear_output(wait=True)\n",
|
|
|
- " if upload.value:\n",
|
|
|
- " file_info = next(iter(upload.value))\n",
|
|
|
- " image_content = file_info.content\n",
|
|
|
- " try:\n",
|
|
|
- " description = generate_description(image_content)\n",
|
|
|
- " description_text.value = description\n",
|
|
|
- " print(f\"Generated description: {description}\")\n",
|
|
|
- " except Exception as e:\n",
|
|
|
- " print(f\"Error generating description: {str(e)}\")\n",
|
|
|
- " \n",
|
|
|
- " def on_search_click(b):\n",
|
|
|
- " with output:\n",
|
|
|
- " clear_output(wait=True)\n",
|
|
|
- " description = description_text.value\n",
|
|
|
- " results = retrieve_similar_items(description)\n",
|
|
|
- " show_images(results, output)\n",
|
|
|
- " \n",
|
|
|
- " upload.observe(on_upload_change, names='value')\n",
|
|
|
- " search_button.on_click(on_search_click)\n",
|
|
|
- " \n",
|
|
|
- " display(VBox([upload, description_text, search_button, output]))\n",
|
|
|
- "\n",
|
|
|
- "# Call this function to start the fashion assistant\n",
|
|
|
- "create_fashion_assistant()"
|
|
|
- ]
|
|
|
- },
|
|
|
- {
|
|
|
- "cell_type": "code",
|
|
|
- "execution_count": 53,
|
|
|
- "id": "765dac51-23fa-44de-a30c-334282b20bcc",
|
|
|
- "metadata": {},
|
|
|
- "outputs": [
|
|
|
- {
|
|
|
- "data": {
|
|
|
- "application/vnd.jupyter.widget-view+json": {
|
|
|
- "model_id": "c1c3ece898fd4e6bbcd7bb3c01233880",
|
|
|
- "version_major": 2,
|
|
|
- "version_minor": 0
|
|
|
- },
|
|
|
- "text/plain": [
|
|
|
- "VBox(children=(FileUpload(value=(), accept='image/*', description='Upload'), Text(value='', placeholder='Enter…"
|
|
|
- ]
|
|
|
- },
|
|
|
- "metadata": {},
|
|
|
- "output_type": "display_data"
|
|
|
- }
|
|
|
- ],
|
|
|
- "source": [
|
|
|
- "import pandas as pd\n",
|
|
|
- "import lancedb\n",
|
|
|
- "from lancedb.pydantic import LanceModel, Vector\n",
|
|
|
- "from lancedb.embeddings import get_registry\n",
|
|
|
- "from pathlib import Path\n",
|
|
|
- "from PIL import Image\n",
|
|
|
- "import io\n",
|
|
|
- "import base64\n",
|
|
|
- "from together import Together\n",
|
|
|
- "import os\n",
|
|
|
- "from ipywidgets import FileUpload, Output, VBox, HBox, Button, Text\n",
|
|
|
- "from IPython.display import display, clear_output\n",
|
|
|
- "\n",
|
|
|
- "# ... (previous setup code remains the same)\n",
|
|
|
- "\n",
|
|
|
- "def encode_image(image_content):\n",
|
|
|
- " return base64.b64encode(image_content).decode('utf-8')\n",
|
|
|
- "\n",
|
|
|
- "def generate_description(image_content):\n",
|
|
|
- " base64_image = encode_image(image_content)\n",
|
|
|
- " response = client.chat.completions.create(\n",
|
|
|
- " model=\"meta-llama/Llama-Vision-Free\",\n",
|
|
|
- " messages=[\n",
|
|
|
- " {\n",
|
|
|
- " \"role\": \"user\",\n",
|
|
|
- " \"content\": [\n",
|
|
|
- " {\n",
|
|
|
- " \"type\": \"image_url\",\n",
|
|
|
- " \"image_url\": {\n",
|
|
|
- " \"url\": f\"data:image/jpeg;base64,{base64_image}\"\n",
|
|
|
- " }\n",
|
|
|
- " },\n",
|
|
|
- " {\n",
|
|
|
- " \"type\": \"text\",\n",
|
|
|
- " \"text\": \"Describe this clothing item in detail.\"\n",
|
|
|
- " }\n",
|
|
|
- " ]\n",
|
|
|
- " }\n",
|
|
|
- " ],\n",
|
|
|
- " max_tokens=512,\n",
|
|
|
- " temperature=0.7,\n",
|
|
|
- " )\n",
|
|
|
- " return response.choices[0].message.content\n",
|
|
|
- "\n",
|
|
|
- "def process_text_input(text):\n",
|
|
|
- " response = client.chat.completions.create(\n",
|
|
|
- " model=\"meta-llama/Llama-Vision-Free\",\n",
|
|
|
- " messages=[\n",
|
|
|
- " {\n",
|
|
|
- " \"role\": \"user\",\n",
|
|
|
- " \"content\": f\"Based on the following description or refinement, provide a detailed description of the clothing item: {text}\"\n",
|
|
|
- " }\n",
|
|
|
- " ],\n",
|
|
|
- " max_tokens=512,\n",
|
|
|
- " temperature=0.7,\n",
|
|
|
- " )\n",
|
|
|
- " return response.choices[0].message.content\n",
|
|
|
- "\n",
|
|
|
- "def retrieve_similar_items(description, n=10):\n",
|
|
|
- " results = tbl.search(description).limit(n).to_pandas()\n",
|
|
|
- " return results\n",
|
|
|
- "\n",
|
|
|
- "def show_images(results, output):\n",
|
|
|
- " with output:\n",
|
|
|
- " clear_output(wait=True)\n",
|
|
|
- " for _, row in results.iterrows():\n",
|
|
|
- " img_path = Path(\"archive\") / \"images_compressed\" / row['Filename']\n",
|
|
|
- " img = Image.open(img_path)\n",
|
|
|
- " display(img)\n",
|
|
|
- " print(f\"Title: {row['Title']}\")\n",
|
|
|
- " print(f\"Description: {row['Description']}\")\n",
|
|
|
- " print(\"---\")\n",
|
|
|
- "\n",
|
|
|
- "def create_fashion_assistant():\n",
|
|
|
- " upload = FileUpload(accept='image/*', multiple=False)\n",
|
|
|
- " output = Output()\n",
|
|
|
- " description_text = Text(placeholder='Enter a description or refinement prompt')\n",
|
|
|
- " search_button = Button(description='Search')\n",
|
|
|
- " \n",
|
|
|
- " def on_upload_change(change):\n",
|
|
|
- " if change['type'] == 'change' and change['name'] == 'value':\n",
|
|
|
- " with output:\n",
|
|
|
- " clear_output(wait=True)\n",
|
|
|
- " if upload.value:\n",
|
|
|
- " file_info = next(iter(upload.value))\n",
|
|
|
- " image_content = file_info.content\n",
|
|
|
- " try:\n",
|
|
|
- " description = generate_description(image_content)\n",
|
|
|
- " description_text.value = description\n",
|
|
|
- " print(f\"Generated description: {description}\")\n",
|
|
|
- " except Exception as e:\n",
|
|
|
- " print(f\"Error generating description: {str(e)}\")\n",
|
|
|
- " \n",
|
|
|
- " def on_search_click(b):\n",
|
|
|
- " with output:\n",
|
|
|
- " clear_output(wait=True)\n",
|
|
|
- " input_text = description_text.value\n",
|
|
|
- " try:\n",
|
|
|
- " processed_description = process_text_input(input_text)\n",
|
|
|
- " print(f\"Processed description: {processed_description}\")\n",
|
|
|
- " results = retrieve_similar_items(processed_description)\n",
|
|
|
- " show_images(results, output)\n",
|
|
|
- " except Exception as e:\n",
|
|
|
- " print(f\"Error processing input or retrieving results: {str(e)}\")\n",
|
|
|
- " \n",
|
|
|
- " upload.observe(on_upload_change, names='value')\n",
|
|
|
- " search_button.on_click(on_search_click)\n",
|
|
|
- " \n",
|
|
|
- " display(VBox([upload, description_text, search_button, output]))\n",
|
|
|
- "\n",
|
|
|
- "# Call this function to start the fashion assistant\n",
|
|
|
- "create_fashion_assistant()"
|
|
|
- ]
|
|
|
- },
|
|
|
- {
|
|
|
- "cell_type": "code",
|
|
|
- "execution_count": 55,
|
|
|
- "id": "605f9456-a396-457c-b738-b85890b18240",
|
|
|
- "metadata": {},
|
|
|
- "outputs": [
|
|
|
- {
|
|
|
- "data": {
|
|
|
- "application/vnd.jupyter.widget-view+json": {
|
|
|
- "model_id": "eeb086a415794690b94887ef83f98159",
|
|
|
- "version_major": 2,
|
|
|
- "version_minor": 0
|
|
|
- },
|
|
|
- "text/plain": [
|
|
|
- "VBox(children=(FileUpload(value=(), accept='image/*', description='Upload'), Text(value='', placeholder='Enter…"
|
|
|
- ]
|
|
|
- },
|
|
|
- "metadata": {},
|
|
|
- "output_type": "display_data"
|
|
|
- }
|
|
|
- ],
|
|
|
- "source": [
|
|
|
- "import pandas as pd\n",
|
|
|
- "import lancedb\n",
|
|
|
- "from lancedb.pydantic import LanceModel, Vector\n",
|
|
|
- "from lancedb.embeddings import get_registry\n",
|
|
|
- "from pathlib import Path\n",
|
|
|
- "from PIL import Image\n",
|
|
|
- "import io\n",
|
|
|
- "import base64\n",
|
|
|
- "from together import Together\n",
|
|
|
- "import os\n",
|
|
|
- "from ipywidgets import FileUpload, Output, VBox, HBox, Button, Text\n",
|
|
|
- "from IPython.display import display, clear_output\n",
|
|
|
- "\n",
|
|
|
- "# Set up the sentence transformer model\n",
|
|
|
- "model = get_registry().get(\"sentence-transformers\").create(name=\"BAAI/bge-small-en-v1.5\", device=\"cuda\")\n",
|
|
|
- "\n",
|
|
|
- "# Define the schema for our LanceDB table\n",
|
|
|
- "class Schema(LanceModel):\n",
|
|
|
- " Filename: str\n",
|
|
|
- " Title: str\n",
|
|
|
- " Size: str\n",
|
|
|
- " Gender: str\n",
|
|
|
- " Description: str = model.SourceField()\n",
|
|
|
- " Category: str\n",
|
|
|
- " Type: str\n",
|
|
|
- " vector: Vector(model.ndims()) = model.VectorField()\n",
|
|
|
- "\n",
|
|
|
- "# Connect to LanceDB and create/load the table\n",
|
|
|
- "db = lancedb.connect(\"~/.lancedb\")\n",
|
|
|
- "tbl = db.create_table(name=\"clothes\", schema=Schema, mode=\"overwrite\")\n",
|
|
|
- "\n",
|
|
|
- "# Load and clean data (assuming this part is already done)\n",
|
|
|
- "df = pd.read_csv(\"./final_balanced_sample_dataset.csv\")\n",
|
|
|
- "df = df.dropna().astype(str)\n",
|
|
|
- "tbl.add(df.to_dict('records'))\n",
|
|
|
- "\n",
|
|
|
- "# Set up the Together API client\n",
|
|
|
- "os.environ[\"TOGETHER_API_KEY\"] = \"714312b00529afd04be6c863e71d813a309f764ecf487c7538ab6da6a2addd5a\"\n",
|
|
|
- "client = Together(api_key=os.environ.get('TOGETHER_API_KEY'))\n",
|
|
|
- "\n",
|
|
|
- "def encode_image(image_content):\n",
|
|
|
- " return base64.b64encode(image_content).decode('utf-8')\n",
|
|
|
- "\n",
|
|
|
- "def generate_description(image_content):\n",
|
|
|
- " base64_image = encode_image(image_content)\n",
|
|
|
- " response = client.chat.completions.create(\n",
|
|
|
- " model=\"meta-llama/Llama-Vision-Free\",\n",
|
|
|
- " messages=[\n",
|
|
|
- " {\n",
|
|
|
- " \"role\": \"user\",\n",
|
|
|
- " \"content\": [\n",
|
|
|
- " {\n",
|
|
|
- " \"type\": \"image_url\",\n",
|
|
|
- " \"image_url\": {\n",
|
|
|
- " \"url\": f\"data:image/jpeg;base64,{base64_image}\"\n",
|
|
|
- " }\n",
|
|
|
- " },\n",
|
|
|
- " {\n",
|
|
|
- " \"type\": \"text\",\n",
|
|
|
- " \"text\": \"Describe this clothing item in detail.\"\n",
|
|
|
- " }\n",
|
|
|
- " ]\n",
|
|
|
- " }\n",
|
|
|
- " ],\n",
|
|
|
- " max_tokens=512,\n",
|
|
|
- " temperature=0.7,\n",
|
|
|
- " )\n",
|
|
|
- " return response.choices[0].message.content\n",
|
|
|
- "\n",
|
|
|
- "def suggest_complementary_clothes(description):\n",
|
|
|
- " response = client.chat.completions.create(\n",
|
|
|
- " model=\"meta-llama/Llama-Vision-Free\",\n",
|
|
|
- " messages=[\n",
|
|
|
- " {\n",
|
|
|
- " \"role\": \"user\",\n",
|
|
|
- " \"content\": f\"Based on this description of a clothing item: '{description}', suggest complementary clothes that would go well with it. Provide a detailed description of these complementary items suitable for a vector database query.\"\n",
|
|
|
- " }\n",
|
|
|
- " ],\n",
|
|
|
- " max_tokens=512,\n",
|
|
|
- " temperature=0.7,\n",
|
|
|
- " )\n",
|
|
|
- " return response.choices[0].message.content\n",
|
|
|
- "\n",
|
|
|
- "def process_human_input(text):\n",
|
|
|
- " response = client.chat.completions.create(\n",
|
|
|
- " model=\"meta-llama/Llama-Vision-Free\",\n",
|
|
|
- " messages=[\n",
|
|
|
- " {\n",
|
|
|
- " \"role\": \"user\",\n",
|
|
|
- " \"content\": f\"Based on this user input about clothing: '{text}', create a detailed query suitable for searching a vector database of clothing items. Expand on the key features and attributes mentioned.\"\n",
|
|
|
- " }\n",
|
|
|
- " ],\n",
|
|
|
- " max_tokens=512,\n",
|
|
|
- " temperature=0.7,\n",
|
|
|
- " )\n",
|
|
|
- " return response.choices[0].message.content\n",
|
|
|
- "\n",
|
|
|
- "def retrieve_similar_items(description, n=10):\n",
|
|
|
- " results = tbl.search(description).limit(n).to_pandas()\n",
|
|
|
- " return results\n",
|
|
|
- "\n",
|
|
|
- "def show_images(results, output):\n",
|
|
|
- " with output:\n",
|
|
|
- " clear_output(wait=True)\n",
|
|
|
- " for _, row in results.iterrows():\n",
|
|
|
- " img_path = Path(\"archive\") / \"images_compressed\" / row['Filename']\n",
|
|
|
- " img = Image.open(img_path)\n",
|
|
|
- " display(img)\n",
|
|
|
- " print(f\"Title: {row['Title']}\")\n",
|
|
|
- " print(f\"Description: {row['Description']}\")\n",
|
|
|
- " print(\"---\")\n",
|
|
|
- "\n",
|
|
|
- "def create_fashion_assistant():\n",
|
|
|
- " upload = FileUpload(accept='image/*', multiple=False)\n",
|
|
|
- " output = Output()\n",
|
|
|
- " description_text = Text(placeholder='Enter a description or refinement prompt')\n",
|
|
|
- " search_button = Button(description='Search')\n",
|
|
|
- " \n",
|
|
|
- " def on_upload_change(change):\n",
|
|
|
- " if change['type'] == 'change' and change['name'] == 'value':\n",
|
|
|
- " with output:\n",
|
|
|
- " clear_output(wait=True)\n",
|
|
|
- " if upload.value:\n",
|
|
|
- " file_info = next(iter(upload.value))\n",
|
|
|
- " image_content = file_info.content\n",
|
|
|
- " try:\n",
|
|
|
- " description = generate_description(image_content)\n",
|
|
|
- " description_text.value = description\n",
|
|
|
- " print(f\"Generated description: {description}\")\n",
|
|
|
- " except Exception as e:\n",
|
|
|
- " print(f\"Error generating description: {str(e)}\")\n",
|
|
|
- " \n",
|
|
|
- " def on_search_click(b):\n",
|
|
|
- " with output:\n",
|
|
|
- " clear_output(wait=True)\n",
|
|
|
- " input_text = description_text.value\n",
|
|
|
- " try:\n",
|
|
|
- " if input_text.startswith(\"Generated description:\"):\n",
|
|
|
- " # For generated descriptions, suggest complementary clothes\n",
|
|
|
- " query = suggest_complementary_clothes(input_text)\n",
|
|
|
- " print(f\"Suggesting complementary clothes for: {input_text}\")\n",
|
|
|
- " print(f\"Query for complementary items: {query}\")\n",
|
|
|
- " else:\n",
|
|
|
- " # For human input, process it to create a query\n",
|
|
|
- " query = process_human_input(input_text)\n",
|
|
|
- " print(f\"Processing human input: {input_text}\")\n",
|
|
|
- " print(f\"Generated query: {query}\")\n",
|
|
|
- " \n",
|
|
|
- " results = retrieve_similar_items(query)\n",
|
|
|
- " show_images(results, output)\n",
|
|
|
- " except Exception as e:\n",
|
|
|
- " print(f\"Error processing input or retrieving results: {str(e)}\")\n",
|
|
|
- " \n",
|
|
|
- " upload.observe(on_upload_change, names='value')\n",
|
|
|
- " search_button.on_click(on_search_click)\n",
|
|
|
- " \n",
|
|
|
- " display(VBox([upload, description_text, search_button, output]))\n",
|
|
|
- "\n",
|
|
|
- "# Call this function to start the fashion assistant\n",
|
|
|
- "create_fashion_assistant()"
|
|
|
- ]
|
|
|
- },
|
|
|
- {
|
|
|
- "cell_type": "code",
|
|
|
"execution_count": null,
|
|
|
"id": "f9b82b17-3270-4b37-80d0-e989e13c4fea",
|
|
|
"metadata": {},
|
|
|
"outputs": [],
|
|
|
- "source": []
|
|
|
+ "source": [
|
|
|
+ "#fin"
|
|
|
+ ]
|
|
|
}
|
|
|
],
|
|
|
"metadata": {
|
Sanyam Bhutani