initial commit

.gitignore

@@ -0,0 +1,68 @@
+flagged
+
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Poetry-specific files
+.poetry/
+
+# Virtual environment
+# If you are using a virtual environment created by Poetry inside the project directory.
+# Uncomment if using virtualenvs.in-project=true in pyproject.toml
+# .venv/
+
+# Distribution / packaging
+build/
+dist/
+*.egg-info/
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# PyInstaller
+# Usually these files are written to this folder when PyInstaller is used to package a Python project
+*.manifest
+*.spec
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+nosetests.xml
+
+# Jupyter Notebook
+.ipynb_checkpoints/
+
+# pyenv
+# This file includes specific Python version, ignore if irrelevant
+.python-version
+
+# dotenv
+.env
+.env.*
+
+# Editor-specific files
+.vscode/
+.idea/
+*.sublime-project
+*.sublime-workspace
+
+# macOS files
+.DS_Store
+
+# Windows files
+Thumbs.db
+desktop.ini
+
+# Python virtual environments
+venv/

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.12
+
+RUN pip install poetry
+
+WORKDIR /app
+
+COPY . .
+
+RUN poetry install
+
+ENV GRADIO_SERVER_NAME="0.0.0.0"
+EXPOSE 7860
+
+CMD ["poetry", "run", "python", "-m", "interface.py"]

README.md

@@ -0,0 +1,15 @@
+# IVA
+
+**IVA (Interactive Virtual Assistant)** is an AI-based middleware that communicates with HomeAssistant in order to retrieve values from different sensors.
+
+To run this project, first create a ```.env``` file containing the following variables:
+- **OPENAI_API_KEY**: a OpenAI API key
+- **HA_URL**: the URL of your HomeAssistant app
+- **HA_API_KEY**: your HomeAssistant API key
+
+Build the Docker image using the following command: 
+```docker build -t <image_name> .```
+
+Then, create a container using: 
+```docker run --name <container_name> -p 7860:7860 -it <image_name>```
+

interface.py

@@ -0,0 +1,37 @@
+import gradio as gr
+from iva import IVA
+
+
+def IVA_demo(audio, text):
+    assistant = IVA()
+
+    if audio:
+        print("Am a")
+        transcript = assistant.transcript(audio)
+    else:
+        transcript = text
+
+    query_result = assistant.process_query(transcript)
+    audio_path = assistant.tts(query_result)
+
+    return query_result, audio_path
+
+
+demo = gr.Interface(
+    fn=IVA_demo,
+    inputs=[
+        gr.Audio(
+            type="filepath",
+            sources="microphone",
+            label="Voice input",
+            show_label=True,
+        ),
+        gr.Textbox(info="...or use text input"),
+    ],
+    outputs=[
+        "text",
+        gr.Audio(type="filepath"),
+    ],
+)
+
+demo.launch(share=True)

iva/__init__.py

@@ -0,0 +1,64 @@
+from .tools import toolkit
+from .prompts import system_message
+
+import tempfile
+from openai import OpenAI
+from dotenv import load_dotenv
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import SystemMessage, HumanMessage
+
+load_dotenv()
+
+
+class IVA:
+    def __init__(self):
+        self.llm = ChatOpenAI(
+            model="gpt-4o-mini",
+            temperature=0,
+        ).bind_tools(toolkit)
+
+        self.openai_client = OpenAI()
+
+    def transcript(self, file):
+        audio = open(file, "rb")
+        transcript = self.openai_client.audio.transcriptions.create(
+            model="whisper-1",
+            file=audio,
+            language="ro",
+        )
+
+        print(f"Transcript: {transcript.text}")
+        return transcript.text
+
+    def process_query(self, query):
+        messages = [
+            SystemMessage(system_message),
+            HumanMessage(query),
+        ]
+
+        response = self.llm.invoke(messages)
+        messages.append(response)
+
+        for tool_call in response.tool_calls:
+            selected_tool = {
+                "select_and_get_sensor_value": toolkit[0],
+            }[tool_call["name"].lower()]
+
+            tool_msg = selected_tool.invoke(tool_call)
+            messages.append(tool_msg)
+
+        final_response = self.llm.invoke(messages)
+
+        return final_response.content
+
+    def tts(self, text):
+        response = self.openai_client.audio.speech.create(
+            model="tts-1",
+            voice="nova",
+            input=text,
+        )
+
+        with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as temp_file:
+            temp_file.write(response.content)
+
+        return temp_file.name

iva/prompts.py

@@ -0,0 +1,16 @@
+system_message = """
+Numele tău este Iva.
+Ești un asistent virtual al laboratorului de IoT din cadrul Institutului Național de Cercetare - Dezvoltare în Informatică ICI București.
+Vei primi întrebări legate de diferite valori ale senzorilor din laborator.
+Vei raspunde politicos, clar și concis la întrebări, oferind valoarea senzorului și, dacă este disponibilă, și unitatea de măsură.
+Daca nu ai acces la anumite informații, vei menționă acest aspect.
+Toate răspunsurile pe care le vei da vor fi în limba Română, folosind obligatoriu diacritice.
+"""
+
+choose_sensor_template = """
+Ai următoarea listă de senzori:
+{sensor_info}
+
+Bazându-te pe descrierea utilizatorului: "{description}", alege cel mai potrivit senzor.
+Returnează doar valoarea câmpului entity_id pentru senzorul selectat, fară a returna și cheia.
+"""

iva/tools.py

@@ -0,0 +1,77 @@
+import os
+from dotenv import load_dotenv
+from typing import Annotated
+from homeassistant_api import Client
+from langchain_core.tools import tool
+from langchain_openai import ChatOpenAI
+from .prompts import choose_sensor_template
+from langchain_core.prompts import PromptTemplate
+
+
+load_dotenv()
+client = Client(os.environ["HA_URL"], os.environ["HA_API_KEY"])
+
+
+def get_all_entities():
+    all_entities = client.get_entities()["sensor"].entities
+    final_entities = []
+
+    objects = all_entities.keys()
+
+    for object in objects:
+        final_entities.append(
+            {
+                "entity_id": all_entities[object].state.entity_id,
+                "friendly_name": all_entities[object].state.attributes["friendly_name"],
+            }
+        )
+
+    return final_entities
+
+
+def get_sensor_state(entity_id: Annotated[str, "The ID of the sensor"]):
+    sensor = client.get_entity(entity_id=entity_id)
+
+    return {
+        "value": sensor.state.state,
+        "unit_of_measurement": sensor.state.attributes.get("unit_of_measurement"),
+    }
+
+
+# Tool to dynamically choose the best sensor based on user input
+@tool
+def select_and_get_sensor_value(
+    description: Annotated[str, "Description of the sensor you need"]
+):
+    """
+    Let the LLM decide which sensor to use based on the provided description and extract its value.
+    And get the current state or value of a sensor.
+    Use this when the user asks you for data related to sensors.
+    """
+    sensors = get_all_entities()
+
+    # Prepare sensor details for LLM to analyze
+    sensor_info = "\n".join(
+        [
+            f"- entity_id: {sensor['entity_id']}, friendly_name: {sensor['friendly_name']}"
+            for sensor in sensors
+        ]
+    )
+
+    llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+    prompt = PromptTemplate(
+        template=choose_sensor_template, input_variables=["sensor_info", "description"]
+    )
+    llm_chain = prompt | llm
+
+    selected_sensor = llm_chain.invoke(
+        {"sensor_info": sensor_info, "description": description}
+    )
+
+    # Now retrieve the value of the selected sensor
+    sensor_value = get_sensor_state(selected_sensor.content.strip())
+
+    return sensor_value
+
+
+toolkit = [select_and_get_sensor_value]

pyproject.toml

@@ -0,0 +1,23 @@
+[tool.poetry]
+name = "iva"
+version = "0.1.0"
+description = "Virtual Assistant app"
+authors = ["Sebastian Balmuș <sebastian.balmus@ici.ro>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.12"
+openai = "^1.46.0"
+langchain = "^0.3.0"
+langchain-openai = "^0.2.0"
+homeassistant-api = "^4.2.2.post1"
+python-dotenv = "^1.0.1"
+gradio = "^4.44.0"
+
+[tool.poetry.plugins.dotenv]
+ignore = "false"
+location = "./.env"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"