{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b7cabb96-2715-492e-825a-8f4ff161dc3b",
   "metadata": {},
   "source": [
    "## This demo app shows:\n",
    "* how to run Llama2 locally on a Mac using llama-cpp-python and the llama-cpp's quantized Llama2 model;\n",
    "* how to use LangChain to ask Llama general questions;\n",
    "* how to use LangChain to load a recent PDF doc - the Llama2 paper pdf - and ask questions about it. This is the well known RAG (Retrieval Augmented Generation) method to let LLM such as Llama2 be able to answer questions about the data not publicly available when Llama2 was trained, or about your own data. RAG is one way to prevent LLM's hallucination. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2922732e-29e8-4ea7-8828-53364f5bf6fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: llama-cpp-python in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (0.2.11)\n",
      "Requirement already satisfied: typing-extensions>=4.5.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from llama-cpp-python) (4.8.0)\n",
      "Requirement already satisfied: numpy>=1.20.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from llama-cpp-python) (1.24.4)\n",
      "Requirement already satisfied: diskcache>=5.6.1 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from llama-cpp-python) (5.6.3)\n",
      "Requirement already satisfied: pypdf in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (3.16.2)\n",
      "Requirement already satisfied: sentence-transformers in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (2.2.2)\n",
      "Requirement already satisfied: chromadb in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (0.4.13)\n",
      "Requirement already satisfied: langchain in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (0.0.308)\n",
      "Requirement already satisfied: typing_extensions>=3.7.4.3 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from pypdf) (4.8.0)\n",
      "Requirement already satisfied: transformers<5.0.0,>=4.6.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from sentence-transformers) (4.34.0)\n",
      "Requirement already satisfied: tqdm in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from sentence-transformers) (4.66.1)\n",
      "Requirement already satisfied: torch>=1.6.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from sentence-transformers) (2.1.0)\n",
      "Requirement already satisfied: torchvision in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from sentence-transformers) (0.16.0)\n",
      "Requirement already satisfied: numpy in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from sentence-transformers) (1.24.4)\n",
      "Requirement already satisfied: scikit-learn in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from sentence-transformers) (1.3.1)\n",
      "Requirement already satisfied: scipy in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from sentence-transformers) (1.10.1)\n",
      "Requirement already satisfied: nltk in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from sentence-transformers) (3.8.1)\n",
      "Requirement already satisfied: sentencepiece in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from sentence-transformers) (0.1.99)\n",
      "Requirement already satisfied: huggingface-hub>=0.4.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from sentence-transformers) (0.16.4)\n",
      "Requirement already satisfied: requests>=2.28 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (2.31.0)\n",
      "Requirement already satisfied: pydantic>=1.9 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (2.4.2)\n",
      "Requirement already satisfied: chroma-hnswlib==0.7.3 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (0.7.3)\n",
      "Requirement already satisfied: fastapi>=0.95.2 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (0.103.2)\n",
      "Requirement already satisfied: uvicorn[standard]>=0.18.3 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (0.23.2)\n",
      "Requirement already satisfied: posthog>=2.4.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (3.0.2)\n",
      "Requirement already satisfied: pulsar-client>=3.1.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (3.3.0)\n",
      "Requirement already satisfied: onnxruntime>=1.14.1 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (1.16.0)\n",
      "Requirement already satisfied: tokenizers>=0.13.2 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (0.14.0)\n",
      "Requirement already satisfied: pypika>=0.48.9 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (0.48.9)\n",
      "Requirement already satisfied: overrides>=7.3.1 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (7.4.0)\n",
      "Requirement already satisfied: importlib-resources in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (6.1.0)\n",
      "Requirement already satisfied: bcrypt>=4.0.1 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (4.0.1)\n",
      "Requirement already satisfied: typer>=0.9.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (0.9.0)\n",
      "Requirement already satisfied: graphlib-backport>=1.0.3 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from chromadb) (1.0.3)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from langchain) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from langchain) (2.0.21)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from langchain) (3.8.5)\n",
      "Requirement already satisfied: anyio<4.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from langchain) (3.7.1)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from langchain) (4.0.3)\n",
      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from langchain) (0.6.1)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from langchain) (1.33)\n",
      "Requirement already satisfied: langsmith<0.1.0,>=0.0.40 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from langchain) (0.0.41)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from langchain) (8.2.3)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (3.3.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
      "Requirement already satisfied: idna>=2.8 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from anyio<4.0->langchain) (3.4)\n",
      "Requirement already satisfied: sniffio>=1.1 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from anyio<4.0->langchain) (1.3.0)\n",
      "Requirement already satisfied: exceptiongroup in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from anyio<4.0->langchain) (1.1.3)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (3.20.1)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (0.9.0)\n",
      "Requirement already satisfied: starlette<0.28.0,>=0.27.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from fastapi>=0.95.2->chromadb) (0.27.0)\n",
      "Requirement already satisfied: filelock in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (3.12.4)\n",
      "Requirement already satisfied: fsspec in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (2023.9.2)\n",
      "Requirement already satisfied: packaging>=20.9 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (23.2)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from jsonpatch<2.0,>=1.33->langchain) (2.4)\n",
      "Requirement already satisfied: coloredlogs in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from onnxruntime>=1.14.1->chromadb) (15.0.1)\n",
      "Requirement already satisfied: flatbuffers in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from onnxruntime>=1.14.1->chromadb) (23.5.26)\n",
      "Requirement already satisfied: protobuf in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from onnxruntime>=1.14.1->chromadb) (4.24.4)\n",
      "Requirement already satisfied: sympy in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from onnxruntime>=1.14.1->chromadb) (1.12)\n",
      "Requirement already satisfied: six>=1.5 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from posthog>=2.4.0->chromadb) (1.16.0)\n",
      "Requirement already satisfied: monotonic>=1.5 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from posthog>=2.4.0->chromadb) (1.6)\n",
      "Requirement already satisfied: backoff>=1.10.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from posthog>=2.4.0->chromadb) (2.2.1)\n",
      "Requirement already satisfied: python-dateutil>2.1 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from posthog>=2.4.0->chromadb) (2.8.2)\n",
      "Requirement already satisfied: certifi in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from pulsar-client>=3.1.0->chromadb) (2023.7.22)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from pydantic>=1.9->chromadb) (0.5.0)\n",
      "Requirement already satisfied: pydantic-core==2.10.1 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from pydantic>=1.9->chromadb) (2.10.1)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from requests>=2.28->chromadb) (2.0.6)\n",
      "Requirement already satisfied: networkx in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from torch>=1.6.0->sentence-transformers) (3.1)\n",
      "Requirement already satisfied: jinja2 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from torch>=1.6.0->sentence-transformers) (3.1.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (2023.10.3)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (0.3.3)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from typer>=0.9.0->chromadb) (8.1.7)\n",
      "Requirement already satisfied: h11>=0.8 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.14.0)\n",
      "Requirement already satisfied: httptools>=0.5.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.6.0)\n",
      "Requirement already satisfied: python-dotenv>=0.13 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.0.0)\n",
      "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.17.0)\n",
      "Requirement already satisfied: watchfiles>=0.13 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.20.0)\n",
      "Requirement already satisfied: websockets>=10.4 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (11.0.3)\n",
      "Requirement already satisfied: zipp>=3.1.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from importlib-resources->chromadb) (3.17.0)\n",
      "Requirement already satisfied: joblib in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from nltk->sentence-transformers) (1.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from scikit-learn->sentence-transformers) (3.2.0)\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from torchvision->sentence-transformers) (10.0.1)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain) (1.0.0)\n",
      "Requirement already satisfied: humanfriendly>=9.1 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from coloredlogs->onnxruntime>=1.14.1->chromadb) (10.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from jinja2->torch>=1.6.0->sentence-transformers) (2.1.3)\n",
      "Requirement already satisfied: mpmath>=0.19 in /Users/jeffxtang/anaconda3/envs/llama-demo-apps/lib/python3.8/site-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n"
     ]
    }
   ],
   "source": [
    "# install all the required packages for the demo\n",
    "!CMAKE_ARGS=\"-DLLAMA_METAL=on\" FORCE_CMAKE=1 pip install llama-cpp-python\n",
    "!pip install pypdf sentence-transformers chromadb langchain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "26bc4912",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.llms import LlamaCpp\n",
    "from langchain.chains import LLMChain\n",
    "from langchain.callbacks.manager import CallbackManager\n",
    "from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler\n",
    "from langchain.prompts import PromptTemplate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "01fe5b9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for token-wise streaming so you'll see the answer gets generated token by token when Llama is answering your question\n",
    "callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dff6aa6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create the Llama2 model - for more info see https://python.langchain.com/docs/integrations/llms/llamacpp\n",
    "llm = LlamaCpp(\n",
    "    model_path=\"<path-to-llama-gguf-file>\"\n",
    "    temperature=0.0,\n",
    "    top_p=1,\n",
    "    n_ctx=6000,\n",
    "    callback_manager=callback_manager, \n",
    "    verbose=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0e78549c-9c93-4bc2-b525-38d578a94fae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "The book \"The Innovator's Dilemma\" was written by Clayton Christensen, a professor at Harvard Business School. It was first published in 1997 and has since become a widely influential book on business strategy and innovation."
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =  1202.24 ms\n",
      "llama_print_timings:      sample time =    46.44 ms /    58 runs   (    0.80 ms per token,  1249.03 tokens per second)\n",
      "llama_print_timings: prompt eval time =  1815.15 ms /    15 tokens (  121.01 ms per token,     8.26 tokens per second)\n",
      "llama_print_timings:        eval time =  5582.64 ms /    57 runs   (   97.94 ms per token,    10.21 tokens per second)\n",
      "llama_print_timings:       total time =  7545.78 ms\n"
     ]
    }
   ],
   "source": [
    "# the simplest way to ask Llama some general questions\n",
    "question = \"who wrote the book Innovator's dilemma?\"\n",
    "answer = llm(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f7305c5b-6f55-4664-9206-2d7467653498",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Clayton Christensen is the author of \"The Innovator's Dilemma,\" which was first published in 1997. The book explores why successful companies often struggle to adapt to disruptive technologies and business models that ultimately lead to their downfall. Christensen argues that these companies are faced with a dilemma because they have invested so heavily in their existing products and processes that it is difficult for them to pivot and embrace new, disruptive technologies. He also introduces the concept of \"disruptive innovation,\" which he defines as a process by which a small company with limited resources is able to successfully challenge established industry leaders."
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =  1202.24 ms\n",
      "llama_print_timings:      sample time =   116.69 ms /   147 runs   (    0.79 ms per token,  1259.79 tokens per second)\n",
      "llama_print_timings: prompt eval time =  1180.31 ms /     8 tokens (  147.54 ms per token,     6.78 tokens per second)\n",
      "llama_print_timings:        eval time = 13192.98 ms /   147 runs   (   89.75 ms per token,    11.14 tokens per second)\n",
      "llama_print_timings:       total time = 14746.13 ms\n"
     ]
    }
   ],
   "source": [
    "# a more flexible way to ask Llama general questions using LangChain's PromptTemplate and LLMChain\n",
    "prompt = PromptTemplate.from_template(\n",
    "    \"who wrote {book}?\"\n",
    ")\n",
    "chain = LLMChain(llm=llm, prompt=prompt)\n",
    "answer = chain.run(\"innovator's dilemma\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8ba66a29-77e9-4149-9523-63a09545584e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Llama2 is a free, open-source tool for generating high-quality, randomized test data for software applications. It is designed to be easy to use and to produce realistic, diverse test data that can help you identify and fix bugs in your application before it is released.\n",
      "\n",
      "Llama2 is the successor to the popular Llama tool, and it includes many new features and improvements over its predecessor. Some of the key features of Llama2 include:\n",
      "\n",
      "* Support for a wide range of data types, including strings, numbers, dates, and more\n",
      "* The ability to generate random data based on user-defined rules and constraints\n",
      "* A powerful and flexible API that allows you to customize and extend the tool to meet your specific needs\n",
      "* Integration with popular testing frameworks and tools, such as JUnit and TestNG\n",
      "* Support for a variety of programming languages, including Java, Python, C#, and more.\n",
      "\n",
      "Overall, Llama2 is a powerful and flexible tool that can help you improve the quality and reliability of your software applications by generating realistic and diverse test data."
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =  1202.24 ms\n",
      "llama_print_timings:      sample time =   191.25 ms /   240 runs   (    0.80 ms per token,  1254.87 tokens per second)\n",
      "llama_print_timings: prompt eval time =   480.79 ms /     6 tokens (   80.13 ms per token,    12.48 tokens per second)\n",
      "llama_print_timings:        eval time = 22013.19 ms /   239 runs   (   92.11 ms per token,    10.86 tokens per second)\n",
      "llama_print_timings:       total time = 23111.55 ms\n"
     ]
    }
   ],
   "source": [
    "# let's see how Llama2 hallucinates, because it doesn't have the knowledge about Llama2 while the model was trained, \n",
    "# but by default it behaves like a know-it-all expert who can't afford to say I don't know\n",
    "prompt = PromptTemplate.from_template(\n",
    "    \"What is {what}?\"\n",
    ")\n",
    "chain = LLMChain(llm=llm, prompt=prompt)\n",
    "answer = chain.run(\"llama2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f3ebc261",
   "metadata": {},
   "outputs": [],
   "source": [
    "# to fix the LLM's hallucination, one way is to use RAG, to augment it with more recent or custom data that holds the info for it to answer correctly\n",
    "# first load the Llama2 paper via the LangChain's PDF loader\n",
    "from langchain.document_loaders import PyPDFLoader\n",
    "loader = PyPDFLoader(\"llama2.pdf\")\n",
    "documents = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "302eaa54",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "77 Llama 2 : Open Foundation and Fine-Tuned Chat Models\n",
      "Hugo Touvron∗Louis Martin†Kevin Stone†\n",
      "Peter Albert Amjad Almahairi Yasmine Babaei Nikolay Bashlykov Soumya Batra\n",
      "Prajjwal Bhargava Shruti Bhosale Dan Bikel Lukas Blecher Cristian Canton Ferrer Moya Chen\n",
      "Guillem Cucurull David Esiobu Jude Fernande\n"
     ]
    }
   ],
   "source": [
    "# quick check on the loaded document for the correct pages etc\n",
    "print(len(documents), documents[0].page_content[0:300])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4f94f6f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# there're more 30 vector stores (DBs) supported by LangChain. Chroma is light-weight and in memory so it's easy to get started with\n",
    "# other vector stores can be used to store large amount of data - see https://python.langchain.com/docs/integrations/vectorstores\n",
    "from langchain.vectorstores import Chroma\n",
    "\n",
    "# embeddings are numerical representations of the question and answer text\n",
    "from langchain.embeddings import HuggingFaceEmbeddings\n",
    "\n",
    "# use a common text splitter to split text into chunks\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2b101485",
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the loaded documents into chunks \n",
    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=20)\n",
    "all_splits = text_splitter.split_documents(documents)\n",
    "\n",
    "# create the vector db to store all the split chunks as embeddings\n",
    "embeddings = HuggingFaceEmbeddings()\n",
    "vectordb = Chroma.from_documents(\n",
    "    documents=all_splits,\n",
    "    embedding=embeddings,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1a2472c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Llama.generate: prefix-match hit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Llama 2 is a new language model developed by Meta AI that has been released openly to encourage responsible AI innovation. It is a fine-tuned version of the original Llama model and is optimized for dialogue use cases. The model has not covered all scenarios and may produce inaccurate or objectionable responses, so developers should perform safety testing and tuning before deploying any applications of Llama 2."
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =  1202.24 ms\n",
      "llama_print_timings:      sample time =    76.83 ms /    97 runs   (    0.79 ms per token,  1262.48 tokens per second)\n",
      "llama_print_timings: prompt eval time = 97067.98 ms /  1146 tokens (   84.70 ms per token,    11.81 tokens per second)\n",
      "llama_print_timings:        eval time = 10431.81 ms /    96 runs   (  108.66 ms per token,     9.20 tokens per second)\n",
      "llama_print_timings:       total time = 107897.31 ms\n"
     ]
    }
   ],
   "source": [
    "# use another LangChain's chain, RetrievalQA, to associate Llama with the loaded documents stored in the vector db\n",
    "from langchain.chains import RetrievalQA\n",
    "\n",
    "qa_chain = RetrievalQA.from_chain_type(\n",
    "    llm,\n",
    "    retriever=vectordb.as_retriever()\n",
    ")\n",
    "\n",
    "# for each question, LangChain performs a semantic similarity search of it in the vector db, then passes the search results as the context\n",
    "# the Llama to answer question about the data stored in the verctor db\n",
    "question = \"What is llama2?\"\n",
    "result = qa_chain({\"query\": question})\n",
    "# it takes close to 2 minutes to return the result (but using other vector store than Chroma such as FAISS can take longer), because \n",
    "# Llama2 is running on a local Mac. To get much faster results, you can use a cloud service with GPU used for inference - see HelloLlamaCloud \n",
    "# for a demo."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd2e62a4-6ea2-4ea7-b7ae-800185177e6c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
