{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "60b5a18c-4465-469d-8597-297679c9e3bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "_ = load_dotenv(find_dotenv())\n",
    "openai_api_key = os.environ[\"OPENAI_API_KEY\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f2e410c-d63d-4b67-8237-6985ec2b35d7",
   "metadata": {},
   "source": [
    "## Basic App for Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fa1fa918-efca-4979-9edf-fd0fdab47daf",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain import OpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aac08fc0-c981-4e48-8abd-97902f81fab4",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = OpenAI()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "597e116d-4cea-4e8c-b2c3-8c24d7dd84c5",
   "metadata": {},
   "source": [
    "**Load document**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e396ce7e-4548-4f73-b055-083dbf3a8ce9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.document_loaders import TextLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "edb24dae-cbbc-4bd1-a149-3de8285145fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "loader = TextLoader(\"data/be-good-and-how-not-to-die.txt\")\n",
    "document = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "70586faa-0f6d-4535-a450-57906420b99e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The document has 27419 characters\n"
     ]
    }
   ],
   "source": [
    "print(f\"The document has {len(document[0].page_content)} characters\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e8cbf6a-8dfd-4064-bc39-6119c543a29b",
   "metadata": {},
   "source": [
    "**Split the document in smaller chunks**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "92d16563-054d-4685-a937-5092927d8c81",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8c56d568-2723-4592-a183-9e2950b6d0e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=3000,\n",
    "    chunk_overlap=400\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6bba8440-5cb5-43e1-a8bc-23bfcf3ab519",
   "metadata": {},
   "outputs": [],
   "source": [
    "document_chunks = text_splitter.split_documents(document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "89b97356-4207-4457-9a12-dc8c81f438fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Now you have 12 chunks\n"
     ]
    }
   ],
   "source": [
    "print(f\"Now you have {len(document_chunks)} chunks\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6c0ba39a-065e-4ee2-9b27-9d878a468802",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n"
     ]
    }
   ],
   "source": [
    "print(type(document_chunks))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec119f5c-4f5f-45ec-9876-838bb50a994a",
   "metadata": {},
   "source": [
    "**Convert text chunks in numeric embeddings and load them to the vector database**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6897127b-c127-44d7-b9cc-979dc9a130ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.embeddings.openai import OpenAIEmbeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "cbfe2eb7-6a0d-4276-99e2-ccfc1c0cea29",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = OpenAIEmbeddings()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "56734fef-a704-4d97-a579-0fa44db4914a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.vectorstores import FAISS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "42bb90ea-635e-41c3-a13c-22a3d1ff50e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "stored_embeddings = FAISS.from_documents(document_chunks, embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba518215-4044-4780-ba07-d3f16506f809",
   "metadata": {},
   "source": [
    "**Create a Retrieval Question & Answering Chain**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a88d8dd3-e857-4f9d-bba3-46115c0e35f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chains import RetrievalQA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ae3925ba-35f7-4af9-a6d4-553418f0018c",
   "metadata": {},
   "outputs": [],
   "source": [
    "QA_chain = RetrievalQA.from_chain_type(\n",
    "    llm=llm,\n",
    "    chain_type=\"stuff\",\n",
    "    retriever=stored_embeddings.as_retriever(),\n",
    "    input_key=\"question\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8265d5bc-e450-48ae-b7aa-d464a442ca95",
   "metadata": {},
   "source": [
    "Notice that we have added input_key in the QA_chain configuration. This tells the chain where will the user prompt be located."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7710d43a-61b3-414f-9832-cf9d0a6133e1",
   "metadata": {},
   "source": [
    "**We are going to evaluate this app with 2 questions and answers we already know (these answers are technically known as \"ground truth answers\")**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "22caba6d-258c-4060-8c0c-22663638ff44",
   "metadata": {},
   "outputs": [],
   "source": [
    "questions_and_answers = [\n",
    "    {\n",
    "        'question' : \"Where is a whole neighborhood of YC-funded startups?\", \n",
    "        'answer' :\"In San Francisco\"},\n",
    "    {\n",
    "        'question' : \"What may be the most valuable  thing Paul Buchheit made for Google?\", \n",
    "        'answer' : \"The motto Don't be evil\"}\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3cf0e7b8-7420-4b66-ae2f-e172f0c10b3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = QA_chain.apply(questions_and_answers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c98131ca-6c69-4b16-8a9a-576d06a319a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'question': 'Where is a whole neighborhood of YC-funded startups?',\n",
       "  'answer': 'In San Francisco',\n",
       "  'result': ' A whole neighborhood of YC-funded startups is in San Francisco.'},\n",
       " {'question': 'What may be the most valuable  thing Paul Buchheit made for Google?',\n",
       "  'answer': \"The motto Don't be evil\",\n",
       "  'result': ' Paul Buchheit is credited with creating the phrase \"Don\\'t be evil,\" which serves as the motto for Google. This phrase may be the most valuable thing Buchheit made for Google, as it serves as a reminder for the company to stay true to its mission.'}]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43e579ee-1f76-4cec-9b3c-15867de4a577",
   "metadata": {},
   "source": [
    "**The evaluation of this App has been positive, since the App has responded the 2 evaluation questions right.**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "824d5775-0930-453a-a8b2-b973755be163",
   "metadata": {},
   "source": [
    "**But instead of confirming that manually ourselves, we can ask the LLM to check if the responses are coincidental with the \"ground truth answers\"**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "81694c68-656c-4abc-b3b0-b9ec2b7926d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.evaluation.qa import QAEvalChain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "d173468e-42ad-477b-b065-d227de7d587d",
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluation_chain = QAEvalChain.from_llm(llm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4f198223-79f3-4029-a1ce-fc45fe61cfae",
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluate_responses = evaluation_chain.evaluate(\n",
    "    questions_and_answers,\n",
    "    predictions,\n",
    "    question_key=\"question\",\n",
    "    answer_key=\"answer\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "c5d86b20-5ca4-4bf7-887d-16a96f4da06c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'results': ' CORRECT'}, {'results': ' CORRECT'}]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00cd019f-01b4-4840-b49f-24e46e36306e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}