{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "ea998dc2-12cc-42c6-9973-e9393022a9f9", "metadata": {}, "outputs": [], "source": [ "import os\n", "from dotenv import load_dotenv, find_dotenv\n", "_ = load_dotenv(find_dotenv())\n", "openai_api_key = os.environ[\"OPENAI_API_KEY\"]" ] }, { "cell_type": "markdown", "id": "54d530b7-4777-46bd-a3af-96acd82666c3", "metadata": {}, "source": [ "## LangSmith" ] }, { "cell_type": "markdown", "id": "31e2b3a3-ac1b-4da1-a5d3-6627d40ef87b", "metadata": {}, "source": [ "*Log In at https://smith.langchain.com*" ] }, { "cell_type": "code", "execution_count": 2, "id": "afd836d0-0454-48a3-8836-132df3d9e016", "metadata": {}, "outputs": [], "source": [ "# LANGCHAIN_TRACING_V2=true\n", "# LANGCHAIN_ENDPOINT=https://api.smith.langchain.com\n", "# LANGCHAIN_API_KEY=\n", "# LANGCHAIN_PROJECT= # if not specified, defaults to \"default\"" ] }, { "cell_type": "code", "execution_count": 3, "id": "ea19f948-566b-4677-9d04-034efa5c99ee", "metadata": {}, "outputs": [], "source": [ "#!pip install langsmith" ] }, { "cell_type": "code", "execution_count": 4, "id": "7d0face3-7eee-4b87-9381-c94f1ce984d6", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'Napoleon Bonaparte had three brothers: Joseph Bonaparte, Lucien Bonaparte, and Louis Bonaparte.'" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from langchain_openai import ChatOpenAI\n", "from langchain.callbacks.tracers import LangChainTracer\n", "\n", "llm = ChatOpenAI()\n", "tracer = LangChainTracer(project_name=\"Napoleon v1\")\n", "llm.predict(\"How many brothers had Napoleon Bonaparte?\", callbacks=[tracer])" ] }, { "cell_type": "markdown", "id": "153d81c1-d0ed-42a8-869a-7c7b55fca6fd", "metadata": {}, "source": [ "**See updates in the Projects Area in LangSmith**" ] }, { "cell_type": "markdown", "id": "a36085d6-eceb-4dae-ac3a-5bd38acbc978", "metadata": {}, "source": [ "## Basic LangSmith Operations" ] }, { "cell_type": "markdown", "id": "7572831d-e265-4e2a-aeed-c333825afbfa", "metadata": {}, "source": [ "**Create a new project with LangChainTracer**" ] }, { "cell_type": "code", "execution_count": 5, "id": "01f8cafb-4f13-42e9-857a-ea0c8acfb5d6", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'Winston Churchill was 65 years old when he was appointed as the Prime Minister of the United Kingdom. He assumed office on May 10, 1940.'" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from langchain.callbacks.tracers import LangChainTracer\n", "\n", "tracer = LangChainTracer(project_name=\"Churchill v1\")\n", "llm.predict(\"How old was Churchill when he was appointed PM?\", callbacks=[tracer])" ] }, { "cell_type": "markdown", "id": "c10a63e5-42a6-4671-bddd-925a0ba70e30", "metadata": {}, "source": [ "**Check updates in the Projects Area in LangSmith**" ] }, { "cell_type": "markdown", "id": "b96e77ae-23b8-4c45-aca6-dd3a4592a8d4", "metadata": {}, "source": [ "**Alternative way to do the same**" ] }, { "cell_type": "code", "execution_count": 6, "id": "7057bd7f-ce18-41a1-a774-a65d253d444a", "metadata": {}, "outputs": [], "source": [ "from langchain.callbacks import tracing_v2_enabled\n", "\n", "with tracing_v2_enabled(project_name=\"Cyrus v1\"):\n", " llm.predict(\"When did Cyrus The Great reign in Persia?\")" ] }, { "cell_type": "markdown", "id": "b35a9b37-f05f-4823-8567-3e76d0ace2c7", "metadata": {}, "source": [ "## Creating Tags in LangSmith Projects" ] }, { "cell_type": "code", "execution_count": 7, "id": "8d7ecac2-caa7-463c-b1e1-3ef70b7b9071", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'input': 'When did the first Cyrus king reign in Persia?',\n", " 'text': 'The first Cyrus king, Cyrus the Great, reigned in Persia from 559 BC to 530 BC.'}" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from langchain.chains import LLMChain\n", "from langchain_openai import ChatOpenAI\n", "from langchain.prompts import PromptTemplate\n", "\n", "llm = ChatOpenAI(temperature=0, tags=[\"History\"])\n", "\n", "prompt = PromptTemplate.from_template(\"Say {input}\")\n", "\n", "chain = LLMChain(\n", " llm=llm, \n", " prompt=prompt, \n", " tags=[\"Cyrus\", \"Persia\"])\n", "\n", "chain(\"When did the first Cyrus king reign in Persia?\", tags=[\"Cyrus\"])" ] }, { "cell_type": "markdown", "id": "832b82b7-36ae-4771-8cdd-8c8aaf3c1107", "metadata": {}, "source": [ "*See that this went to the default project since we did not set that differently*" ] }, { "cell_type": "markdown", "id": "4dd4dc45-b3ec-4c30-9219-8129534c04c1", "metadata": {}, "source": [ "## Creating Groups in LangSmith Projects" ] }, { "cell_type": "code", "execution_count": 8, "id": "1c485bfe-8418-43c1-bbb8-c6425acbaeaf", "metadata": {}, "outputs": [], "source": [ "from langchain.callbacks.manager import (\n", " trace_as_chain_group\n", ")\n", "\n", "with trace_as_chain_group(\"American History v1\") as group_manager:\n", " pass\n", "\n", "from langchain.chains import LLMChain\n", "from langchain.prompts import PromptTemplate\n", "\n", "roman_llm = ChatOpenAI(temperature=0.9)\n", "\n", "prompt = PromptTemplate(\n", " input_variables=[\"question\"],\n", " template=\"What is the answer to {question}?\",\n", ")\n", "\n", "chain = LLMChain(\n", " llm=roman_llm, \n", " prompt=prompt\n", ")\n", "\n", "with trace_as_chain_group(\"Roman History v1\") as group_manager:\n", " chain.run(question=\"Who did Julius Caesar marry?\", callbacks=group_manager)\n", " chain.run(question=\"Where did Julius Caesar fight?\", callbacks=group_manager)\n", " chain.run(question=\"What was the name of the horse of Julius Caesar?\", callbacks=group_manager)" ] }, { "cell_type": "markdown", "id": "41bd92c6-9a8f-49f2-9709-bedda1e5b4e6", "metadata": {}, "source": [ "## LangSmith Client" ] }, { "cell_type": "code", "execution_count": 18, "id": "f11f46d9-d947-408d-82ae-8588503243df", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from langsmith import Client\n", "\n", "client = Client()\n", "project_runs = client.list_runs(project_name=\"default\")\n", "project_runs" ] }, { "cell_type": "code", "execution_count": 24, "id": "3d671891-1330-4040-9d2c-6057f287c53c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from datetime import datetime, timedelta\n", "\n", "todays_runs = client.list_runs(\n", " project_name=\"default\",\n", " start_time=datetime.now() - timedelta(days=1),\n", " run_type=\"llm\",\n", ")\n", "todays_runs" ] }, { "cell_type": "code", "execution_count": 21, "id": "9bf04f18-46ab-4332-b377-185329c43fa1", "metadata": {}, "outputs": [], "source": [ "# for run in todays_runs:\n", "# print(run)" ] }, { "cell_type": "code", "execution_count": 22, "id": "36f1debf-da80-482e-abf9-6aaaca02bfad", "metadata": {}, "outputs": [], "source": [ "# todays_runs = client.list_runs(\n", "# project_name=\"Churchill v1\",\n", "# start_time=datetime.now() - timedelta(days=1),\n", "# run_type=\"llm\",\n", "# )\n", "\n", "# for run in todays_runs:\n", "# print(run)" ] }, { "cell_type": "markdown", "id": "06f5ff50-9fd8-42aa-b9a3-cce04c9449b8", "metadata": {}, "source": [ "## Adding metadata to filter runs\n", "One possible use of this: making A/B tests." ] }, { "cell_type": "code", "execution_count": 13, "id": "05f0807d-ddea-4ae7-a463-d6e415af7448", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'input': 'Who was the companion of Don Quixote?',\n", " 'text': 'The main companion of Don Quixote in the novel \"Don Quixote\" by Miguel de Cervantes is Sancho Panza.'}" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "chat_model = ChatOpenAI()\n", "chain = LLMChain.from_string(\n", " llm=chat_model, \n", " template=\"What's the answer to {input}?\")\n", "\n", "chain(\n", " {\"input\": \"Who was the companion of Don Quixote?\"}, \n", " metadata={\"source\": \"Cervantes\"}\n", ")" ] }, { "cell_type": "code", "execution_count": 25, "id": "c600284e-c2a5-4a0c-85a1-a31ffbbcb477", "metadata": {}, "outputs": [], "source": [ "runs = list(client.list_runs(\n", " project_name=\"default\",\n", " filter='has(metadata, \\'{\"source\": \"Cervantes\"}\\')',\n", "))\n", "\n", "#print(list(runs))" ] }, { "cell_type": "markdown", "id": "978e63f8-8c49-43b5-88f9-42c21e6a645f", "metadata": {}, "source": [ "## Evaluating your LLM App with a Test Dataset in LangSmith" ] }, { "cell_type": "code", "execution_count": 15, "id": "37619317-242f-4a52-b05e-71ceaa8ac5d1", "metadata": {}, "outputs": [], "source": [ "from langsmith import Client\n", "\n", "example_inputs = [\n", " (\"What is the largest mammal?\", \"The blue whale\"),\n", " (\"What do mammals and birds have in common?\", \"They are both warm-blooded\"),\n", " (\"What are reptiles known for?\", \"Having scales\"),\n", " (\"What's the main characteristic of amphibians?\", \"They live both in water and on land\"),\n", "]\n", "\n", "client = Client()\n", "\n", "dataset_name = \"Elementary Animal Questions v1\"\n", "\n", "# Storing inputs in a dataset lets us\n", "# run chains and LLMs over a shared set of examples.\n", "dataset = client.create_dataset(\n", " dataset_name=dataset_name, \n", " description=\"Questions and answers about animal phylogenetics.\",\n", ")\n", "\n", "for input_prompt, output_answer in example_inputs:\n", " client.create_example(\n", " inputs={\"question\": input_prompt},\n", " outputs={\"answer\": output_answer},\n", " dataset_id=dataset.id,\n", " )" ] }, { "cell_type": "code", "execution_count": 16, "id": "73cca294-91b2-44a8-9ac0-99e681ca7f78", "metadata": {}, "outputs": [], "source": [ "from langsmith import Client\n", "from langchain.smith import RunEvalConfig, run_on_dataset\n", "\n", "evaluation_config = RunEvalConfig(\n", " evaluators=[\n", " \"qa\",\n", " \"context_qa\",\n", " \"cot_qa\",\n", " ]\n", ")" ] }, { "cell_type": "code", "execution_count": 17, "id": "8577490a-fbab-4c08-98dd-7690df4a6542", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "View the evaluation results for project 'evalproject v1' at:\n", "https://smith.langchain.com/o/ec6a7494-139b-5170-8044-143bc78622a9/projects/p/72b8d49b-782d-44dd-81be-6a6970762986?eval=true\n", "\n", "View all tests for Dataset Elementary Animal Questions v1 at:\n", "https://smith.langchain.com/datasets/fd0aa228-8310-4082-b949-776429b7eac3\n", "[------------------------------------------------->] 4/4" ] }, { "data": { "text/plain": [ "{'project_name': 'evalproject v1',\n", " 'results': {'303e45d0-2324-48ca-af44-bcf9884cdd32': {'output': 'The main characteristic of amphibians is their ability to live both in water and on land. They have a dual life cycle, starting as aquatic larvae (such as tadpoles) and then transforming into terrestrial adults. Amphibians also have moist, permeable skin, which allows them to breathe through their skin. They typically lay their eggs in water and undergo metamorphosis during their development.',\n", " 'input': {'question': \"What's the main characteristic of amphibians?\"},\n", " 'feedback': [EvaluationResult(key='correctness', score=1, value='CORRECT', comment='CORRECT', correction=None, evaluator_info={'__run': RunInfo(run_id=UUID('b46ee973-6887-4450-94a9-af85613ea29d'))}, source_run_id=None, target_run_id=None),\n", " EvaluationResult(key='Contextual Accuracy', score=1, value='CORRECT', comment='CORRECT', correction=None, evaluator_info={'__run': RunInfo(run_id=UUID('910e6d60-29ee-4f53-aaa5-ebd5cb57ad71'))}, source_run_id=None, target_run_id=None),\n", " EvaluationResult(key='COT Contextual Accuracy', score=1, value='CORRECT', comment=\"The student's answer correctly identifies the main characteristic of amphibians as their ability to live both in water and on land, which aligns with the context provided. The additional information provided by the student about the life cycle, skin, and reproduction of amphibians is also accurate and does not conflict with the context. Therefore, the student's answer is factually correct.\\nGRADE: CORRECT\", correction=None, evaluator_info={'__run': RunInfo(run_id=UUID('49498807-e027-4130-a109-47851095e578'))}, source_run_id=None, target_run_id=None)],\n", " 'reference': {'answer': 'They live both in water and on land'}},\n", " '9c7870a5-cbc5-4e44-8cd0-03653de9dab2': {'output': 'Reptiles are known for several characteristics:\\n\\n1. Cold-blooded: Reptiles are ectothermic animals, meaning they cannot regulate their body temperature internally. They rely on external sources of heat, such as sunlight, to warm their bodies.\\n\\n2. Scales: Reptiles have dry, scaly skin that helps prevent water loss and protects them from the environment. These scales can vary in texture and appearance, from smooth to rough and from brightly colored to camouflaged.\\n\\n3. Laying eggs: Most reptiles lay eggs, although some species give birth to live young. The eggs are typically leathery-shelled and are deposited in nests or buried in the ground.\\n\\n4. Terrestrial and aquatic habitats: Reptiles occupy a wide range of habitats, including deserts, forests, grasslands, and water bodies such as rivers, lakes, and oceans. Some reptiles are adapted to live in both aquatic and terrestrial environments.\\n\\n5. Breathe through lungs: Reptiles have lungs for breathing, unlike amphibians that also rely on their skin for respiration. They have specialized respiratory systems that allow them to efficiently extract oxygen from the air.\\n\\n6. Predators: Reptiles are often carnivorous, feeding on a variety of prey including insects, small mammals, birds, fish, and other reptiles. Some larger reptiles, like crocodilians, can even prey on large mammals.\\n\\n7. Diversity: Reptiles are a diverse group, including various orders such as snakes, lizards, turtles, crocodilians, and tuataras. They come in a wide range of sizes, shapes, and colors, exhibiting different adaptations and behaviors.\\n\\n8. Longevity: Many reptiles have long lifespans compared to other animals. Some tortoise species, for example, can live for more than 100 years.\\n\\n9. Ectothermic metabolism: Reptiles have a slower metabolic rate compared to warm-blooded animals. This allows them to survive on lower energy requirements and go for extended periods without food.\\n\\n10. Ancient lineage: Reptiles are descendants of some of the earliest land-dwelling vertebrates. They have been on Earth for millions of years and have evolved numerous adaptations to thrive in diverse environments.',\n", " 'input': {'question': 'What are reptiles known for?'},\n", " 'feedback': [EvaluationResult(key='Contextual Accuracy', score=1, value='CORRECT', comment='CORRECT', correction=None, evaluator_info={'__run': RunInfo(run_id=UUID('15750b38-b46d-4e14-a983-aa1102bf6ed6'))}, source_run_id=None, target_run_id=None),\n", " EvaluationResult(key='correctness', score=1, value='CORRECT', comment='CORRECT', correction=None, evaluator_info={'__run': RunInfo(run_id=UUID('0e4886d3-9b8c-4b35-9eb0-f7022660c25c'))}, source_run_id=None, target_run_id=None),\n", " EvaluationResult(key='COT Contextual Accuracy', score=1, value='CORRECT', comment=\"The student's answer includes the fact that reptiles are known for having scales, which is the context provided. The student also provides additional accurate information about reptiles, such as being cold-blooded, laying eggs, living in various habitats, breathing through lungs, being predators, their diversity, longevity, ectothermic metabolism, and ancient lineage. None of this additional information contradicts the context provided. Therefore, the student's answer is factually accurate.\\nGRADE: CORRECT\", correction=None, evaluator_info={'__run': RunInfo(run_id=UUID('5f948ca6-76c8-432d-b3f4-f7d532f8bee9'))}, source_run_id=None, target_run_id=None)],\n", " 'reference': {'answer': 'Having scales'}},\n", " '44af2bb9-c4d2-46fd-99a4-4e06f4e0a563': {'output': 'Mammals and birds are both vertebrate animals, meaning they have a backbone or spinal cord. Additionally, they are warm-blooded animals, maintaining a constant internal body temperature. Both mammals and birds have lungs for respiration and possess a four-chambered heart. They also have a relatively high metabolic rate and possess specialized adaptations for reproduction, such as giving birth to live young (in most mammals) or laying eggs (in birds). Furthermore, both mammals and birds exhibit diverse ecological adaptations and have evolved various locomotion methods, including walking, running, swimming, and flying.',\n", " 'input': {'question': 'What do mammals and birds have in common?'},\n", " 'feedback': [EvaluationResult(key='Contextual Accuracy', score=1, value='CORRECT', comment='CORRECT', correction=None, evaluator_info={'__run': RunInfo(run_id=UUID('0a86d842-2c4c-4fd5-ab08-d121bbadfabf'))}, source_run_id=None, target_run_id=None),\n", " EvaluationResult(key='correctness', score=1, value='CORRECT', comment='CORRECT', correction=None, evaluator_info={'__run': RunInfo(run_id=UUID('2c232276-4a6b-44ed-a3a4-851ad200717d'))}, source_run_id=None, target_run_id=None),\n", " EvaluationResult(key='COT Contextual Accuracy', score=1, value='CORRECT', comment=\"The student's answer is factually correct. The student correctly identifies that both mammals and birds are warm-blooded, which is the information provided in the context. The student also provides additional accurate information about the similarities between mammals and birds, such as being vertebrates, having lungs for respiration, possessing a four-chambered heart, and having diverse ecological adaptations. Although the context does not provide this additional information, the student's answer does not conflict with the context. Therefore, the student's answer is correct.\\nGRADE: CORRECT\", correction=None, evaluator_info={'__run': RunInfo(run_id=UUID('c9b05c23-44ea-4cc2-a2f8-a5e97e514770'))}, source_run_id=None, target_run_id=None)],\n", " 'reference': {'answer': 'They are both warm-blooded'}},\n", " '4ebcf822-8010-4f5f-9e60-e62ee8319f44': {'output': 'The blue whale (Balaenoptera musculus) holds the title for being the largest mammal on Earth. It can reach lengths of up to 98 feet (30 meters) and weigh up to 200 tons (181 metric tons).',\n", " 'input': {'question': 'What is the largest mammal?'},\n", " 'feedback': [EvaluationResult(key='correctness', score=1, value='CORRECT', comment='CORRECT', correction=None, evaluator_info={'__run': RunInfo(run_id=UUID('75f75635-ff0f-46fe-95c1-c389ee4a7b5e'))}, source_run_id=None, target_run_id=None),\n", " EvaluationResult(key='Contextual Accuracy', score=1, value='CORRECT', comment='CORRECT', correction=None, evaluator_info={'__run': RunInfo(run_id=UUID('1d2b27fa-89e8-4716-8b35-5d065b145595'))}, source_run_id=None, target_run_id=None),\n", " EvaluationResult(key='COT Contextual Accuracy', score=1, value='CORRECT', comment=\"The student's answer correctly identifies the blue whale as the largest mammal, which matches the context provided. The additional information about the size and weight of the blue whale does not conflict with the context, but rather provides more detail. \\nGRADE: CORRECT\", correction=None, evaluator_info={'__run': RunInfo(run_id=UUID('a0370f62-6018-49ce-b6a5-fb7ca3c179cf'))}, source_run_id=None, target_run_id=None)],\n", " 'reference': {'answer': 'The blue whale'}}}}" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "client = Client()\n", "llm = ChatOpenAI()\n", "run_on_dataset(\n", " dataset_name=dataset_name,\n", " llm_or_chain_factory=llm,\n", " client=client,\n", " evaluation=evaluation_config,\n", " project_name=\"evalproject v1\",\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "db141aac-f2af-4a6e-bc98-c42536b3aaf2", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.4" } }, "nbformat": 4, "nbformat_minor": 5 }