430.00 c

#400#ML_Engineer_Basic#430#ML_Development_Tools#430.00#LangChain#430.00 c#RAG_with_FAISS

# -*- coding: utf-8 -*-
"""Langchain RAG_FAISS.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1WEBqVjZqax9yYPBbg6LrUdvlcY0kgrrQ
"""

from google.colab import userdata

! pip install -q openai langchain tiktoken faiss-cpu

! pip install pydantic==1.10.13 docarray==0.32.1

!pip install -U langchain-openai

## import modules required

# from langchain.embeddings import OpenAIEmbeddings

from langchain.chat_models import ChatOpenAI
from langchain_openai import OpenAIEmbeddings
from langchain.prompts import ChatPromptTemplate
from langchain.vectorstores import FAISS
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnablePassthrough

import os



os.environ["OPENAI_API_KEY"] = userdata.get('OPENAI_API_KEY')
model = "text-embedding-3-small"



# Create a FAISS VectorStore
vectorstore = FAISS.from_texts(
    ["harrison worked at kensho"], embedding=OpenAIEmbeddings(openai_api_key=userdata.get('OPENAI_API_KEY'), model="text-embedding-3-small") # deployment="text-embedding-ada-002"
)

!pwd && ls -l

# Create a Retriever
retriever = vectorstore.as_retriever()

# Create a prompt, model
template = """Answer the question based only on the following context:
{context}

Question: {question}
"""
prompt = ChatPromptTemplate.from_template(template)
model = ChatOpenAI() # model_name='gpt-3.5-turbo'

# Chaining a pipeline
retrieval_chain = (
    {"context": retriever, "question": RunnablePassthrough()}
    | prompt
    | model
    | StrOutputParser()
)

# Run retrieva_chain
retrieval_chain.invoke("where did harrison work?")
# 'Harrison worked at Kensho.'

"""from langchain.text_splitter import CharacterTextSplitter
from langchain_community.document_loaders import TextLoader



loader = TextLoader("./abc.txt")
documents = loader.load()
text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
docs = text_splitter.split_documents(documents)
embeddings = OpenAIEmbeddings()
db = FAISS.from_documents(docs, embeddings)
"""

from langchain.text_splitter import CharacterTextSplitter
from langchain_community.document_loaders import TextLoader

loader = TextLoader("./abc.txt")
documents = loader.load()
text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
docs = text_splitter.split_documents(documents)
embeddings = OpenAIEmbeddings()
db = FAISS.from_documents(docs, embeddings)
FAISS.from_documents



from langchain.schema import HumanMessage, SystemMessage

# Assume the initial setup for FAISS and ChatOpenAI has been completed

# Step 1: Query the FAISS db to retrieve relevant document content
query = "Where do you live? What you ate today?"
retriever = db.as_retriever()
retrieved_docs = retriever.invoke(query)
retrieved_context = retrieved_docs[0].page_content  # Assume using first doc's content

# Step 2: Combine the retrieved document content with the original query
combined_query = f"Context: {retrieved_context}\nQuery: {query}"

# Step 3: Use the combined query as a prompt for the ChatOpenAI model
chat = ChatOpenAI()  # Ensure the ChatOpenAI instance is correctly initialized
response = chat([
    SystemMessage(content="Please use the following information to assist."),
    HumanMessage(content=combined_query)
])

# Print the model's response

print(response.content)  # Assuming the response is structured appropriately

from langchain.schema import HumanMessage, SystemMessage

# Assume the initial setup for FAISS and ChatOpenAI has been completed

# Step 1: Query the FAISS db to retrieve relevant document content
query = "Where do you live? What you ate today?"
# retriever = db.as_retriever()
# retrieved_docs = retriever.invoke(query)
retrieved_context = "None"  # Assume using first doc's content

# Step 2: Combine the retrieved document content with the original query
combined_query = f"Context: {retrieved_context}\nQuery: {query}"

# Step 3: Use the combined query as a prompt for the ChatOpenAI model
chat = ChatOpenAI()  # Ensure the ChatOpenAI instance is correctly initialized
response = chat([
    SystemMessage(content="Please use the following information to assist."),
    HumanMessage(content=combined_query)
])

# Print the model's response

print(response.content)  # Assuming the response is structured appropriately

from langchain.text_splitter import CharacterTextSplitter
from langchain_community.document_loaders import TextLoader

loader = TextLoader("./abc.txt")
documents = loader.load()
text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
docs = text_splitter.split_documents(documents)
embeddings = OpenAIEmbeddings()
db = FAISS.from_documents(docs, embeddings)

from langchain.schema import HumanMessage, SystemMessage

# Assume the initial setup for FAISS and ChatOpenAI has been completed

# Step 1: Query the FAISS db to retrieve relevant document content
query = "What numpy functions are there?"
retriever = db.as_retriever()
retrieved_docs = retriever.invoke(query)
retrieved_context = retrieved_docs[0].page_content  # Assume using first doc's content

# Step 2: Combine the retrieved document content with the original query
combined_query = f"Context: {retrieved_context}\nQuery: {query}"

# Step 3: Use the combined query as a prompt for the ChatOpenAI model
chat = ChatOpenAI()  # Ensure the ChatOpenAI instance is correctly initialized
response = chat([
    SystemMessage(content="Please use the following information to assist."),
    HumanMessage(content=combined_query)
])

# Print the model's response
print(response)
print(response.content)  # Assuming the response is structured appropriately