Delete Ingest.py
Browse files
Ingest.py
DELETED
|
@@ -1,18 +0,0 @@
|
|
| 1 |
-
from langchain_community.document_loaders import PyPDFLoader,DirectoryLoader
|
| 2 |
-
from langchain.embeddings import HuggingFaceEmbeddings
|
| 3 |
-
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 4 |
-
from langchain_community.vectorstores import FAISS
|
| 5 |
-
|
| 6 |
-
loader = DirectoryLoader('data', glob="./*.pdf", loader_cls=PyPDFLoader)
|
| 7 |
-
documents = loader.load()
|
| 8 |
-
|
| 9 |
-
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1024, chunk_overlap=200)
|
| 10 |
-
texts = text_splitter.split_documents(documents)
|
| 11 |
-
|
| 12 |
-
embedings = HuggingFaceEmbeddings(model_name="nomic-ai/nomic-embed-text-v1",model_kwargs={"trust_remote_code":True,"revision":"289f532e14dbbbd5a04753fa58739e9ba766f3c7"})
|
| 13 |
-
|
| 14 |
-
# Creates vector embeddings and saves it in the FAISS DB
|
| 15 |
-
faiss_db = FAISS.from_documents(texts, embedings)
|
| 16 |
-
|
| 17 |
-
# Saves and export the vector embeddings databse
|
| 18 |
-
faiss_db.save_local("ipc_vector_db")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|