switch from Unstructured Loader to PyPDF as its results have page nubmer

.env.example

@@ -18,6 +18,8 @@ HF_PIPELINE_DEVICE_TYPE=
 
 CHAT_HISTORY_ENABLED=true
 
+PDF_FILE_BASE_URL=
+
 # if unset, default to "hkunlp/instructor-xl"
 HF_EMBEDDINGS_MODEL_NAME="hkunlp/instructor-large"
 

README.md

@@ -1,5 +1,5 @@
 ---
-title: Chat With AI Books
+title: Chat with AI Books
 emoji: 👀
 colorFrom: indigo
 colorTo: blue

app_modules/qa_chain.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import urllib
 from queue import Queue
 from typing import Any, Optional
 
@@ -528,4 +529,14 @@ class QAChain:
             self.streamer.reset(q)
 
         qa = self.get_chain(tracing)
-        return qa(inputs)
+        result = qa(inputs)
+
+        base_url = os.environ.get("PDF_FILE_BASE_URL")
+        if base_url is not None:
+            documents = result["source_documents"]
+            for doc in documents:
+                source = doc.metadata["source"]
+                title = source.split("/")[-1]
+                doc.metadata["url"] = f"{base_url}{urllib.parse.quote(title)}"
+
+        return result

data/chromadb_1024_512/chroma-collections.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19ef26d149b26a03e85a8341d84187c920cfe877e0265871e9950829f553a1ea
+oid sha256:1d0e4364f9a67d91e3185cc597297b8651ca02bdfddb8467767c8a71cbb89d4e
 size 557

data/chromadb_1024_512/chroma-embeddings.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:38d53221ceafcfb1af1d4a068ba81076a998cfc4ecbc09952c02ee14353b0daa
-size 156890027
+oid sha256:8b050c60c5fd263355aabc3cc35e6308930cb4b8a1929e7209b6777da0782d59
+size 7513430

data/chromadb_1024_512/index/{id_to_uuid_67de6665-0585-4559-85bd-e044c61f64df.pkl → id_to_uuid_44a39155-bdc7-450c-8532-01db0e4b66cc.pkl}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a58ac77e06b03413e225e1e2fc0fb956883fe6ac387be6332942bbb4ab6ac946
-size 997779
+oid sha256:4eb4fe05362f4052e3af173b0915e9758bb7bc7f9f681850e765cbde35d8783f
+size 47652

data/chromadb_1024_512/index/{uuid_to_id_67de6665-0585-4559-85bd-e044c61f64df.pkl → index_44a39155-bdc7-450c-8532-01db0e4b66cc.bin}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e971f597172b30a15507fdaf90438399dfd9b54c0599196c3f9c3422218bcfe6
-size 1166688
+oid sha256:5a26db7cd65749049856321b4aef559a0ffbef7f4286131c1bcd5f5dc4cc3849
+size 4743996

data/chromadb_1024_512/index/{index_metadata_67de6665-0585-4559-85bd-e044c61f64df.pkl → index_metadata_44a39155-bdc7-450c-8532-01db0e4b66cc.pkl}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47aef36a38d7c29f39b1b7e69e7788020b31f6c52c96a3f1bd8d13e79eeecda1
+oid sha256:ae5e0c780f18efa625dc2d0ad2d60328b51d2842cac144446196e4032e7c2c43
 size 105

data/chromadb_1024_512/index/{index_67de6665-0585-4559-85bd-e044c61f64df.bin → uuid_to_id_44a39155-bdc7-450c-8532-01db0e4b66cc.pkl}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a74d2560e304e1c78499c16465d6135d227330f021c98bb1c83a4fc8a641eeeb
-size 98876604
+oid sha256:49aab2d749c5650688e4b1b566d8773889ca59d92ea2083d04fd5882a626ecc0
+size 55737

ingest.py

@@ -3,9 +3,7 @@ import os
 from timeit import default_timer as timer
 from typing import List
 
-import torch
-from dotenv import load_dotenv
-from langchain.document_loaders.directory import DirectoryLoader
+from langchain.document_loaders import PyPDFDirectoryLoader
 from langchain.embeddings import HuggingFaceInstructEmbeddings
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores.chroma import Chroma
@@ -14,7 +12,7 @@ from app_modules.utils import *
 
 
 def load_documents(source_pdfs_path) -> List:
-    loader = DirectoryLoader(source_pdfs_path, glob="./*.pdf", show_progress=True)
+    loader = PyPDFDirectoryLoader(source_pdfs_path, silent_errors=True)
     documents = loader.load()
     return documents