Update process_documents.py

process_documents.py

@@ -35,9 +35,13 @@ def process_documents(urls):
 
 def process_web(url, source_id):
     data = WebBaseLoader(f"https://r.jina.ai/{url}").load()[0]
+    try:
+        page_content = data.page_content[data.page_content.index("Markdown Content:") + len("Markdown Content:"):].strip()
+    except Exception as e:
+        page_content = data.page_content.strip()
     document_snippets = [
         Document(
-            page_content=data.page_content,
+            page_content=page_content,
             metadata={
                 "header": data.metadata["title"],
                 "source_url": url,