Update process_documents.py
Browse files- process_documents.py +1 -1
process_documents.py
CHANGED
|
@@ -34,7 +34,7 @@ def process_documents(urls):
|
|
| 34 |
|
| 35 |
|
| 36 |
def process_web(url, source_id):
|
| 37 |
-
data = WebBaseLoader(url).load()[0]
|
| 38 |
document_snippets = [
|
| 39 |
Document(
|
| 40 |
page_content=deep_strip(data.page_content),
|
|
|
|
| 34 |
|
| 35 |
|
| 36 |
def process_web(url, source_id):
|
| 37 |
+
data = WebBaseLoader(f"https://r.jina.ai/{url}").load()[0]
|
| 38 |
document_snippets = [
|
| 39 |
Document(
|
| 40 |
page_content=deep_strip(data.page_content),
|