recommit

.chainlit/config.toml

@@ -0,0 +1,84 @@
+[project]
+# Whether to enable telemetry (default: true). No personal data is collected.
+enable_telemetry = true
+
+# List of environment variables to be provided by each user to use the app.
+user_env = []
+
+# Duration (in seconds) during which the session is saved when the connection is lost
+session_timeout = 3600
+
+# Enable third parties caching (e.g LangChain cache)
+cache = false
+
+# Follow symlink for asset mount (see https://github.com/Chainlit/chainlit/issues/317)
+# follow_symlink = false
+
+[features]
+# Show the prompt playground
+prompt_playground = true
+
+# Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript)
+unsafe_allow_html = false
+
+# Process and display mathematical expressions. This can clash with "$" characters in messages.
+latex = false
+
+# Authorize users to upload files with messages
+multi_modal = true
+
+# Allows user to use speech to text
+[features.speech_to_text]
+    enabled = false
+    # See all languages here https://github.com/JamesBrill/react-speech-recognition/blob/HEAD/docs/API.md#language-string
+    # language = "en-US"
+
+[UI]
+# Name of the app and chatbot.
+name = "Chatbot"
+
+# Show the readme while the conversation is empty.
+show_readme_as_default = true
+
+# Description of the app and chatbot. This is used for HTML tags.
+# description = ""
+
+# Large size content are by default collapsed for a cleaner ui
+default_collapse_content = true
+
+# The default value for the expand messages settings.
+default_expand_messages = false
+
+# Hide the chain of thought details from the user in the UI.
+hide_cot = false
+
+# Link to your github repo. This will add a github button in the UI's header.
+# github = ""
+
+# Specify a CSS file that can be used to customize the user interface.
+# The CSS file can be served from the public directory or via an external link.
+# custom_css = "/public/test.css"
+
+# Override default MUI light theme. (Check theme.ts)
+[UI.theme.light]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.light.primary]
+        #main = "#F80061"
+        #dark = "#980039"
+        #light = "#FFE7EB"
+
+# Override default MUI dark theme. (Check theme.ts)
+[UI.theme.dark]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.dark.primary]
+        #main = "#F80061"
+        #dark = "#980039"
+        #light = "#FFE7EB"
+
+
+[meta]
+generated_by = "0.7.700"

.gitignore

@@ -0,0 +1,8 @@
+/venv
+__pycache__/*
+.env
+download-hf-model.ipynb
+temp
+start_qdrant_services.sh
+requirements copy.txt
+Dockerfile copy

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.11.9
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+WORKDIR $HOME/app
+COPY --chown=user . $HOME/app
+COPY ./requirements.txt ~/app/requirements.txt
+RUN pip install -r requirements.txt
+COPY . .
+CMD ["chainlit", "run", "app.py", "--port", "7860"]

app.py

@@ -0,0 +1,271 @@
+import os
+from typing import List
+import uuid
+import chainlit as cl
+from chainlit.types import AskFileResponse
+from langchain.memory import ConversationBufferMemory
+from langchain_core.chat_history import BaseChatMessageHistory
+from langchain_community.document_loaders import PyMuPDFLoader, TextLoader
+from langchain.prompts import MessagesPlaceholder
+from langchain.prompts import ChatPromptTemplate
+from langchain_community.chat_message_histories import ChatMessageHistory
+from langchain.chains.history_aware_retriever import create_history_aware_retriever
+from langchain.chains.retrieval import create_retrieval_chain
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_experimental.text_splitter import SemanticChunker
+from langchain_qdrant import QdrantVectorStore
+from langchain_core.documents import Document
+from qdrant_client import QdrantClient
+from qdrant_client.http.models import Distance, VectorParams
+from langchain_openai import ChatOpenAI
+from embedding_model import get_embeddings_openai_text_3_large,get_embeddings_snowflake_arctic_embed_l
+from pdfloader import PDFLoaderWrapper
+from langchain_core.runnables.history import RunnableWithMessageHistory
+from chainlit.input_widget import Select, Switch, Slider
+from dotenv import load_dotenv
+from langchain_huggingface import HuggingFaceEmbeddings
+
+load_dotenv()
+
+BOR_FILE_PATH = "https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf"
+NIST_FILE_PATH = "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf"
+SMALL_DOC = "https://arxiv.org/pdf/1908.10084"  # 11 pages Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks
+documents_to_preload = [
+    BOR_FILE_PATH,
+    NIST_FILE_PATH
+    # SMALL_DOC
+]
+collection_name = "ai-safety"
+
+welcome_message = """
+Welcome to the chatbot to clarify all your AI Safety related queries.:
+Now preloading below documents:
+    1. Blueprint for an AI Bill of Rights
+    2. NIST AI Standards
+Please wait for a moment to load the documents.
+"""
+chat_model_name = "gpt-4o"
+embedding_model_name = "Snowflake/snowflake-arctic-embed-l"
+chat_model = ChatOpenAI(model=chat_model_name, temperature=0)
+
+async def connect_to_qdrant():
+    embedding_model = HuggingFaceEmbeddings(model_name=embedding_model_name)
+    qdrant_url = os.environ["QDRANT_URL"]
+    qdrant_api_key = os.environ["QDRANT_API_KEY"]
+    collection_name = os.environ["COLLECTION_NAME"]
+    qdrant_client = QdrantClient(url=qdrant_url,api_key=qdrant_api_key)
+    vector_store = QdrantVectorStore(
+        client=qdrant_client,
+        collection_name=collection_name,
+        embedding=embedding_model,
+    )
+    return vector_store.as_retriever()
+
+def initialize_vectorstore(
+    collection_name: str,
+    embedding_model,
+    dimension,
+    distance_metric: Distance = Distance.COSINE,
+):
+    client = QdrantClient(":memory:")
+    client.create_collection(
+        collection_name=collection_name,
+        vectors_config=VectorParams(size=dimension, distance=distance_metric),
+    )
+
+    vector_store = QdrantVectorStore(
+        client=client,
+        collection_name=collection_name,
+        embedding=embedding_model,
+    )
+    return vector_store
+
+def get_text_splitter(strategy, embedding_model):
+    if strategy == "semantic":
+        return SemanticChunker(
+            embedding_model,
+            breakpoint_threshold_type="percentile",
+            breakpoint_threshold_amount=90,
+        )
+
+def process_file(file: AskFileResponse, text_splitter):
+    if file.type == "text/plain":
+        Loader = TextLoader
+    elif file.type == "application/pdf":
+        Loader = PyMuPDFLoader
+
+    loader = Loader(file.path)
+    documents = loader.load()
+    title = documents[0].metadata.get("title")
+    docs = text_splitter.split_documents(documents)
+    for i, doc in enumerate(docs):
+        doc.metadata["source"] = f"source_{i}"
+        doc.metadata["title"] = title
+    return docs
+
+def populate_vectorstore(vector_store, docs: List[Document]):
+    vector_store.add_documents(docs)
+    return vector_store
+
+def create_history_aware_retriever_self(chat_model, retriever):
+    contextualize_q_system_prompt = (
+        "Given a chat history and the latest user question which might reference context in the chat history, "
+        "formulate a standalone question which can be understood without the chat history. Do NOT answer the question, "
+        "just reformulate it if needed and otherwise return it as is."
+    )
+    contextualize_q_prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", contextualize_q_system_prompt),
+            MessagesPlaceholder("chat_history"),
+            ("human", "{input}"),
+        ]
+    )
+    return create_history_aware_retriever(chat_model, retriever, contextualize_q_prompt)
+
+def create_qa_chain(chat_model):
+    qa_system_prompt = (
+        "You are an helpful assistant named 'Shield' and your task is to answer any questions related to AI Safety for the given context."
+        "Use the following pieces of retrieved context to answer the question."
+        # "If any questions asked outside AI Safety context, just say that you are a specialist in AI Safety and can't answer that."
+        # f"When introducing you, just say that you are an AI assistant powered by embedding model {embedding_model_name} and chat model {chat_model_name} and your knowledge is limited to 'Blueprint for an AI Bill of Rights' and 'NIST AI Standards' documents."
+        "If you don't know the answer, just say that you don't know.\n\n"
+        "{context}"
+    )
+    qa_prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", qa_system_prompt),
+            MessagesPlaceholder("chat_history"),
+            ("human", "{input}"),
+        ]
+    )
+    return create_stuff_documents_chain(chat_model, qa_prompt)
+
+
+def create_rag_chain(chat_model, retriever):
+    history_aware_retriever = create_history_aware_retriever_self(chat_model, retriever)
+    question_answer_chain = create_qa_chain(chat_model)
+    return create_retrieval_chain(history_aware_retriever, question_answer_chain)
+
+
+def create_session_id():
+    session_id = str(uuid.uuid4())
+    return session_id
+
+
+@cl.on_chat_start
+async def start():
+    # cl.user_session.set("memory", conversation_buffer_memory)
+    msg = cl.Message(content=welcome_message)
+    await msg.send()
+
+    # Create a session id
+    session_id = create_session_id()
+    cl.user_session.set("session_id", session_id)
+
+    # Preserve chat history
+    conversation_buffer_memory = ConversationBufferMemory(
+        memory_key="chat_history",
+        output_key="answer",
+        chat_memory=ChatMessageHistory(),
+        return_messages=True,
+    )
+
+    # todo: if logged in user is admin then allow them to upload new pdfs.
+
+    # # Embedding model
+    # # embedding_model, dimension = get_embeddings_openai_text_3_large()
+    # embedding_model, dimension = get_embeddings_snowflake_arctic_embed_l()
+    # msg.content = "Embedding model loaded"
+    # await msg.update()
+    # cl.user_session.set("embedding_model", embedding_model)
+    # cl.user_session.set("dimension", dimension)
+
+    # # Pdf loader
+    # pdf_loader = PDFLoaderWrapper(
+    #     documents_to_preload, PDFLoaderWrapper.LoaderType.PYMUPDF
+    # )
+    # msg.content = "Embedding model loaded"
+    # await msg.update()
+    # cl.user_session.set("pdf_loader", pdf_loader)
+    # documents = await pdf_loader.aload()
+
+    # text_splitter = get_text_splitter("semantic", embedding_model)
+
+    # chunked_docs = text_splitter.split_documents(documents)
+
+    # vector_store = initialize_vectorstore(
+    #     collection_name, embedding_model, dimension=dimension
+    # )
+
+    # vector_store = populate_vectorstore(vector_store, chunked_docs)
+
+    retriever = await connect_to_qdrant()
+
+    rag_chain = create_rag_chain(chat_model, retriever)
+
+    store = {}
+
+    def get_session_history(session_id: str) -> BaseChatMessageHistory:
+        if session_id not in store:
+            store[session_id] = ChatMessageHistory()
+        return store[session_id]
+
+    conversational_rag_chain = RunnableWithMessageHistory(
+        rag_chain,
+        get_session_history,
+        input_messages_key="input",
+        history_messages_key="chat_history",
+        output_messages_key="answer",
+    )
+
+    # Let the user know that the system is ready
+    msg.content = msg.content + "\nReady to answer your questions!"
+    await msg.update()
+
+    cl.user_session.set("conversational_rag_chain", conversational_rag_chain)
+
+
+@cl.on_message
+async def main(message: cl.Message):
+    session_id = cl.user_session.get("session_id")
+    conversational_rag_chain = cl.user_session.get("conversational_rag_chain")
+
+    response =  await conversational_rag_chain.ainvoke(
+        {"input": message.content},
+        config={"configurable": {"session_id": session_id},
+                "callbacks":[cl.AsyncLangchainCallbackHandler()]},         
+    )
+    answer = response["answer"]
+
+    source_documents = response["context"]
+    text_elements = []
+    unique_pages = set()
+
+    if source_documents:
+
+        for source_idx, source_doc in enumerate(source_documents):
+            source_name = f"source_{source_idx+1}"
+            page_number = source_doc.metadata['page']
+            #page_number = source_doc.metadata.get('page', "NA")  # NA or any default value
+            page = f"Page {page_number}"
+            text_element_content = source_doc.page_content
+            text_element_content = text_element_content if text_element_content != "" else "No Content"
+            #text_elements.append(cl.Text(content=text_element_content, name=source_name))
+            if page not in unique_pages:
+                unique_pages.add(page)
+                text_elements.append(cl.Text(content=text_element_content, name=page))
+            #text_elements.append(cl.Text(content=text_element_content, name=page))
+        source_names = [text_el.name for text_el in text_elements]
+        
+        if source_names:
+            answer += f"\n\n Sources:{', '.join(source_names)}"
+        else:
+            answer += "\n\n No sources found"
+
+    await cl.Message(content=answer, elements=text_elements).send()
+
+if __name__ == "__main__":
+    from chainlit.cli import run_chainlit
+
+    run_chainlit(__file__)

chainlit.md

@@ -0,0 +1,3 @@
+# Welcome to Chat with Your Text File
+
+With this application, you can chat with an uploaded text file that is smaller than 2MB!

embedding_model.py

@@ -0,0 +1,58 @@
+
+import tiktoken
+import os
+from langchain_openai import OpenAIEmbeddings
+from langchain_community.embeddings import HuggingFaceBgeEmbeddings
+import torch
+from transformers import AutoModel, AutoTokenizer
+from transformers import AutoModel, AutoTokenizer
+from langchain_huggingface import HuggingFaceEmbeddings
+
+# def get_embeddings_model_bge_base_en_v1_5():
+#     model_name = "BAAI/bge-base-en-v1.5"
+#     model_kwargs = {'device': 'cpu'}
+#     encode_kwargs = {'normalize_embeddings': False}
+#     embedding_model = HuggingFaceBgeEmbeddings(
+#         model_name=model_name,
+#         model_kwargs=model_kwargs,
+#         encode_kwargs=encode_kwargs
+#     )
+#     return embedding_model
+
+# def get_embeddings_model_bge_en_icl():
+#     model_name = "BAAI/bge-en-icl"
+#     model_kwargs = {'device': 'cpu'}
+#     encode_kwargs = {'normalize_embeddings': False}
+#     embedding_model = HuggingFaceBgeEmbeddings(
+#         model_name=model_name,
+#         model_kwargs=model_kwargs,
+#         encode_kwargs=encode_kwargs
+#     )
+#     return embedding_model , 4096
+
+# def get_embeddings_model_bge_large_en():
+#     model_name = "BAAI/bge-large-en"
+#     model_kwargs = {'device': 'cpu'}
+#     encode_kwargs = {'normalize_embeddings': False}
+#     embedding_model = HuggingFaceBgeEmbeddings(
+#         model_name=model_name,
+#         model_kwargs=model_kwargs,
+#         encode_kwargs=encode_kwargs
+#     )
+#     return embedding_model
+
+def get_embeddings_openai_text_3_large():
+    embedding_model = OpenAIEmbeddings(model="text-embedding-3-large")
+    dimension = 3072
+    return embedding_model,dimension
+
+# def get_embeddings_snowflake_arctic_embed_l():
+#     current_dir = os.path.dirname(os.path.realpath(__file__))
+#     model_name = "Snowflake/snowflake-arctic-embed-l"
+#     tokenizer = AutoTokenizer.from_pretrained(f"{current_dir}/cache/tokenizer/{model_name}")
+#     model = AutoModel.from_pretrained(f"{current_dir}/cache/model/{model_name}")
+#     return model,1024
+
+def get_embeddings_snowflake_arctic_embed_l():
+    embedding_model = HuggingFaceEmbeddings(model_name="Snowflake/snowflake-arctic-embed-l")
+    return embedding_model,1024

pdfloader.py

@@ -0,0 +1,27 @@
+from enum import Enum
+from typing import List
+from langchain_community.document_loaders import PyMuPDFLoader
+from langchain_core.documents import Document
+import asyncio
+
+class PDFLoaderWrapper():
+    class LoaderType(str, Enum):
+        PYMUPDF = "pymupdf"
+
+    def __init__(self, file_path: str | List[str] , loader_type: LoaderType = LoaderType.PYMUPDF):
+        self.file_path = file_path if isinstance(file_path, list) else [file_path]
+        self.loader_type = loader_type
+
+    async def aload(self) -> List[Document]:
+        all_docs = []
+        for file_path in self.file_path:
+            if self.loader_type == self.LoaderType.PYMUPDF:
+                try:
+                    loader = PyMuPDFLoader(file_path)
+                    docs = await loader.aload()
+                    all_docs.extend(docs)
+                except Exception as e:
+                    print(f"Error loading file {file_path}: {e}")
+                    continue
+        return all_docs
+

pre-processing.ipynb

@@ -0,0 +1,595 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "% pip install numpy==1.26.4 \\\n",
+    "openai==1.44.1 \\\n",
+    "qdrant-client==1.11.2 \\\n",
+    "langchain==0.3.0 \\\n",
+    "langchain-text-splitters==0.3.0 \\\n",
+    "langchain-community==0.3.0 \\\n",
+    "langchain_experimental \\\n",
+    "langchain_qdrant \\\n",
+    "langchain_openai \\\n",
+    "pypdf==4.3.1 \\\n",
+    "PyMuPDF==1.24.10 \\\n",
+    "pymupdf4llm \\\n",
+    "sentence_transformers \\\n",
+    "langchain_huggingface "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "BOR_FILE_PATH = \"https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf\"\n",
+    "NIST_FILE_PATH = \"https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf\"\n",
+    "SMALL_DOC = \"https://arxiv.org/pdf/1908.10084\"  \n",
+    "documents_to_preload = [\n",
+    "    BOR_FILE_PATH,\n",
+    "    NIST_FILE_PATH\n",
+    "    # SMALL_DOC\n",
+    "]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/jeevan/Documents/Learnings/ai-engineering-bootcamp/AIE4/AIE4/mid-term/ai-safety-chatapp/venv/lib/python3.11/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:13: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from tqdm.autonotebook import tqdm, trange\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Embedding model - snowflake-arctic-embed-l\n",
+    "from langchain_huggingface import HuggingFaceEmbeddings\n",
+    "\n",
+    "model_name = \"Snowflake/snowflake-arctic-embed-l\"\n",
+    "embedding_model = HuggingFaceEmbeddings(model_name=model_name)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pdfloader import PDFLoaderWrapper\n",
+    "from langchain_experimental.text_splitter import SemanticChunker\n",
+    "\n",
+    "\n",
+    "pdf_loader = PDFLoaderWrapper(\n",
+    "    documents_to_preload, PDFLoaderWrapper.LoaderType.PYMUPDF\n",
+    ")\n",
+    "documents = await pdf_loader.aload()\n",
+    "\n",
+    "text_splitter = SemanticChunker(embedding_model, breakpoint_threshold_type=\"percentile\",breakpoint_threshold_amount=90)\n",
+    "\n",
+    "chunked_docs = text_splitter.split_documents(documents)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import getpass\n",
+    "\n",
+    "os.environ[\"QDRANT_API_KEY\"] = getpass.getpass(\"Enter Your Qdrant API Key: \")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['eddeba090cf64372b937fdeeb4a66a05',\n",
+       " '04d716b884124244876b0dd6bba15b4e',\n",
+       " 'db68b0d183214d95a3b8be26f9a3072f',\n",
+       " 'cb21583a20c748aa898821c475825aa1',\n",
+       " '503b0d1da1354b3dba9903d889fa1dcf',\n",
+       " 'f5db16617a4b4ed69cf46c7739ce1705',\n",
+       " '1e1532cacc434b988de2039a9b07bd95',\n",
+       " 'eb62a186469e4d6a860ed9f2c32264cf',\n",
+       " 'e621542bdc944c35adad13321669a782',\n",
+       " '0a8ad7cbf78b488bbcb19bc046f991ea',\n",
+       " 'e24af2031ccc4b86afc5c5b868ce0875',\n",
+       " '6eed4c2596e14f9b8fbcad5a16682bf7',\n",
+       " '0adb7d6c0ed641fdb1aa7c38b44f205e',\n",
+       " '69559e35c6284d1a9711aa0d67cf3663',\n",
+       " '1ce9c69941e446aa83325edd2a43e6af',\n",
+       " '67472b47adf3441dbc315be9369b5bac',\n",
+       " '0470ecb0cb3c48e49d2aeabf8c7b8764',\n",
+       " '603575c0eff4430f914b1ff491c6cff0',\n",
+       " 'cbff97eea69343c4b2073c3264337c82',\n",
+       " '9f0a9425534041e5a117f546d5aa9e0f',\n",
+       " '0f551ca5bb74476d8bed3183315cb687',\n",
+       " 'd79db53763084978bffeb3d2a9ba888e',\n",
+       " '4adcbf0531504ec3a7672b6ac5e88695',\n",
+       " '5fa8f04f748d4fcc8bdb0e803ca38053',\n",
+       " '855328fce2db47ec9a128d8eab41b3b8',\n",
+       " '9be9afb0e477463db19d507179e18436',\n",
+       " '9356ffe511ae4f659f079895695a67f3',\n",
+       " '26b1283ce9064aab9e6399db9371e542',\n",
+       " 'f9e92a690fcf4a58a978abd173f39503',\n",
+       " 'd0fd1190717046d1a02d7c21f15e982c',\n",
+       " '48ddfed6388e48c2bbc260d77d2ea7b2',\n",
+       " '7391be7bceba44d7974ff05604422ff6',\n",
+       " 'a1645bb4440f4a2eb1ff3e5a818e1857',\n",
+       " '9265bbde2d9549c5aefc2da778943bb1',\n",
+       " 'c0e85a23783e4eaab735bf4906c5c4ee',\n",
+       " '2a8e0ade33134a16b696a6f274815682',\n",
+       " '1c4df2e13e314e419286bff9441848ca',\n",
+       " '74cd8670d0e748c1883497732ec03814',\n",
+       " '7cfad9000dd7406f86e12c740d5ef6eb',\n",
+       " '0ee580a675574c42ab6f2a191279c7e1',\n",
+       " 'f24963b680ae448eb3a5a98a629207c8',\n",
+       " '2d85b61d77244dfa8d15d665ff354bb4',\n",
+       " 'f38706bb1fe34e6db224c8fde5bf29b4',\n",
+       " 'e0a651dab4f14ab8bb27f886acb6f81d',\n",
+       " '8e41d96e0c40420fa79ce5b9460eea90',\n",
+       " '0f56a8ddaf7a41828b98e0ad531b571c',\n",
+       " '03d42d339edc4dc9bd0081b619f55ecc',\n",
+       " 'ba42fded48a249c694bf9a9b2e94c146',\n",
+       " 'c36879e922d745ce824c3c6c5391c671',\n",
+       " 'cf782c0ed9d3430a884e8ecf11fc4628',\n",
+       " '85334a4e52e04aa8b364131dd90857f9',\n",
+       " '796507d959704177af9fda3d8b5429a4',\n",
+       " 'd614bad2456842e7b8f3e92de44d68ec',\n",
+       " '6042cdd6a45a42228e8fa2bd909f3b53',\n",
+       " '6a0fbab037cd4d079d7abdc0694392b5',\n",
+       " '9bb6e193fe1f47a5a8efb433ca026586',\n",
+       " 'af5664a283ca4262a22cb706ba0866ee',\n",
+       " '868ea113f0b2470eb85bc7fbf7355ac7',\n",
+       " '13d0c504b19d4794b7f4e0b0d0ecb41b',\n",
+       " 'd24f3f8f3967458689d5315992b51f44',\n",
+       " '363d7278858142e1a9a8580dff2469b1',\n",
+       " 'c9f1bb8520344d5fb6cf7db77986c326',\n",
+       " '8a738ce3bafd4cdf94a2984e20016146',\n",
+       " '904f9cb924be4e9aa1d8d01e7c49e5c2',\n",
+       " '8dbdb46493964a15af5a7550442e076d',\n",
+       " '957d5363e2904646999b45fca59c10df',\n",
+       " '2f519f70fee240eaaf5b9cf19dc9c058',\n",
+       " 'afc3204b02c64c7e9a7bd10d4000acac',\n",
+       " 'df1d427abe9348a0baf4561b4fab104b',\n",
+       " 'ac96b17eb3f343c983120a9f4c5577d9',\n",
+       " '47f1f02b36d441aa9db960fa593323b0',\n",
+       " 'a604fa5fb1344139a6149836d6bc20dd',\n",
+       " '574857debc5c42b49b341a04f41c0d47',\n",
+       " 'ca59bb9be5e64470b226b36b78f7a11c',\n",
+       " 'f590db78eceb4c22b0c224b77ffa03b5',\n",
+       " 'bcbcbe3628214fd89b0f5f0c008700b8',\n",
+       " '58cf91d633794df09036a3246d44d3f4',\n",
+       " '58788414f5314fb3b9fbbe9d3b82ce40',\n",
+       " '3eb765a9b00a458b9c352f9ecd66069c',\n",
+       " 'c8ae3298d8b64ac3ab9f45dc6e0b1712',\n",
+       " '4d4b558e40994a919700b19d6ce3fb78',\n",
+       " '2dec1ed15f52459fada667c171b6cd15',\n",
+       " '70b7b33f7b3f4a89b22bdc587e51408f',\n",
+       " '5301362517fb475b916fa4dee82c0ece',\n",
+       " 'ac25b4c2668d44bea14c8299ddd17c92',\n",
+       " 'f0509d5035c14fa2aa6f91181134b0f3',\n",
+       " 'd53f9f0dc38149ffa3ea8b0aa6bbf849',\n",
+       " 'a2962de10d784220a72a380e8d409786',\n",
+       " 'b25e3cd5b3b148aba2627d88b93728d8',\n",
+       " 'a344f02041bc45e984452fe693bd26a6',\n",
+       " '53ec4ac26d6342f498fee283a5235d3f',\n",
+       " '323f6f0dd916451c9231730257d9b166',\n",
+       " '62f04a85af0c4b8aacf2bde02d0bbed3',\n",
+       " 'aa840da6722e458caeac99cc503ddd10',\n",
+       " '7d4fe147f0984ffc8d2f59ce86d5312a',\n",
+       " '2fee83e32198453986a73a7e3311bb14',\n",
+       " '9757507874714b57b7ff6c6a51aef718',\n",
+       " 'c7f42a04c66c4ab28c2c7c18ed33700a',\n",
+       " '49bd88e28a244910b888e79af4039282',\n",
+       " '420985e4c15c4c18aad851e95bdbebd8',\n",
+       " 'aaf353cf3b3b4e2c9709895271f55649',\n",
+       " '7a52229e16944616a1108caa86fc815d',\n",
+       " '629098069e2c4d09b845065c653a206d',\n",
+       " 'b37f1984a07e480b957057932b5e5f4d',\n",
+       " '7dfce2615e0344eb8147a10c57ef790e',\n",
+       " '07a1d0a768384b4aa77f986faa02a111',\n",
+       " 'eeeea24632f6475a9248613e6f6b7273',\n",
+       " '5ffbfc5623db434d807438c8a5c2a3db',\n",
+       " '83bf4a99c26f4374b39eb5685b6029a8',\n",
+       " 'd2176d5ac1934206ac13a25e3d3b4af9',\n",
+       " '8929fa8903a748748f51bb3e55e3aac2',\n",
+       " '21cedcf7d73a4a3cbb10319e64ebf145',\n",
+       " 'c1ba5a05221542838764f923be43d9e2',\n",
+       " '4d07a54bf62d4a929bec86ba278fa935',\n",
+       " '0cfbdccc2fb24637bd061e456996aad1',\n",
+       " '403ed966e49f44deb05101c4fa6ae485',\n",
+       " '073d20ce62514518b0a7ec495b317fef',\n",
+       " 'cbde5f3792b04bef8632c91906fc7146',\n",
+       " 'cb5bc23be3cf4ae389c346526c0323ae',\n",
+       " '7ca053a641394108b41dd8a91b1347df',\n",
+       " '027ad2187b8d4a80b84e95c59809d111',\n",
+       " 'bc49807d6f8c4f87a923366f83586ed8',\n",
+       " '41312c394807424ab191bf3bbe40416f',\n",
+       " '8390108ed60e465da899e0a2a688efaf',\n",
+       " '72353287c709410fafbb8096dfdf72c8',\n",
+       " 'b2902f5c90894246966ed217e6fb9574',\n",
+       " '10b79371e2364b0a9773fcacdc3daf05',\n",
+       " '3625bc3a69564ea39a28ae9bf70c4614',\n",
+       " '9177954079094aeaba7884bd0dffcc06',\n",
+       " '394fd830247141248a7454aea3aa489d',\n",
+       " '96877971f66644e6b8ab5767d4119a71',\n",
+       " 'a7e88bf685c14ba48c820f106b221881',\n",
+       " '216ec8e4e3fe439998fafe51e826db5f',\n",
+       " '1b66efa55a114c2985cc5236fb0b0742',\n",
+       " 'aa991a0cb36d4a75967c5e591bc08b97',\n",
+       " '6a6ce074204a48bf95032ae16c7db23c',\n",
+       " 'ff8e1f9be7804049af8e7c41974e4c5a',\n",
+       " '31a71b7f81674adf8c93d65a2d2eec63',\n",
+       " '9cc71e212bb14b909c79649596170c2a',\n",
+       " '9b042e1551d747bf9070b0b4ba8c0212',\n",
+       " '65f8e2ed927d41658069977d684be32e',\n",
+       " '6ed2ac368f3b4c04b2898fb6be21e98b',\n",
+       " 'f0924724eb644c1880e208dc4fab03e1',\n",
+       " '721f66c77001440b9d952fc8f6f4c56f',\n",
+       " '4843d3717aaf47d4b34e1f3a3cba498d',\n",
+       " '6465a722293d4645afeaaa3bd132d63e',\n",
+       " 'd26331770ecb486c9a5335da6bb0740d',\n",
+       " '8c7894b2ea4c421ba2937fdb4e34be86',\n",
+       " 'b20695156c6d42f4828e70d10b6f0277',\n",
+       " '6d64009de2774beab0f7c18424bb7327',\n",
+       " '2765c0b9b2a24917ab36b119ca8eef84',\n",
+       " 'bcf0337addee484882291f8cf9de408b',\n",
+       " '92b1f5a04ef340f486dd31b4eceb4a7c',\n",
+       " '512bfff959224382a066921c89bb735b',\n",
+       " 'e0558260a83c465eacff77ed9cb7308a',\n",
+       " '4074348ca89546bda70812b37b92fe14',\n",
+       " '21633e3b024e45ac998d58410d16e51d',\n",
+       " '0797be77777644129b6e907d0fb897bc',\n",
+       " 'd5d49a75b7a846e1b020bc3e49d62da7',\n",
+       " '10941d6aa34c45bbb0168578a486db2c',\n",
+       " '87b58399cb2a45c781ecf4d8e2f4e307',\n",
+       " 'e9d77e36c77b4de1bec4ae7951e38511',\n",
+       " '317c0b2c1d7d4652be7d51a1512e7f3f',\n",
+       " 'a2c77fe449e242c29ee83a43f28f8c58',\n",
+       " '6f8d84d606e94f66a9bcbd82941e5133',\n",
+       " '38aefb805fdf40f287e505154560f7b4',\n",
+       " '8e720f4f5fde454d94d941cc8c027d8e',\n",
+       " '33dda74031614e2dbae31283d2674e65',\n",
+       " '68b4b01b9bb2441cad5182ac4d0a8c01',\n",
+       " '97bcb73c830b4429b446e3ab718de1b1',\n",
+       " 'c510dce650ec469e99b4653b80284d64',\n",
+       " '9ed79818fd9d406790961038e968d87c',\n",
+       " 'b7fc935f6b4f4922ba2c9695f1c6f253',\n",
+       " '9c5c6635c12a48e4bf50a5823aa13bec',\n",
+       " 'ea3882dac1c345d4a5f24589b64d273b',\n",
+       " 'd77d8d6861104592aa4849b6a6e21ded',\n",
+       " '260626f4c965432bb342768026f132e5',\n",
+       " '56e3ecd4eb64413a9a943bffe1ad86af',\n",
+       " '6638ad84fef94984a6082915855935c3',\n",
+       " '37eb01f2393e4402acc1547a954d2b7d',\n",
+       " 'aeffe25827314271b00284704a245dc1',\n",
+       " '922a3f98218c4dc7b3a1a574e5bfab41',\n",
+       " 'adf1623b0f2e490b8f5f62cc15e3fd03',\n",
+       " 'aaebb687fb1945009cbe473dae7b48e6',\n",
+       " '5be6cd8b0f224c8d94e92b32b35f2d46',\n",
+       " '4f9ce579447a4bd4a76d6288cf2652e2',\n",
+       " 'c0456f7c254346399654aa02ba1796d2',\n",
+       " '32cafe4487d04f498fd721b105d2a1ca',\n",
+       " 'b321bdeb2d9e4cb7a017aed9cedee7d6',\n",
+       " 'b8c3466d506842e1af7ac94deee57b81',\n",
+       " '93a3dba99695448f8123c13bb8b779c8',\n",
+       " '1a22fc8c63164c12bc2e6dcc1e558f4c',\n",
+       " 'bde4793c8ed242dcbec5da1cf329912b',\n",
+       " '1c45a001e9894b4ab2e8ba13db0c47c6',\n",
+       " '0971bf69ee1043c69d3126c9e056d7a8',\n",
+       " '2c8864207467424e98a32256e13d78e9',\n",
+       " 'a5712b28bc7244b69b2f1f4bd203383c',\n",
+       " 'd14300d16e404c8087980ea322d7c923',\n",
+       " '96af51f4384a472c8ed57f5fd85d6d0f',\n",
+       " 'dc43cafd35384e169a90ff6a4dd7e603',\n",
+       " 'ee49f93bc1924ee895f2ed1cd9dd5807',\n",
+       " '883e971384d74702894bd84742e9e01d',\n",
+       " '4cb32ece2cc1453c8b283966d77a573f',\n",
+       " '87da85de0eb24c0291409b33f20f4fe0',\n",
+       " '71d0f05cdd8e49c1b12fa6c4290e19b9',\n",
+       " '5787fd6f5f934396bdba27c0982c69fd',\n",
+       " 'ac3c132b8fa646b998bce3f8e816bfae',\n",
+       " '9e17210401cb4d0998520db1313023c7',\n",
+       " 'd24cbb397d16417290039b255903c915',\n",
+       " '3ba47099629d48249da0ff0633c2547b',\n",
+       " 'f07dce036bf24a7e88af1a021fc97d42',\n",
+       " '165045264b9b4c2881f675d51e66a2a6',\n",
+       " 'feb749b3910f416597e18b6573f5919f',\n",
+       " '022e7607e8ba428b9213f763e032a895',\n",
+       " '5ada3caafac145ee946015d373d46cb2',\n",
+       " '9895042a53e547e1a4df189e719783c8',\n",
+       " '73c1c19ba70146e8a7703b0bdf3d29c8',\n",
+       " '9f5714b0d63547a3ac6b62c0a8795af3',\n",
+       " '754785ee7b6d44f5a474940eade68323',\n",
+       " '93aa78a59c044242a3c774d66eb0ce38',\n",
+       " '19fbf46edf5b42ea98ce35f4becb22c9',\n",
+       " '88f6394d368c4a4dbaa91f2ca3f06e83',\n",
+       " '2e8e6595709a4e77a0656833e14de87a',\n",
+       " '83a8ea55e3374628a8d2817f949f4a33',\n",
+       " 'bca22c47143c46dcbc350e393f43a260',\n",
+       " '32dfd090a865443cafc87324f0f583bb',\n",
+       " '069a75a888bd43ca9b425f76eba236ba',\n",
+       " 'e4240b6768a24309960f6671510e2b97',\n",
+       " '29068f3ec43b4aabb78f49b93bbc0704',\n",
+       " '25f21b35cbdf49729887854567e8f093',\n",
+       " 'bb01bcd9b6f9464fb19296279cd228ce',\n",
+       " '6c7c0873830241c5acfc7a5ff68c3e7e',\n",
+       " 'fa6aab7a0587489f8218465112ca1027',\n",
+       " '0a3d9574f789460c8ac54e69aa5a896d',\n",
+       " '9cdf3f1556964e0fb95dce551b9d80d0',\n",
+       " 'e02b0fea19f24f40b9e21aa065aec5e2',\n",
+       " '02b5b6915447495a9b40d28631e20621',\n",
+       " 'fd7e17383d434aff8e3c1578381a2be5',\n",
+       " 'f54d6b530dc44c81861a3421ddf85477',\n",
+       " '31016f3fe8f44f2a95ec885eea02a1a0',\n",
+       " '89a6bb3ef75045a8a13d0e1717536a7b',\n",
+       " '10ad95ca700f4d8095230aa7ccacdf45',\n",
+       " 'fa02f1851c694e98b9fc8b40f0789fca',\n",
+       " 'e307d59d2bda4bb498810bd088d58c26',\n",
+       " '09531751f0ad46c990d830f784d49920',\n",
+       " '463a5cc86f014d9b9c934019f8a6a3b0',\n",
+       " 'c6da3d870a664ae5a58d5b0f24f86afa',\n",
+       " '0347b993d5104dbe9c58cd87ec0e0920',\n",
+       " 'baf4197276f34479bf254bbb95457a78',\n",
+       " '4dc1052154984cdfbab206cb21389316',\n",
+       " '9201ed1c46b842fbb57c14abf6dc681c',\n",
+       " '67d60aae5043431f9296735656c1c578',\n",
+       " '96b88d3220c1438584ce83d20e7126d5',\n",
+       " 'cb24d605797f42b1956bdf34121734b8',\n",
+       " '6d180b3524eb43f5b1629e0edbeed005',\n",
+       " '422ab68c4bb44d9485a7e891de8d8bb2',\n",
+       " '2efea4ce3a1f489490e0eb277e2f9b51',\n",
+       " 'f39aeee6512644f78ae848bd0b17925c',\n",
+       " '922ba55579cc40a9ae6de8ff4d8f2fcb',\n",
+       " 'c94dce24f53b4fc597b0a9844cb9f2d1',\n",
+       " '92e5324c55f54c74bfeeae5a5ee472d0',\n",
+       " 'da29c33c41d54889926938c763559184',\n",
+       " '24175ca064f94ab18ffd755537fbeb08',\n",
+       " '737a9a163ab04803ab4811b083e84a7f',\n",
+       " 'c48c7210f7e04f42ba749352a90a2e14',\n",
+       " '33d1da5d12e84f039af98e53805463db',\n",
+       " '168384cba6a94e7ab9801b98eb25a83e',\n",
+       " '216c155d1910410e9ff3e2981305b0a1',\n",
+       " '9d2baad2bf734ec395e388832f375428',\n",
+       " '8f427298554343db9afef5f0ca04508f',\n",
+       " '91d16d31d0c84742ac7b410d634a77b4',\n",
+       " '49eead9aa61346fc8c20e6b320968f9c',\n",
+       " '2726ba58fdca4fab8b3d504088f5bb16',\n",
+       " '7cbfe53e55d94c9eab33049257b5b996',\n",
+       " '1829f7c9c7f64741a02e66cf07d7e4ca',\n",
+       " 'dc51c1d3980a41be85154cb81492f0d8',\n",
+       " '30babae9382a4204b65dfcea45b94111',\n",
+       " 'b3cf2c85a1aa43369f0d8bd3d1148214',\n",
+       " '8e66f9797e9542df8401cbeba12bf576',\n",
+       " 'e158952bb2a444b3b0137cc9ed09b2b3',\n",
+       " 'cc424f6c708d46c3b6ff0ccc532256a8',\n",
+       " '9f15d95cc0484a8b84f52c4e6aa6a3d1',\n",
+       " 'dd5123a7824b41fcbf9553691f4f8ce4',\n",
+       " '0c568b98747f4deda37a886d8176eaef',\n",
+       " '71b8843b17914ae2926ab5f7cc6c2afc',\n",
+       " '7123c69c56fc4ca79e510fe962fda11f',\n",
+       " '0662e249e7e846f7a34cdfb6b0ab97a4',\n",
+       " '37614a4fa3bc4b6892ed640136432595',\n",
+       " 'ba977631881b4e7887541f304cbdbf84',\n",
+       " 'a9e9604bbe8546db8379df385c6775bb',\n",
+       " '29cbaedfdc4842be8af5ef11fb0ae880',\n",
+       " 'fb96faf7d811465197ecf0bebb5ef7af',\n",
+       " 'a386736ae3e34fd8810f61554daff3a8',\n",
+       " '3322d87785f74f7bb03fff6184a88959',\n",
+       " '8d56b24bcce647ef92bcea0b9e730535',\n",
+       " '9df83b49b7b347e4880663e59fd15a4c',\n",
+       " 'e6beb25ea09243edab2d0b917d4ed658',\n",
+       " 'b788547e806548e4a6c0be68c37783c8',\n",
+       " 'd0cc604bb0aa47158eda39e6ea990c9a',\n",
+       " 'fa8ea860f8604a78a544a21ff65a9fa6',\n",
+       " '73efff02cfa3420dbc5d8412282093e4',\n",
+       " '8a891d866e8f41839dfa04dcfab7704d',\n",
+       " 'af9020b82fcb4f7dac6466a661c26915',\n",
+       " '10ea6f99054b4844bb42258530fb0058',\n",
+       " 'c4a6866e6ce84c468caf7cdb6e6d535b',\n",
+       " 'b0c1794f0f7f4f7c8fd21ab6598eccac',\n",
+       " 'ac530d4f83ee4c418e52245ab06630c2',\n",
+       " '860d9613fdfb4b969cb5dec21b29be06',\n",
+       " '220bd37a37ca4f259844deb952c32245',\n",
+       " '2eaa29f0eb34476fa3f67669e3da5b62',\n",
+       " '550db6a68b394795926cb1faaa1e0a52',\n",
+       " 'e86011e86a524ee8bf17f674fbb687e3',\n",
+       " '7d7b1c4d811d42d4aa80f5582c64f9ac',\n",
+       " '5a4c8245a6394f7db6562a9324683a03',\n",
+       " '96aa6bdf11964b2cb2531bba32f34249',\n",
+       " '168db84dd98248a88d5de07aa13496ae',\n",
+       " 'd67d59079f4a4c10b80d571cb438de23',\n",
+       " 'd7158abe270d496d9985420c9b03a077',\n",
+       " '2fde1e21c14a4f90a113b395a3712743',\n",
+       " '2510f9f4fe8b4bb78bb880469169476a',\n",
+       " 'a4a1b06704044d4eadf8b2e6142e79c9',\n",
+       " '0a466939b2d54d5293770a7b1bfecda1',\n",
+       " '79aee364cb994e36b1adde5b8e5e3086',\n",
+       " '5592d4e1f1ec4307956ff1a56c7a9a95',\n",
+       " '3deee26006c8406da04f85d82ef7c52b',\n",
+       " '4f3bb845b65f4ff9b6dc44bc0e6ff645',\n",
+       " '63b8bbe1a5fc461dad75fa7aed4e1382',\n",
+       " 'bd833bdc626741e1bd3af21e90b598e0',\n",
+       " 'dfb14880e30d4dbf8caf0074d68d97c1',\n",
+       " 'f8d84d5b0edb487db8cd381569a5d79e',\n",
+       " 'b43b34c5ec3449bebe6eaec3ecf141b4',\n",
+       " '57c7120546b1405984982cfdc6198077',\n",
+       " '9a53cef8c1344f75ab49b328c11f98a0',\n",
+       " 'bf49b6b8d8594e7588b1b108007034b2',\n",
+       " 'c0b6a58b37444cc0b556cfeee6ca8b55',\n",
+       " 'a6a8db63ba4a4baf9d32f320c21c6313',\n",
+       " 'c61e63e1f9b34e3dae8129e3010d8706',\n",
+       " '6be3ec9a2fca469f9c9915d056cc8324',\n",
+       " '746efb41eb2c40a88aedb82e5b29dfca',\n",
+       " '46c5071d7e204ff888dce1ed7dfffff4',\n",
+       " '04b68080947d4c96b13f7e9eb428070d',\n",
+       " '66309b0484914b099cda9744b3bfae55',\n",
+       " '5fe570e4ef224d41aab64aab07f7ef18',\n",
+       " '8a698955d62d4505be78c0914e1ec7ab',\n",
+       " 'b1f0063f145e4c2f9900dc7ba64a092f',\n",
+       " 'f3ab3e53fb204ef7a6d30c6513e42bc3',\n",
+       " '03295952d1d241b3bc7ed1abd990e32a',\n",
+       " 'efd0faadabbe4dad9ae7af70fadee972',\n",
+       " 'e398133bbea24e48af9f8cab8f32310e',\n",
+       " 'db3e34ee20c043be804e1c1a9db594fb',\n",
+       " '991a44d7e9ab4ae486964e6d26034918',\n",
+       " '55910ec6f0f64a49b167553ff9a20f61',\n",
+       " '1d9d5538d4d34e3e992e3891ead9cae8',\n",
+       " '863753ca91b74fbebd46d2a3f0f0b7a6',\n",
+       " 'b13efd943e7947259e5f807b2229ccf3',\n",
+       " 'b0fddb4609a54fc8b0dcc6cab57161a9',\n",
+       " '0f943749f0d24d8e94fd1dc209b09b9c',\n",
+       " '10af8065bf0b4eb5aca05993ac546176',\n",
+       " '23ed6a4ca13343099d920007034f7d34',\n",
+       " 'ceb0455e738b47d7b3b7fd99506dbb65',\n",
+       " '940d178005994c83b17652f086e292d6',\n",
+       " '69ee5697b0b742dd86990299a8c8cefe',\n",
+       " 'ab48f4fad8d34410b9949d071abdfb42',\n",
+       " '135ba4ad88fb4300a560a7323535e37b',\n",
+       " '66c9e4f32ca8418d9922de33bf84dcd4',\n",
+       " 'c776e9698a424f8a9271f3035f02c041',\n",
+       " '7544393d08454b2191837910d05db972',\n",
+       " 'd9a62e36fdeb40799071931e14b52963',\n",
+       " 'dc66a111d00344b5af5963c0d118d37b',\n",
+       " '9d1f6a6c09a14334b3adc4799cfd8d7e',\n",
+       " '27587b7eec344848a4fd80724d74c916',\n",
+       " '502aed42f63440c791cd27e8c4d0ebeb',\n",
+       " '5f1b1c46c7b04b27a8821b2c26425cba',\n",
+       " 'cd51cdfa35934fe19059aed56da1e35b',\n",
+       " '6fe71508c191419db508a9958ccf94e3',\n",
+       " '3e33955d1f0c4729b92f5560bc9a5f48',\n",
+       " '5d6837486bae4495857be79cac8fbab6',\n",
+       " '7e526fdb9dd149b39718bbdf652fa10f',\n",
+       " '6a7504d312b34b70925f761ec113e06e',\n",
+       " 'cdbc28b9da1846d8a6a9209fc4052d9d',\n",
+       " 'd5e7d7ca7c6b46689385ef8ec9acf389',\n",
+       " 'c521c62ec85349208aef52f72a0ab0a3',\n",
+       " '5af3b7b361884f22b1ccd94b1bf56718',\n",
+       " '3da65a3dfebe4074bfee9df2216ac6a0',\n",
+       " '989436d80f11412786de32631a20164c',\n",
+       " 'b8934952cb644d6186db12fea1f36900',\n",
+       " '8961e893462e4750ace14ab8089730a2',\n",
+       " 'df6d995cc7a94f19bb82c56be27bcb8d',\n",
+       " '9802e6e00fb641dd901a90b7c101b578',\n",
+       " '2c77bc0c03d245ca8692ba2a05b9ff57',\n",
+       " '2c42f0f9da6d4ef9986bc069c1dde12d',\n",
+       " '6263199a92654cddbe70aaaaee82b0a3',\n",
+       " '20c172b306cc4262929dab4a7b734760',\n",
+       " '2961b6f93919473185e6c407801e4a2e',\n",
+       " 'f430faeec2984de0b12cb14d35200475',\n",
+       " '2455745f94554dd694969decf6232d67',\n",
+       " 'a8307d768d574b71a50e69aa3e906913',\n",
+       " 'a8af04cdc83b4bcca4b3334cc4f87a31',\n",
+       " 'e0105deb25104259922024363eb278f9',\n",
+       " '6dbba361217f4f65845ad1e58f52f66d',\n",
+       " 'd6b391d2232b439d861c85d81aeffd13',\n",
+       " 'f0357e35b61649968204afe0afcaa986',\n",
+       " '84b5f1ea889a438495be7f5286516439',\n",
+       " 'fefa48d67f004ae991280d670ff0525f',\n",
+       " 'b49d4c918add4577b2ba23a7cae95a2f',\n",
+       " '8227801e19af44e6bd5552ca958dc83d',\n",
+       " '011c11d3da5b45aa98eca658fe04934c',\n",
+       " '5e5348434bf94d50a57995b5e64ebf13',\n",
+       " '6c704ef7e4cf4ccb9e911e34c12a503a',\n",
+       " '739b2782f9ad4e8cb9b5157bdbc2f92d',\n",
+       " 'cdb3dbcb67fa4e4b84f2b89cc0fa35e0',\n",
+       " '1bea36f66ac64c95b895cec1540da580',\n",
+       " '8566fba4d0aa43b293cf64db66b53051',\n",
+       " 'c3ec323138fc42678d77c8c483d8e73a',\n",
+       " '008c74419bd544899d9d83212e531ba5',\n",
+       " '3c3237238c6b4bfe990734daa6344fb7',\n",
+       " '7a44fe6828c54c469b52a0d215e38135',\n",
+       " 'bbbc1bbb596d4ba28b78eee1908e57a4',\n",
+       " 'bde8eb1d0f314d728fd3f94cce68c5ae',\n",
+       " '39c307c0404141c1a51784fcce1834f2',\n",
+       " '3a82fd7dca36498495bb3b6400656bc6',\n",
+       " '1c790b01db7a4bfda6a59542322ce975',\n",
+       " 'f56b41ea938547eaac61edabd71e0cc2',\n",
+       " '55708b9971954a77a64440b4e2a4d437',\n",
+       " '6052cc5180aa43359948f92a2fba7fd2',\n",
+       " '2a9f782eb0b94d2381c2b902b89313db',\n",
+       " 'fcb302874996442296870bdff15b2d4f',\n",
+       " '0864ff8559dc43be94959f7493dd6067',\n",
+       " '7bc9df2622734502bfacdd235b66edd1',\n",
+       " 'a01170164ec84b7194848a9021586d99',\n",
+       " 'e2c7ea2f03cd4100bef06b31c15d5df6',\n",
+       " 'b85b7a4a8660444fa704ecef67e5978c']"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from langchain_qdrant import QdrantVectorStore\n",
+    "from langchain_core.documents import Document\n",
+    "from qdrant_client import QdrantClient\n",
+    "from qdrant_client.http.models import Distance, VectorParams\n",
+    "\n",
+    "dimension = 1024\n",
+    "collection_name = \"ai-safety-sr-arctic-embed-l-semantic\"\n",
+    "qdrant_server = \"https://500cb0e8-ea08-4662-b4f2-3eca11e635da.europe-west3-0.gcp.cloud.qdrant.io:6333\"\n",
+    "qdrant_client = QdrantClient(url=qdrant_server,api_key=os.environ[\"QDRANT_API_KEY\"])\n",
+    "qdrant_client.create_collection(\n",
+    "    collection_name=collection_name,\n",
+    "    vectors_config=VectorParams(size=dimension, distance=Distance.COSINE),\n",
+    ")\n",
+    "\n",
+    "vector_store = QdrantVectorStore(\n",
+    "    client=qdrant_client,\n",
+    "    collection_name=collection_name,\n",
+    "    embedding=embedding_model,\n",
+    ")\n",
+    "\n",
+    "vector_store.add_documents(chunked_docs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements.txt

@@ -0,0 +1,15 @@
+numpy==1.26.4
+chainlit==0.7.700   # 1.1.402 
+openai==1.44.1
+qdrant-client==1.11.2
+langchain==0.3.0
+langchain-text-splitters==0.3.0
+langchain-community==0.3.0
+langchain_experimental
+langchain_qdrant
+langchain_openai
+pypdf==4.3.1
+PyMuPDF==1.24.10
+pymupdf4llm
+sentence_transformers
+langchain_huggingface