{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "% pip install numpy==1.26.4 \\\n", "openai==1.44.1 \\\n", "qdrant-client==1.11.2 \\\n", "langchain==0.3.0 \\\n", "langchain-text-splitters==0.3.0 \\\n", "langchain-community==0.3.0 \\\n", "langchain_experimental \\\n", "langchain_qdrant \\\n", "langchain_openai \\\n", "pypdf==4.3.1 \\\n", "PyMuPDF==1.24.10 \\\n", "pymupdf4llm \\\n", "sentence_transformers \\\n", "langchain_huggingface " ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "BOR_FILE_PATH = \"https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf\"\n", "NIST_FILE_PATH = \"https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf\"\n", "SMALL_DOC = \"https://arxiv.org/pdf/1908.10084\" \n", "documents_to_preload = [\n", " BOR_FILE_PATH,\n", " NIST_FILE_PATH\n", " # SMALL_DOC\n", "]\n" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/Users/jeevan/Documents/Learnings/ai-engineering-bootcamp/AIE4/mid-term/ai-safety-chatty/.venv/lib/python3.11/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:13: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from tqdm.autonotebook import tqdm, trange\n" ] } ], "source": [ "# Embedding model - snowflake-arctic-embed-l\n", "from langchain_huggingface import HuggingFaceEmbeddings\n", "\n", "model_name = \"Snowflake/snowflake-arctic-embed-l\"\n", "embedding_model = HuggingFaceEmbeddings(model_name=model_name)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "from pdfloader import PDFLoaderWrapper\n", "from langchain_experimental.text_splitter import SemanticChunker\n", "\n", "\n", "pdf_loader = PDFLoaderWrapper(\n", " documents_to_preload, PDFLoaderWrapper.LoaderType.PYMUPDF\n", ")\n", "documents = await pdf_loader.aload()\n", "\n", "text_splitter = SemanticChunker(embedding_model, buffer_size=5, breakpoint_threshold_type=\"percentile\",breakpoint_threshold_amount=90)\n", "\n", "chunked_docs = text_splitter.split_documents(documents)\n" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "# Recursive splitter\n", "from langchain.text_splitter import RecursiveCharacterTextSplitter\n", "recursive_text_splitter = RecursiveCharacterTextSplitter(\n", " chunk_size = 1024,\n", " chunk_overlap = 100,\n", " length_function = len,\n", ")\n", "recursive_chunked_docs = recursive_text_splitter.split_documents(documents)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "import os\n", "import getpass\n", "\n", "os.environ[\"QDRANT_API_URL\"] = getpass.getpass(\"Enter Your Qdrant API URL: \")\n", "os.environ[\"QDRANT_API_KEY\"] = getpass.getpass(\"Enter Your Qdrant API Key: \")\n", "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"Enter Your OpenAI API Key: \")\n" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['8dd5b1e7fd464e2a90c28a8eea8b0cb9',\n", " '906e0c268d564dbc89c0b8398e235407',\n", " '4b81191a4cc94fbd835dc9c942e9543a',\n", " '25c3b7fffa8d4bc29790057fe2f4d025',\n", " '3ad5906a8a274b56bd05e4ac39ffe459',\n", " 'e3fa01bef57c489ca014be2e589b7ef1',\n", " 'af5fc5121c6a438a8fc5dea454b7e92f',\n", " '80500cf02d5748c39b1c62288459c306',\n", " '5db6eebee14b4aafa948e4f9aa4f7aa2',\n", " '99385298e8744643822e01525bdff89e',\n", " 'eddc9704820d4005b7c62a5085f69454',\n", " '4324a624f4054ae5baa7270d9f6aaa56',\n", " '9eb24bea31a749f1b7a86ac2b186ec14',\n", " '7e9c9763bebf40cea1833ea6ad376eeb',\n", " 'cc8846008cac472e88eb16497c560a15',\n", " '5af0886e387449fc89f1d0e82c32c590',\n", " '824ae7c1c15a43c8b62713f02d91e0b5',\n", " 'f0ef1b30251b4429ad7d902b85fafcf8',\n", " '314a75e55d1b4c1fa46f49610d745f95',\n", " '66828a5f9536480bbd08d94f087bc44b',\n", " '8230b8add982486f9ac8e120a27d3aec',\n", " 'dd1c75bb5c1441468ac8e7d4595bf0b9',\n", " 'a9b1b1b87eeb48b78ed4cf6adddee9d2',\n", " 'eeacab16c9d94d08a791c516e0a65f6b',\n", " '187badb4dc064743898f5e5218114250',\n", " '0ecc4e873fe047ce8afc33e19fe40c3f',\n", " 'be7b81185ce140229bee6d1306120528',\n", " 'd8a9a361dc8a4917aefc2e0a17efafad',\n", " '7ca3b3bf947e449e8f58cd4fca12d884',\n", " '07a8b46151e74641b97ad823ef91082f',\n", " '421004f00dbb4a47a81c424ad5f64e39',\n", " '53750f6e3cfa481d971d7e5a6b9f55c5',\n", " 'e9f9524be6884599893590ad5acdb12b',\n", " '446225894d9747e0a888b596875ac83b',\n", " '7677215528c44061bfb018e42a13e528',\n", " '183002bfeeaa4b75968dda61451c2f37',\n", " '818cd888dd6b4735a602949aea2ea900',\n", " 'c803c1690b5549a5a13578b2cd757b17',\n", " '864bc201486e42bdb4caef6a266fa1ee',\n", " '02cae44fcb9e4d2eb60a6da08b01a4e6',\n", " '384e2a4f36d14359b22b0c0cdf463cd9',\n", " '6de4fdb915164aa68e076a2d6e1913ae',\n", " '8da0e4ecc526416ca3be4043879ea17b',\n", " 'e63359693473494d922f996b57c65d3b',\n", " '76313322b20b4e368f2aaefbf911df6b',\n", " '455bced7b2544b69a5c547d998548748',\n", " 'd1abce93130d4eb49cdc1aca8b7c9c60',\n", " '966dde23cfa144b899b60a3659f32eb4',\n", " '4fe01c77f1ae4b70a18c56fc6cb9679d',\n", " '7aa14bc014fa445cbc061c47a3fe1c31',\n", " '0aa076f900614fcfb37883057c67e6c1',\n", " '2bae6b24f8234792b914a75712fb89fc',\n", " '74fb5f5dfdb8413a9afaaf472a009ac2',\n", " 'b1c95e4126e842d598eae6fbc455b82c',\n", " '2e326893157844fe88f762b96aa46b99',\n", " 'd27b5c85573d44658a4d338c39890629',\n", " '1f967a5cae05418d94f9f3d07dcce74e',\n", " '7667f8b8e5914417a68e7d41256cfc98',\n", " '7fcd90b390494d3686e532a6528bd021',\n", " '8c64aecd850b48b3bac216e73ebad1e0',\n", " 'ca845f98d6b44c02b9f9edafcd75aa2e',\n", " '571b12484be44ea6b406e6cdcd0662a5',\n", " 'fa099bf314614075bf8bfc58110f52f1',\n", " 'b98bfe7e8c234ed59b5d893929bac64e',\n", " '3bc58c54665d4a4ca394ebb13debfabc',\n", " 'a276d45eccd54056a8855ccdb5907df3',\n", " '0f9ac79d967942d0b56b9fe200dc7846',\n", " '07c5e3794b244e28b384dad31a2c63a2',\n", " 'deb29326ff1e4fec934c35710d4e0dcf',\n", " '5278dac4122044879b9dd1a7c557b7a8',\n", " '041338e04daa4482a7d65c311ab0f3f5',\n", " '1ed9bbc381b1423a95ef935cc16e277c',\n", " '20bb221a9b654bbc99edc812475adc12',\n", " '188fba73978143e8a22370774e1d31fd',\n", " '3a65a41271a947848aba4939473d0f85',\n", " 'c5cee035027048338a81b9bf0830cf57',\n", " '03325dbfeb164512a565172407ae0ec3',\n", " '4494b4f19cff440281c034ede5e675e1',\n", " '2b55dd78b0f5461d917eeaec2a75dcb7',\n", " 'a2aef7ef0741438fb643a4fb225f1ffd',\n", " '3886d787efdd4d2bb3fc702ffd911db8',\n", " '71455bc57478429e8a1269ce7332302a',\n", " '0c2896f473c749f9888b5723ee834a5d',\n", " '1a8507e2597049dc8287d21172dfe518',\n", " '943e65c033774aed99116750cfbb5f5a',\n", " '78bfa949218f48c7b365569f2c3396ff',\n", " '6812e197478a40bf86cc363d11fc0856',\n", " 'e9cd2bf8ad454aa7af9446e62d9d845d',\n", " 'ca970fdebb1e4d79853090a7c73722f5',\n", " '84400697300c4d468f5c58d09fd63d56',\n", " 'ad1bf5c566a147e6b66f9f3502a227fd',\n", " '0c620c86587a4ae8975a0d066eb80e97',\n", " '1aa1ebd384774410ac011ca3e535808f',\n", " 'f30f7dba0182402c8abce8d9b07df99b',\n", " '73dd6c906bd14d3494dc8def54680e0f',\n", " '632c7eafa1c048979b65c5e8ecacb98e',\n", " 'b3edcc98879747dbbed2e8b6e19e1baf',\n", " '4f4bef9639bb454e87cb61051e2d4c82',\n", " 'd0c059bc4e04474f9355c87964b3b470',\n", " '412fa6deb62546c4a092988f96ddb425',\n", " '2283f3ef3e7141738c8966fa4333ba05',\n", " '7ed4340667d643b193b45f7f21a238ed',\n", " '9209830e86c54dbd9974cc737bbdbe91',\n", " '89e28024e5f14377a6b2efa1997f370a',\n", " '2f4b769a22b24bb49bed8917adae1f9e',\n", " 'fcfc743d434d43b886afb80c5377e1fd',\n", " '300f3cfca0874ee2818241856f2175df',\n", " '55ea80ccf78f4c9cb622c1451951e723',\n", " '4b755a511dbd46f79eb4b3bda119e79c',\n", " '956a7f2f70854e1a82dfc542fd761492',\n", " '190c9cd3f01a4ea894877f4ab35000bf',\n", " '381a0acf443244c78f303b9f6b72535f',\n", " 'd7afcec2075343b19d1320605cc41b46',\n", " '7898f81259be4c42b44cdfd3b41aa25b',\n", " 'd86e65ae1f8140e299ccf27583735b7a',\n", " 'ee553712557545a0b0320adc4d563bca',\n", " '6fb6a0b739a64b909cee096d722a4f6e',\n", " 'be1514e798f04581af33447b86f002c8',\n", " 'c92113c2d4344ce8a10e7a6c1d089f4e',\n", " 'dc47dd1e29bb49768c2b88054f91c69f',\n", " 'd72643a36bf6415795a3694f93e5c376',\n", " '0709e83c2d974cb7aa30b17f8a5e5050',\n", " 'f4bc420a5b4c4cdd91c4441837fbcfae',\n", " '712e9898ff5f44eda54aebcf54931760',\n", " '1916383ddb404a32ab833c2add8e2511',\n", " '4a01c037505943a3be4fa183de7d5c73',\n", " 'cb29c514ccf8416491654441749f9889',\n", " '57cc735133754979862b8dd27ccf45d8',\n", " '6e28a749e0ee49d8b6ba562ab268e474',\n", " 'd915b593c4194759a4ca48304ce54b56',\n", " '83bf4923ef4847b1b3faa0a85ce85d9c',\n", " 'c7165907a5774d7a9cfe034328875f16',\n", " 'fb83c16fe2ac4a0b8a2cdd3a099b0751',\n", " 'd8d56a63f4104e1d9b06c5c8d6246d4d',\n", " 'b92d479d4fea486980ce1133ef0d9049',\n", " 'b264b04721c14738a6e018c3d089e3e2',\n", " 'bf50880e770d4e2a80415e87b8f95788',\n", " '5ca16b29007f4b919ac1f3fdf261aa10',\n", " '30f3d89d1ae042afb3b745451e0a5fdc',\n", " '41dd324a662a4e79935980dc8e53ab8f',\n", " '7590bcc7d6d540b1bd92a7ce69c0e9b2',\n", " '72973a23774d4bebb9c42dcf885ae06a',\n", " 'ce3e692e73084116ae834e72349032a7',\n", " '044469614deb404f8d3b1860907e0f75',\n", " '3805bb3205c5411daf2a64a7742e59fe',\n", " '03aa772e62b44423b75ec05c90e8687e',\n", " '740e824d876d44c7b30599b4dcb8eb44',\n", " 'ba27340cf2144d15bee2a5f5b7e00622',\n", " '4c6a7847bc554fddadd0a884c26612bc',\n", " '8a13a8d664c4453b8f71c01b28ec8dff',\n", " '113a3db0cd0d4e0f8067c5fa074967df',\n", " '28af19cb148f49049336aa1b52c14a98',\n", " '795bc0bc5a9c4ff8b472e2a9c9c59dbd',\n", " 'fb36fe1dad1c4280a7186ea5c20e64dc',\n", " 'a50e1d30a67b4144bd8ce5ab32f1cd3a',\n", " 'f1ccd9cc27b0414f96243f1c63a07fd5',\n", " '3d2b64f6ceb74744b6b8374728142334',\n", " '15f314cbb8a14f9286a814cafef76192',\n", " 'ac540651b7a34d50b70e4c44cf25b3ca',\n", " '6b87356c50d1404abe0a676b7f322a72',\n", " '67902b525d014249900e54257590f7ea',\n", " 'fc1189d79c824a74bd60dd5dc341aa2b',\n", " 'b26845eaa60246399cef48b0a13d11b5',\n", " '7b592acd329743a8a7a3b2569a048416',\n", " '189e134f601441cc9f1514a778e3c820',\n", " 'f7f1425e4c2d4e1fa9040ee85d368bf1',\n", " '0297a5233d6f4275baf0a9957b0dc586',\n", " '5afeb076d11841c890517fc92d0aa6f9',\n", " '79fa6ed91f7a47b6bd764e1c8b412fa5',\n", " 'ff4b75e4daaf4588ae69ba2f83816c15',\n", " '84b7e45334a3477f8d8a64e3504fb620',\n", " 'dbf780a26828491da830425df5a7a03c',\n", " '25a6912b64f442f99f5787bef114ebff',\n", " '00c07386007a4dc18072a431f7cf83b1',\n", " 'f84828d74c0c446389732b8eb4d6570f',\n", " 'a12ba2aaf84640a8816d9ce8e8a417e9',\n", " '02223a887b2c4ec0891d45e75836b00e',\n", " 'a62632e5379a4af5be885b4750d18650',\n", " '5ef2d149eb314d879897648027e7e8aa',\n", " '1a2c86d6906141b18700239300599566',\n", " '584530895cc74af58cdb016c0ed63bc5',\n", " 'e19a1e82a1ec4884a7c72f2996ac927f',\n", " 'dbcd348813cc4365bf65c549333e669d',\n", " '8bac146f886b4272bd40f51adb35c32b',\n", " 'cae2438601594d6fb39d99d617fe6c0e',\n", " 'e8306f6a959b4a219d096b968784c44e',\n", " 'ebbdefa7da15403294655048c6fe3624',\n", " '60579bd40852405b8345114456963981',\n", " '211f66d20b5c451d91f310594b854ea0',\n", " '6d726194ac8641a6a5f6d8ce3f192a7c',\n", " 'c0154e0f56b049048f9bcb7f718173f2',\n", " '081a6179661e41f69ab10b92027d161e',\n", " '2c149c226d504053bfa94532a850efd1',\n", " 'f833250f67bc4329922a5a7f7b7d07f7',\n", " 'a0c1e7c49351406ca3567622b6b1e38e',\n", " '72abd1e5dc824d6c8852f7331990b6f0',\n", " '35c02c07ae484045a325940fcbe098fb',\n", " '63b83f297d1d486e84ddb42c2af32900',\n", " '17fb247ed7bc4599a8de06966e744b2a',\n", " '371ea02f2a7e4e11b82ed0593a26a806',\n", " '6847a89d7a944bf2bd95430c4d63def7',\n", " 'a186a88983e64831bf42523b6522d706',\n", " 'e3538719cffb4cb59efa815b27b4bf81',\n", " '8ae7bdfed4e249dd98727cbb4f34259d',\n", " 'd4f18e23e8444ffba7b13661b22ba1a9',\n", " 'ab0aace578b2457cb10966f9a57dbfce',\n", " '641437d62c1940d7a7d0711391802aa6',\n", " '3f88fe5f258244d581a053a53b844bdc',\n", " '5022a64b46dc4989b2a919193cdec7da',\n", " 'e620e702fb604457b4724ce0f753138d',\n", " 'ff9557d7d3c446a39829de03605a5254',\n", " '5314c1de556d4b96af06fd3ac37cf1f4',\n", " '56d98670e8f74b1881bac44ccfb9267d',\n", " '2d06bf2cf37a48f98708e345e86a6114',\n", " '1d6be38e70b74ce69cf68c32fdef0b9f',\n", " '936184dab9ab4ff9905cbaccc0844e61',\n", " 'ec080c630727417fa858340935e0557d',\n", " 'f6792c2c58774d4e9ba97575ae5a9ddb',\n", " 'c1df1df5060e41899363e2a0649100e4',\n", " 'b2eb147898c64b359c951297318e6831',\n", " '216d56dfdaae4c098b826c2c6dbe8132',\n", " '7921cab4d11646168b5d186794f5db24',\n", " 'b89e6d297f064e708c4fb903c6ebf15a',\n", " '4ee471c58dbe4185b6968113228bb20e',\n", " 'd64a74ed5b57427c9c6ce98a9f945b70',\n", " '58cd9cbd849c456e85fb72a4abc5c69c',\n", " '2e1835610aa749c896c8c165e3d84470',\n", " '8008134cc1c44751bb95a3270cb89a44',\n", " 'e4f098d6b9024392adc396aad0efb94f',\n", " 'e8fcb91812d048efa5ba38a46cf40531',\n", " '987e19cc2d674e2aa0d555af45ef874f',\n", " '2816810a2eea4f0081baae4b28614796',\n", " 'ffd0647c27664a779dabf843fcf83981',\n", " '7a5e718e26b14f4daf674c901b3dde93',\n", " 'a7248347d0b743d7b5db65f3b1b87cac',\n", " '640f3399c6c340f19a11919a6402ba85',\n", " 'fe73657d7e884bd48d93afdc193b73bb',\n", " 'a9c649a44de94b5f82af06c804e3bb08',\n", " '8265df5e7847443a8c91478ae1cabf72',\n", " 'a3bb9ab12f814c4faa382b586fe13680',\n", " '775f072a48674d6b9fdd0671c4064891',\n", " '21048806b4c74f5db5b7f873c45adedb',\n", " '1525157eac174ab684089f50e6c29969',\n", " '136132cedfec4e3bacf2a8adc1fbd50f',\n", " '08099904461749ddba96b17b61226622',\n", " '1da17dd87dd448c4902f8e3a1ad1c51e',\n", " 'fa9c4880b4a34b9aaca3f5363ff1b7f0',\n", " '30eeafb17ae74a42b370e173e22abfea',\n", " '0d3ad50f8c524e90b6c440865aaf63d8',\n", " '51d52addb9df4c04afaf8f008fe89259',\n", " '2aa50c713fe241f1b9b44107c0d47945',\n", " '8615b8c9442c4031aee25316cdfc7cb5',\n", " '8f8fd8a1edc043ee82c77381bf39a83c',\n", " 'f15223b9a98445f2b7613e518e7bca83',\n", " '6a437e8d6655430aaef679f6c6a84831',\n", " 'c0a3331686754cd9929c4abd5d81dc7f',\n", " 'ff36401f33d9424cb7112033ead5f58c',\n", " '841220093a1242f0b04a4ac8d852e280',\n", " '655ec122f3d24c069eecacc8e8bc8f82',\n", " '6da4281d97ad46ed9ef6dd169c640afe',\n", " 'eeef919024d54063ae3cd6c6f8f7a73a',\n", " '9f55e4818e0c4bef82dec178dc64eaa4',\n", " '8a652387c6eb424288a0022be058d00a',\n", " '9a19d1114e674c618d23a1299f14f1ba',\n", " '9e4b5fdbbda24ed5a2fbbc3923847a44',\n", " '8b442e6de7bf401b8300c567a642a759',\n", " '20c8152b25514d018eeb8542b4450ad2',\n", " '5ae8d5d8230f46cab713cecbd97c847e',\n", " 'e159ce134b7b46308fc919b22a9e808f',\n", " '5a0d312175db4d15b85c0255b68bc027',\n", " '6f82139d091145cba88a7f0fc367063d',\n", " 'ee60e7e10d924f01b530b0291d939aae',\n", " '3b73fe9ad953458bbd3d11f44b85fce5',\n", " 'd15e56f06ba24e3cb6a1c4dd0568201d',\n", " '16f3d27489ff423dbf7d027844d957db',\n", " '6091bda6320149a1af5ccfb541e75148',\n", " 'ec527d7203164f07ae7349cfa33829b3',\n", " '26b4090286e346b4b686b13360cceea3',\n", " '856dd023a8cd41108eea38b403eadd09',\n", " '5434f50e81db44e5b80d3bc8816eb5bf',\n", " 'e9022b1d39f24ec09981e8c66478705d',\n", " 'd3f895478ed74239b4bc88e04e215f1a',\n", " 'c1fb1bae1b5e42f8a65f8260c259e133',\n", " '2a86b477a07e48afb2658742c30494e8',\n", " 'da64e968fc3047089de2ffa4b62a8c0a',\n", " '0a42ebaaedae4f73914398ad1486afb4',\n", " 'dc66a40fba5f4e348216910fce0d2428',\n", " 'e125ed2241b24a31ba40ed768a21d4f6',\n", " '850eda324b734ffdad0fe63c92c91038',\n", " 'd9e9ca7b0a634afdac1f4da62f2dcddf',\n", " '7cf441e12164420fa8b58e0aa6d244de',\n", " '0a6c2f48e5094e3399f7e1d0f38d873f',\n", " 'c8c29dbfc7f840d7a7195aa74388b30c',\n", " '07a90e50dcbb4352baa6636e9b687aab',\n", " '85d60bfe6d684c1a8578c1d6710c867b',\n", " 'ceb46b27e8994626a6d6d1c1acabff5d',\n", " '506dd325656145ebb7d976de3b4953c0',\n", " 'a83e954196874363b13c7cb3d7d8e025',\n", " '16fc61ea959d4427b3fa723d7e58f2bf',\n", " '3d4599e2ad2f47deae8c1c25d30dec68',\n", " '260482de224a4ec998459a5d2f9384bd',\n", " 'd480305e9fc34a55b8f146343fe1dd8e',\n", " '9851b805fcf54766bd482d5a0d4a8d0f',\n", " '5147fb1a9a904ff09b7c6885567fa94e',\n", " '509f1c4ef2b348af821461d751850e93',\n", " 'a045c48dd9444211a2f0087229df189d',\n", " '3b90d03add21451aa40990b1f2dad9e1',\n", " 'fa68102a0555422db1cc0f3822496a48',\n", " 'ade010ff9a2644a38c7c3de875a3ac78',\n", " '390f54300e1f41ac9224da683f00d31a',\n", " 'aacaee53fce14e9395259a0609cc1646',\n", " '00ca0c3998b64339874ad036983a0922',\n", " '2b3b851a8bb6422abab843dc2148255f',\n", " '363ed4276aa144b2baeec9dbc1fba38c',\n", " 'e9bf60fe2f184793b37f268ffa486abc',\n", " '98b185639a5d441ca60d7a5fe7620f8c',\n", " '9ae478f07ca3465a9a447b3c7eab4b26',\n", " '01fc2f0676754dc7baae898343e2bebf',\n", " 'cf3dd2f39b1a48919b90571555e4befb',\n", " 'b0ef26aca0404662b5706ccfa737a52b',\n", " '9afcf7171eb74e628f99fa44a753c131',\n", " '26f3917cc2274e998e115212273fe2ba',\n", " 'ec369428b5fe43138d049f293dcd21a4',\n", " '947d581cb3cd4555933504b8c64c54d3',\n", " '16b495459a4e4bfd96a12655defa9551',\n", " 'af15b6c943834e96a24363fbdca209f3',\n", " 'a01a9d5cb1c041889bfd1cf29cd4c08a',\n", " 'd3d0ee8f3c394ed680324fdcdb442241',\n", " 'd56148720f974dffab53a4e8917c3833',\n", " '32593dde550f45af9a36349bcd63192c',\n", " '1d43c4caf83b470897e96410f4dae5ee',\n", " 'ee1cd8b353cf45e3a88ce76faeebc9a1',\n", " '8e7135cd24764e94b8d04e15ec86b9c0',\n", " 'a7a918cf594e4a2992398acc924e6015',\n", " '13837b8571154abe83bb0b8d8e08d406',\n", " '63a4ce49d82d4da6bcb4da66db26bf35',\n", " '2214b1db8432499286a9ad49d8a2391c',\n", " 'af23e4413b7c42cc982b011d6432ec5a',\n", " 'e4b5e669227c4112aec7a7c53f568b75',\n", " '2d5f634bb7414afcac7b78ce7c0a864b',\n", " 'beda5449b3124e379f35601a33ab4651',\n", " '271af180d99846e4a0d8c57f444df81d',\n", " '4af16168d5a1432e8ca9719c9000f58c',\n", " '405113fc9e334cada56589b758cd9fd7',\n", " 'c95f295e46ba4b82b9f92fc0dcc8c1df',\n", " '475897fbe33347cf907f3cc381f40c0e',\n", " 'b6779e2220c444d38741c06cc2bb380c',\n", " '7d56c936c7d84514a67cd75e369449f7',\n", " '272a9892cdb742dcbe5f90e29eefae72',\n", " 'c28ace207c9d437da68cf599ac028bbb',\n", " '6d3c684dd6894bd9bf24486175ed834d',\n", " 'd8766ca5bb7d468399e6b864756a04e6',\n", " 'f5501aae471447fba9a4ac7ccf88c1fe',\n", " 'bb4d8f7876a141e0ba82eaebe7899c5a',\n", " '1c87be78b3fd48a093c23a54904bf8bc',\n", " '7cfe5d24a86645e1928a4700e2175e82',\n", " '79a9e904f5bb48a3830647c6afccbb85',\n", " 'deda349c16f54a9f85cd302269c22456',\n", " '5339992d8dcd464294260f5c0c857fff',\n", " 'd86ca1d2c07f4784956acb34d4d8c48c',\n", " 'dadf0561bb1c4ad9a87cc33a21424d32',\n", " '63c86b1adaed4514a75e0409a66b15c3',\n", " '61425b5443b840f2b7d28347d4002192',\n", " 'e7166ad200694bb7ae645e63495dbfa5',\n", " '9a4e61507dbd4fcd96b9c4b8eb24e74d',\n", " '5cb628ca8b8245e0ae326ccb8ae5635d',\n", " 'cbf3322896f8445ead83a6907a9aae08',\n", " '9156196800e64996891c0703499ffbb5',\n", " '47c88e62be7e4cb88b7d4935ba38cff0',\n", " '0fee655d64c34f84a07e6b889866a486',\n", " 'e4472727736f4fa59d49536d8e331f95',\n", " 'f933c36480f64d8b9600c5075a085e61',\n", " '808c6b3ce87345b391843aaa6b253bfb',\n", " 'f5d6ee781bd742fc88d5ddc2e5f0a7f4',\n", " '281ac17550864cf5827193ddd577aad1',\n", " '3d3aa0ebb1574fa7b498a13abb1b7c40',\n", " 'f43bf31cfe994208b24e363f9459a7a0',\n", " '9931894cfd004a20991a7fef40c23c86',\n", " '1b34b9f61f164993b7387a73e961bf2b',\n", " 'e2ca8df6b02d4240b7f1e4474b4765c7',\n", " '42741aaeaec7422f8ab9c59d18430455',\n", " '6a0a67f326704e11baae384eb567fa09',\n", " 'f7894024f0764978a9eca821c29d3449',\n", " '7f88bf5493764642a14a5bd8bbf04a71',\n", " '93a5412c61204d53b94fda693fb561b8',\n", " '3d265760e45a45d990240628c46fde6f',\n", " '62112a36dcab48379590ca210ca09918',\n", " '1abadc8684e64e6c8cbe1f7427d39678',\n", " 'f1f8c1a6c7534c5bb386425728cfa2b3',\n", " '81b640e8ae0747daadeda29da9f677f6',\n", " '6428b2a89f384a6985d69b0183fc71c3',\n", " '107fb02d9c7e4bffa9669509015e8af1',\n", " '73b8fd9b8aea45a6adfc02d5795bec62',\n", " '931cdf852b634abfb01b656221a8d0ae',\n", " '81e9f8177fe9430a9fb17fd20522c955',\n", " '7dbc18b381454afeb2a6041f60c2b23b',\n", " 'f621fcf8f34f4629909ca455ecaa4f55',\n", " 'c358be2860cf436d8fabb3200888c307',\n", " '81516460f65740e9aef0f4babc29b2f1',\n", " 'ecd2409c27cf47aeadeff569bd25ea85',\n", " 'a4c997a1ad7f4990b2d71cb028463610',\n", " '1ed8f8421791456db0543cd3e1ede40f',\n", " '56f5077192e74e09a58017d0c3368bc4',\n", " 'a639e85d3bee4530a53d132bfa7c58de',\n", " '4c75146c59dd4541a8500f89dd060a2c',\n", " '95c74438067f4bada1fee37942e06ed7',\n", " '0b3f010515574c48b02bebf7a451052e',\n", " '4db576ae022d42beadd921a81e977096',\n", " '3998f2de8bf44929afa7ad0e2e86eccd',\n", " '73cf1599b76d4061874e660228ca5f06',\n", " '962fb1291b984d60adb133201b7eae48',\n", " '365a19df65514c698d826e86fcdc6091',\n", " '5e86e99df25a4cc287d8ea0605f8cb08',\n", " 'f2cccab55efc43d5b098c38c31f687fb',\n", " '3388b8f7db314bf5a60cd10dbbc45f9c',\n", " '2a83e0ed7b4e4d2f906cfbc8dca7c512',\n", " '5b4f99c2acab40248de70a0e92506bc0',\n", " '278560e5a9e244e1a0a2ffa0ef7c261a',\n", " '864b65e24dea4473ad0e4a5bc32f4c69',\n", " '6b089bf4dd004ed78f1b92c50d414e47',\n", " '5bebbff6685649b99fa304d40b9b6362',\n", " 'aecb11bcf1444ad589508ea8bec77bdb']" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from langchain_qdrant import QdrantVectorStore\n", "from langchain_core.documents import Document\n", "from qdrant_client import QdrantClient\n", "from qdrant_client.http.models import Distance, VectorParams\n", "\n", "dimension = 1024\n", "collection_name = \"ai-safety-sr-arctic-embed-l-semantic\"\n", "qdrant_server = os.environ[\"QDRANT_API_URL\"]\n", "qdrant_client = QdrantClient(url=qdrant_server,api_key=os.environ[\"QDRANT_API_KEY\"])\n", "qdrant_client.create_collection(\n", " collection_name=collection_name,\n", " vectors_config=VectorParams(size=dimension, distance=Distance.COSINE),\n", ")\n", "\n", "vector_store = QdrantVectorStore(\n", " client=qdrant_client,\n", " collection_name=collection_name,\n", " embedding=embedding_model,\n", ")\n", "\n", "vector_store.add_documents(chunked_docs)\n", "\n" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "retriever = vector_store.as_retriever(search_type=\"similarity_score_threshold\",\n", " search_kwargs={'k':10,'score_threshold': 0.8})" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 44, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': 'b6779e22-20c4-44d3-8741-c06cc2bb380c', '_collection_name': 'ai-safety-sr-arctic-embed-l-semantic'}, page_content='Human-AI Configuration \\n'),\n", " Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 33, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': '26f3917c-c227-4e99-8e11-5212273fe2ba', '_collection_name': 'ai-safety-sr-arctic-embed-l-semantic'}, page_content='Human-AI Configuration \\n'),\n", " Document(metadata={'source': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'file_path': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 11, 'total_pages': 73, 'format': 'PDF 1.6', 'title': 'Blueprint for an AI Bill of Rights', 'author': '', 'subject': '', 'keywords': '', 'creator': 'Adobe Illustrator 26.3 (Macintosh)', 'producer': 'iLovePDF', 'creationDate': \"D:20220920133035-04'00'\", 'modDate': \"D:20221003104118-04'00'\", 'trapped': '', '_id': '07a8b461-51e7-4641-b97a-d823ef91082f', '_collection_name': 'ai-safety-sr-arctic-embed-l-semantic'}, page_content=' \\n \\n \\nFROM \\nPRINCIPLES \\nTO PRACTICE \\nA TECHINCAL COMPANION TO\\nTHE Blueprint for an \\nAI BILL OF RIGHTS\\n12\\n'),\n", " Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 37, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': '1d43c4ca-f83b-4708-97e9-6410f4dae5ee', '_collection_name': 'ai-safety-sr-arctic-embed-l-semantic'}, page_content='Human-AI Configuration \\nAI Actor Tasks: AI Deployment, AI Impact Assessment, Domain Experts, Operation and Monitoring, TEVV \\n \\n'),\n", " Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 61, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': '73cf1599-b76d-4061-874e-660228ca5f06', '_collection_name': 'ai-safety-sr-arctic-embed-l-semantic'}, page_content='et al. (2023) Whose Opinions Do Language Models Reflect? arXiv.'),\n", " Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 28, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': '00ca0c39-98b6-4339-874a-d036983a0922', '_collection_name': 'ai-safety-sr-arctic-embed-l-semantic'}, page_content='Make sure these tests cover various scenarios, such as crisis \\nsituations or ethically sensitive contexts. Human-AI Configuration; \\nInformation Integrity; Harmful Bias \\nand Homogenization; Dangerous, \\nViolent, or Hateful Content \\nAI Actor Tasks: AI Design, AI Development, Domain Experts, End-Users, Human Factors, Operation and Monitoring \\n \\n'),\n", " Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 59, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': '81516460-f657-40e9-aef0-f4babc29b2f1', '_collection_name': 'ai-safety-sr-arctic-embed-l-semantic'}, page_content='https://www.rand.org/pubs/research_reports/RRA2977-2.html. Nicoletti, L. et al. (2023) Humans Are Biased. Generative Ai Is Even Worse. Bloomberg. https://www.bloomberg.com/graphics/2023-generative-ai-bias/. National Institute of Standards and Technology (2024) Adversarial Machine Learning: A Taxonomy and \\nTerminology of Attacks and Mitigations https://csrc.nist.gov/pubs/ai/100/2/e2023/final \\nNational Institute of Standards and Technology (2023) AI Risk Management Framework. https://www.nist.gov/itl/ai-risk-management-framework \\nNational Institute of Standards and Technology (2023) AI Risk Management Framework, Chapter 3: AI \\nRisks and Trustworthiness. https://airc.nist.gov/AI_RMF_Knowledge_Base/AI_RMF/Foundational_Information/3-sec-characteristics \\nNational Institute of Standards and Technology (2023) AI Risk Management Framework, Chapter 6: AI \\nRMF Profiles. https://airc.nist.gov/AI_RMF_Knowledge_Base/AI_RMF/Core_And_Profiles/6-sec-profile \\nNational Institute of Standards and Technology (2023) AI Risk Management Framework, Appendix A: \\nDescriptions of AI Actor Tasks. https://airc.nist.gov/AI_RMF_Knowledge_Base/AI_RMF/Appendices/Appendix_A#:~:text=AI%20actors%\\n20in%20this%20category,data%20providers%2C%20system%20funders%2C%20product \\n'),\n", " Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 57, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': '1b34b9f6-1f16-4993-b738-7a73e961bf2b', '_collection_name': 'ai-safety-sr-arctic-embed-l-semantic'}, page_content='(2020) Overcoming Failures of Imagination in AI Infused System Development and \\nDeployment. arXiv.'),\n", " Document(metadata={'source': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'file_path': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 0, 'total_pages': 73, 'format': 'PDF 1.6', 'title': 'Blueprint for an AI Bill of Rights', 'author': '', 'subject': '', 'keywords': '', 'creator': 'Adobe Illustrator 26.3 (Macintosh)', 'producer': 'iLovePDF', 'creationDate': \"D:20220920133035-04'00'\", 'modDate': \"D:20221003104118-04'00'\", 'trapped': '', '_id': '8dd5b1e7-fd46-4e2a-90c2-8a8eea8b0cb9', '_collection_name': 'ai-safety-sr-arctic-embed-l-semantic'}, page_content=' \\n \\n \\n \\n \\n \\n \\n \\n \\n \\nBLUEPRINT FOR AN \\nAI BILL OF \\nRIGHTS \\nMAKING AUTOMATED \\nSYSTEMS WORK FOR \\nTHE AMERICAN PEOPLE \\nOCTOBER 2022 \\n'),\n", " Document(metadata={'source': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'file_path': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 23, 'total_pages': 73, 'format': 'PDF 1.6', 'title': 'Blueprint for an AI Bill of Rights', 'author': '', 'subject': '', 'keywords': '', 'creator': 'Adobe Illustrator 26.3 (Macintosh)', 'producer': 'iLovePDF', 'creationDate': \"D:20220920133035-04'00'\", 'modDate': \"D:20221003104118-04'00'\", 'trapped': '', '_id': '8c64aecd-850b-48b3-bac2-16e73ebad1e0', '_collection_name': 'ai-safety-sr-arctic-embed-l-semantic'}, page_content='Some companies have instituted bias testing as part of their product \\nquality assessment and launch procedures, and in some cases this testing has led products to be changed or not \\nlaunched, preventing harm to the public. Federal government agencies have been developing standards and guidance \\nfor the use of automated systems in order to help prevent bias.')]" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "retriever.invoke(\"What steps can organizations take to minimize bias in AI models?\")" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['dd370438231c41dbb7b1b4f1e7673cf7',\n", " '02ebba25e01941849b9e2c9d5097b55d',\n", " '099f0083356a4914b53fcb30df633b50',\n", " 'f8aefa25a4544c869ca4caaf686b3d47',\n", " '9ec0798fb4554f95ab65bd05315af118',\n", " '33bdad4db0ab4145b85726f77f1789ad',\n", " '98a75a601b114b07953b5aef4e032b4a',\n", " '1e49952c0d6743ba8ad52a049c18daa3',\n", " 'c3babb9205e54ca99ba6e5a03679bdba',\n", " '74cecdae132c4a5e953bd7e72ac6850e',\n", " '29529ea9530541a0bb446a8e82fab913',\n", " '4193dcf34f6249b1a29c49a52239deef',\n", " '84cb5d0f2cee47beabd72baa54161155',\n", " '622f279ac5bd40b082725d90972e9ae3',\n", " '48e366f92aa449e89cf7158584d2cf6a',\n", " 'e2ffb7cb2ac3482fb9290940fabe9582',\n", " 'f52a4c3353544fff93f241cba063028a',\n", " '0c81aa08ddd4496a9aaea4b001f3596c',\n", " '3e9d8d7785b04d5fad063219c94ef0dd',\n", " '76796785c7b64d428e48b7cf699e155a',\n", " '593ab20fc2494634959b0bfd8821ea91',\n", " '654421ae91df4739bfb1ebdfb7c9dda2',\n", " '27ffe059aafd4d5fa795b2f893b1d57e',\n", " 'f1468d8276444858acb33bd6e2d36e73',\n", " '5a6a15255cdd438abd9b2c3358dca939',\n", " 'fbb13ef430ca47d28013dda9feaf4625',\n", " 'fc16826ddd504038bb5f32fd97cdd98e',\n", " '72c878c56d8746dea51fdcf506e48894',\n", " '257ac1e04a4b478ab3b84c81e5dfc3f4',\n", " '68b157c05ced46828ac39894e69b8d08',\n", " '535e59df03184e86b30a09cd2d169dcd',\n", " '1a6d76252d364a758564a41b922d44a4',\n", " '61e497e66868447988198ba831096707',\n", " 'dd9f18bedfed443c8bed0fc4c34c5e23',\n", " '0ca0575097c24b50a613d5a19de61cfc',\n", " '7dfba6cbbfe34756ba3f40b1be282324',\n", " 'e9b68e9579194b04ad65bbf85332d351',\n", " '7545cee6d2e345ba90e95082a15271b8',\n", " 'df1e9db6843a4ddbb788b1e9117db9a1',\n", " 'bb0687d2f3d047138d0414d0b2a22917',\n", " 'b79ed7024a064c1f9360692c93615657',\n", " '70fb8aa096a74d0a975705ac44f08577',\n", " '41bf93d83ebe414e91253e7a96f50ec7',\n", " 'bdab13de5b514bf68921751b3051ce60',\n", " 'ccd47e89a09c4519981dc5d9be7b1ad9',\n", " '334ac2db387848f1829e174c6584288b',\n", " '5484df8c41cb41babd01c3f8d62121a2',\n", " 'dd1f97aef70e439ea02c8f0d0ea397e0',\n", " '99ccff600f8b4470af445f1f060e5518',\n", " 'd4de01e6623741d3b06c8ee973ad6670',\n", " '6217b664cbba4a64bf6e4f2ffde27831',\n", " '3974c50b7e3a4503925f1c397254d259',\n", " '4959c05e7d8049a4b75cff3bdc6fc30d',\n", " '9d3aeacd6513463fbe9d13c1fb2441fc',\n", " '8777904b546e4ef5b2759f0a60fd1fca',\n", " '7a73c81712804111b6145b57888455ae',\n", " '87036e89882546b69613378b17610332',\n", " '5c508cd4449449c486b811d65b9b6db1',\n", " 'fd5a25bb9038481aabaed2b34a7f2cc9',\n", " '80b47526b0224fc0ba54cf4a61da11cc',\n", " 'a2c5d0697278407fb0d89c9c138bfba0',\n", " '49cb7eb52ea043f2baf21b611709d83a',\n", " '96d002f0aa0a4cd7a86b188ef7811e9e',\n", " 'd5a4fd354f904b99a8700363f7bcec7d',\n", " 'ec384dfa0a5d4caeab593a4d013e40de',\n", " 'b613bc8f681141249d11f2eea7691f32',\n", " '7b9e491ad88b48f19ce2698c4d8ef5ec',\n", " '011946a9fed74b14b7d4be2ff4eaadf5',\n", " '2c07a4769e85425a9a32a053f1293ff2',\n", " 'c859a58fd5a54447a22217564e610e77',\n", " '2d6fcf19e009459e82b344a699c6556f',\n", " '22ec0544ffc44be1bd557cd91e96caf4',\n", " '1a9fed777bc8454faed8c60a12dce190',\n", " '898d974afb3f472a96c9cca3c698fee1',\n", " '21cb161a28f34fc89b90b369d1895fd6',\n", " '9409c811bf6542feae01351580bcb32b',\n", " '1219e011429b4e29a84268bdbb66d7a3',\n", " '3293d7aeecf54778a4b1e63f09f3f362',\n", " '543f56abfcda469a826e797aa2a4ae36',\n", " 'b714fd2157d145658860f0db0bd95163',\n", " '8be2abe5fee54507a03f0a5d0ba2f0c7',\n", " '76bf633b2d6d49b98845023de4024f09',\n", " 'eb00c39f80904f79847f0156e1e88ff0',\n", " 'c9695522dbb243cbaaf48b9a5b9f4105',\n", " 'aa3a41bf64fb4136b1fc097ad40378eb',\n", " '4ba6b441be194008aac5fec9aa0eac53',\n", " '7b5e6c78bcc64d4c879cc0817436ab35',\n", " '90229474433449648237b410db3cdfb7',\n", " '1ace5952f4c043e0a0864b9926475add',\n", " '2208ad4c34bc4fdb940e1cec9df0f6bc',\n", " '15e5cc7345b64cc08ad9825085af5486',\n", " 'ccac823288be4ed2a3d5617dda575120',\n", " 'e01e349952e54ef4849519acdaa6725b',\n", " '794cb0ec54724738a48d16e18f6cb3b9',\n", " 'd9ebc60044124f0a890f7836cb58f4a2',\n", " '9fda4771fa994fcaa609088cfc961dd4',\n", " '67fd7a926392436a9b344903bbbc08f7',\n", " '7892533c0b57466e8249f11d5cba07d3',\n", " 'ba1b7ad3addd4557aa6983cc309fdd49',\n", " '3663e44b29e14192abe4f49c98e3db45',\n", " '476728c883404adea36c903128c98139',\n", " 'dbec4491b8b94aa0b9327ead46b4251f',\n", " 'b635ebf0a3644e71ab64bfd43faec517',\n", " '149ee2b23af9448c896c5bf87f1c9257',\n", " 'f6082f115738427db840c3ff58a7c48d',\n", " 'd56f7d290c184a5aabdd80251ee807b5',\n", " '37738bf3df9048c381a26116632aff03',\n", " 'a139a0ccd8834a3293c65b8a9fb0a2ad',\n", " '6cce500b28a944fbb6ad637ae5d3c227',\n", " 'e16444a94afb409b827c3ee3f57237b9',\n", " 'd2d0cd1eac154f17b31a6a97d87bca88',\n", " '75bc3a67451e44a487939e6ca74e39ad',\n", " '9074393c760844908a39e806f7d4714e',\n", " 'fe16063331f848d897ac4466e9237fd7',\n", " 'a9504de4c7cd427cae397ddd551e3bba',\n", " '791979c2b3d747bf857370c4ab7e7757',\n", " 'cc31a5edfbfd486482b202b8b87c8e9f',\n", " '6cf68d7de2334461809f1867dfef1280',\n", " 'e8acd67d2b614ebd8e0786dcd961c05c',\n", " 'e30854b096ab48b3b7f3729bd07914e1',\n", " 'ae46462add43450d899655e2e0819e59',\n", " '1718ec5c859c4b37a8c8ac0bbcffa616',\n", " 'ae4a9b112f364a34a47f0cf255b882a4',\n", " '5ab58a4ae877437898e06968149195b2',\n", " '76401834e26049b096cdbb054cb37c7a',\n", " '52b9f9683ba14fb2a6f84f5b6d619b40',\n", " '96d56773bf2e4d858f62654f20ccd53d',\n", " 'd8a258c7815040808dbab44252e15e77',\n", " '9f4ca5df27674f4cba5628dd98f1ef2b',\n", " '8e0679ac4cbc44839a7f77a12ce52220',\n", " '964aa8b0653c4bdaa236d5bb6f1eff1d',\n", " '219697ddd2d545c484443ca116943b63',\n", " '607bf4aa92c64da9a4873477e2e9b363',\n", " 'efd6114026d2454d983e7b4063656266',\n", " '81fdf32f6fb949508196043e4142261b',\n", " '14f82f6a28a9488a93ce3724bdc2a476',\n", " '3d44caaa301046af9acdd238d1e3cbda',\n", " 'd8d11ff667ed4c0f94958e794e9b2c60',\n", " '90ba7f9ad22d46d99316b36cf213ec01',\n", " '76e251b11fb04240ae381af227c136bd',\n", " '3dec921799b24369a5b9189dd28c0f55',\n", " 'e7d706e31326405aa1a8acaa627ee2f6',\n", " '9b8c00a9f69649a1a66decee2aa77c9c',\n", " 'e2902a1405cf41db9a0428259089cbef',\n", " '5d43e9e802fb49098531b257c5633723',\n", " '413aeaa9a2bc4970b21371940026dba8',\n", " 'bfb4471ff2af4bada2ac8bcd14429d24',\n", " '533ff7a97fca4578999e6e0434df17d5',\n", " '04ba353da230424bbf25ab29a18e20f4',\n", " '1ed6dc9440f14ac9b1deadc10fbb660b',\n", " '734aa1cc14a34825be7dbc947ecdb525',\n", " '482153d0c05f4453a3bc9f57a1804406',\n", " '76d988efad614d3dafc0ecb8fbdb2189',\n", " '6d9ada3651704a0ca2b40664a59c8579',\n", " 'ac9272192bfc416d8386487c9b381ccf',\n", " '0d8c4c52de304f7f9778455f8ad178ec',\n", " '9daf6aed58624240b7fb1fcf79d9dda2',\n", " '7ee8e6bfacca440f9ef345e59eba7401',\n", " 'e80b50ed3a3a42d2a24bfe85cd5d45db',\n", " 'c0a0384aad9345f3b6162bf63c5bc0d3',\n", " 'df7c71db74af415b96cb2cea32d0ba30',\n", " '329b6e0ed9cb43cca48970fcc286e299',\n", " 'fd4c170eb78d401fa3eec3253be26b98',\n", " 'a98ea19a24cb4b04bd943d226dc41bba',\n", " '8e917ca54af2400cbd83e08e28ef0bb0',\n", " '78a3cf3647cc44f9abb083eaa8b79947',\n", " '140a48e4acbd44da9fede75e12fa80f3',\n", " '92d6e0856f5848869f7feb8cd17d7088',\n", " 'bf482ac0daea44a789182b52fbd2f413',\n", " 'f562e2d3c5b9422b845b6f87806e4d6c',\n", " 'e8b0bdec066a4bb0b7a47ec7e10c10e1',\n", " '45a054f0a9824867a52db472b2b65ad4',\n", " '40ff751a666347ed9c2326341587ea51',\n", " '1e35fbf001de48599063d4fe6dae165b',\n", " 'fa3b6f2ebe274d54851e9a31975470a6',\n", " '42242351bcb844c589f44a80cc139fdf',\n", " 'e5eb97e74797481d998677225cbaf365',\n", " '85f4bc0ed08c49feb6fd69f5659eaa36',\n", " '87df9b4b3b8a44eaa8756bbf8c967d8e',\n", " '7c04f74911aa4d629f0a545155e60b8d',\n", " '133597b368564215a8b71b2535a07032',\n", " '97473d57f4bd40af96f63bc06a1c6117',\n", " '3b3f2e2c08774f42bf0c904230b06c4c',\n", " '9661e9d3901e429d9030b04d28d98a19',\n", " '488e7a22bcdd400fb1ea9e52a102cd8f',\n", " '95086a933034484b9eeca08343c0dc21',\n", " 'e5aa0c58bc1448169bdebc7e99fcbd42',\n", " 'e6ba2f1ca8284d11a309757bc42ead7f',\n", " '6510e8c73d10408fa038b242f95cae2d',\n", " '36c2ba7197ce4c238472599a256f60db',\n", " '12ce3404c5024cc5a3ca4d1c0773d759',\n", " 'd0b909588d804b1aa89b496efcb6d16f',\n", " 'e830a0b4f0fa483b9bb9816162bde54f',\n", " 'eea7403faa024fc8a477b4b2e12bfc99',\n", " '1fbbcd091c3948229841d1a1e53cedef',\n", " '0bc14c99eb8141fc9f5ad1a13e8c5f90',\n", " '544d8570f3e847aea814771f3af2397e',\n", " 'd4e66a38f60c47deb7307e0f65829409',\n", " 'b520c61605aa473d89c88e3d277f40c5',\n", " '3f01a965995e495a8b3067fd0fdcc978',\n", " '9a5da84235c14817a3f0bda30a2bbcbb',\n", " '94792a1b19654f45a2fd8cc362dacddf',\n", " 'b46df9b21e764b39a0180bf42f9a835b',\n", " '47a4942336cd45cea7afcc68d99f1cd5',\n", " '2081541601de49199f881ffcb1625d4d',\n", " '972dcffe6c2e4cf2815cd571e9f4021d',\n", " 'e4613bb436fe46aaa3c236a209038124',\n", " '355cc289660d494db52e039127ecde34',\n", " 'b2b84fffde454e2d967ec6330e637b37',\n", " 'ab32b49526ee485d998dca366dace258',\n", " '48e0545b903f49bca771a22861166708',\n", " '47f62df12d494ba48b1bfee4bc1820f0',\n", " 'c131bfc970324e068ff4e04df6191c8d',\n", " '02a961ad52a34d1ead8d5c1a9ee12031',\n", " 'dcc8094b71444056a9c85f1b69b7e6df',\n", " 'd95234dd8571413ab9dc2cf2bd4031ef',\n", " '2b80007625d04c3a8b3c10fd35181861',\n", " 'b2b150d718c64b38925de1ee0abf14ed',\n", " '49f5645c803b4da78f09c7f0d337867a',\n", " 'af890841867746499efe8600704630b4',\n", " '7218a0d2f3e34a729da8a10e41a591fa',\n", " 'bafcc1d1244f40da99adcfb72f87b170',\n", " '14f2b87359524bfbac74a4948fdd135d',\n", " 'b209da5ac4ad4110834f018a3301f5cb',\n", " 'bf06ddade01d466592ea9cadbada320d',\n", " '865d5986afde44f4ab593708125e90ad',\n", " '3be18dd8e0bb4940bc58b257bad9c5b9',\n", " 'c02da0af95774b39af650dc268c8eed0',\n", " '7fae32ee9e934e2b8f164212ad9190b0',\n", " '8d0c1b678ea742cca445577d36a58e26',\n", " '4c1667fd01804d08bca4485a427b7cf3',\n", " '5d3bf9345565447095a9ffd9319997d9',\n", " 'ca5889dd43c5498ca449a733c36631d9',\n", " '5ba45f56b2f0412c835d4328b88037d3',\n", " 'e1d3f4649f234a8395a63ae1de670449',\n", " '17e91fcb2ae14f56a0f60c6acaf4258d',\n", " 'ed1d8496e014462db3aae0a046d4aeed',\n", " '8dd4483ba29448b7a285cefcaeb135fa',\n", " '77db975e07284a7a814ef386664c97aa',\n", " '93c28d0a7eb646969d0511b786fa7a71',\n", " 'c596f3b0927c49c1a4193eb5f0479395',\n", " 'e1648b2975284446bbbaefba431cdd78',\n", " '76bcc756b84a4b169a128973ef7228fe',\n", " 'ba57f715ffaf477fa15a733ddf5339aa',\n", " '644902133d4645bfbd02d9629fb737da',\n", " '124a4a8ec036421486e8501be3af4692',\n", " '0b95233f27f54043aee48dd77096c62c',\n", " '1ef09ede43d546b4a6a73b48c4cb48f1',\n", " '9dbee7e4bb32427f8fd0b0229ca0d2a6',\n", " 'f915f126e0f24f2299e6bfd16a5d3c1d',\n", " '3686b153c85248f6a2fc1fff12eaafe3',\n", " '850e99ca1e58439c8ccf36e2b6a7ecde',\n", " '25c84a37812b47c8adfd41b30af8c0bb',\n", " 'f96eb4a5818e4ebf8ae654d35cdc08a4',\n", " '31aaf38fa0bf49f4964d317f000840fd',\n", " '8c914f8f496741dab0d661f8bf84e061',\n", " '215a8d37eb5249ae97b9471c8ec0f888',\n", " '9f5e41e99d314cba824998d58ca1a611',\n", " '6b01b294c1774e34919059a7388aabd9',\n", " 'c1c6025360a9458085d5342cf8e703e0',\n", " 'b3fdbd3082794bd4b0ab4d4f2c8149f8',\n", " '700188e4d52b44fa9fde7512f54d7b1b',\n", " '00dad32bd08f4b39b153cc96b8497f4b',\n", " '9bb6f94103404153b68855d9993e9493',\n", " '56e07e044da34280830555b42799444d',\n", " '1d57812e47de41bf99efbbfa34865acd',\n", " '39c2def37c7d4b15a4e766632ea9eb98',\n", " '93e308150c2b44688cd13847402815b0',\n", " '1f595875c8ee4ac9a261bfb0a429067c',\n", " 'e7043593429e48a6bdcd8095f3ee2993',\n", " '8827c0bc83eb4dbeb48befa28e6ded29',\n", " 'acb099517d0449239adf6c9dde626772',\n", " 'f94e571e12af4903bfe866f6e028124e',\n", " 'd027a4d37b3640ad894587dab59a7494',\n", " 'f35c824977cc4d46962b01ad10f5ceb4',\n", " '836279f1552c4417a642da79743aeb33',\n", " '002978f8f7ab4cada169ea0d054499a5',\n", " 'e7a1c0978a2a4cd0b6b317ceda9874fc',\n", " '0f977a8cf0514392882756b1f7c6fa26',\n", " '7682cce87cfd470d95274b61e4eef8b8',\n", " '3947f0e87d00475387486b47326ed258',\n", " '6ddc01005056438cb611cd958b7a2d1d',\n", " 'bc81516df0f440d8a1faa7363f011b75',\n", " 'd676819f49004a56b7dc89cc5d5343ec',\n", " 'b8fd60c1b629499dac6ea2cdeb837502',\n", " '4e33d391a5634f0d82dd84dda6957811',\n", " 'b007e62d2fe749a6ab713c005211a73f',\n", " '18de6dda198e4f36befe7c81f88a7f42',\n", " '029857b9fae04c498b62b46c40267afa',\n", " '3be29a9773f24c079f24d9db9c662801',\n", " 'a095542002ce46ca95e59094332f0228',\n", " '40d61bdffba64369af605108a12a2999',\n", " '7edeaefb56544debb539a1bafb766796',\n", " '2f5335dc55594a04b67b07d86d937139',\n", " 'f89a5a9d2d5047d7a74dd991ae0e8102',\n", " 'fd4b6403ba4249b19c692a9dbdcdba01',\n", " '3e02d74c82764b6997c4f965cfd6c233',\n", " 'dfbe54ff42b4457db6cf921dc4ca0753',\n", " '7abb841a671b40649ad478bc45c75b47',\n", " 'a687e1b6e9dc40099a7d7d4ecd021a46',\n", " 'ffdea99f5cec44e4abc9f4b8c6949fc1',\n", " 'cb103ab7aa1a46f09d858dcf6880c862',\n", " '4709ecfdce6c4392a245fad38093d1c4',\n", " 'dbceabef7601444ba3a76c6bed960802',\n", " 'a7f1acab40b145cbb2d8d84fc72733af',\n", " '4df54db71ffa4dc4b1f10b022e3e6ef8',\n", " 'a7fc14d6dbb14af6ae3ef0a3f68f1d07',\n", " 'd5048b397eb04c6fb97a083b66aa6ac2',\n", " 'b6420e8387b94f85aa36b5bfe589463f',\n", " '767857ebb1fe41269ed4d82d967956d9',\n", " '400246f61fa94320a267b7ab3f2e8cc7',\n", " '8b763f3947974bd192d2d884c05c6428',\n", " 'ea8f1f4852d64e09be5ac90b04404dad',\n", " 'f6866c404c2c40b3a4314563846a911f',\n", " '837631a58af84142b5042772b24da3bf',\n", " '9d1586da31e44e6f9fb64ce9ff157673',\n", " '098b85fd43c24f1aa3fd7c93a48fb98b',\n", " 'f324b2ea7aef49979b23bb34f78846e8',\n", " '6c35050aaecf43e7b7fcf40e9edfaa2f',\n", " '910792f650bb463493a1b85488133ab5',\n", " '2d428a2e50194db792e5a02146be9364',\n", " '90ba4b5f05124c26aaec626e14ff2138',\n", " '5c4b11be82e54ebab89237c6a4928284',\n", " '11c9ce7a16094f788485a13218479435',\n", " '27ef6c0ee62d4f4d9767d255f5b0bda6',\n", " '603495b3760e453bbc7264b287754bd6',\n", " '2afe6de14bc34f6381f30124a069d391',\n", " '8666745692804c2bbd16b997d75f9426',\n", " 'f9789e9010d44d2fa2eeadd121f9186b',\n", " '3fdd94668daa4bd6a96e125c2725d9b4',\n", " '02cf5c44de76414bb431468c721ea6ad',\n", " 'ea3dc743311b46c6b7c6117c57de0333',\n", " 'cac324fb19374fc892f4db768850823d',\n", " 'ab4126594df44beb906274bbb1c0f40b',\n", " '4cb686862a9a46ecaaa414860edba1cd',\n", " '0e6481c6b52e43e5856a779b814b509c',\n", " '73c0feaefcd44cffaa712880076005c7',\n", " '147e1975a5b545c39eedde3c9e112d3f',\n", " 'b52a309d67e44730a3e13f395aec79d4',\n", " '6068165a21b64de191a203024d30275b',\n", " '939af554456e4f9cb33268bd36d792c5',\n", " '3dffe1eb87754b6ab3c932d8d77cfa00',\n", " 'bf7419dc8ad84fc9b15e09b9125fe6b8',\n", " 'f76718fe634243029f02130498d5afcd',\n", " 'ef23c33828ff408abab8607b82eeb016',\n", " 'da5f19bce12048a2aa11b06b85072f9c',\n", " '32f57a9a758a4e41849fd85cedef76f0',\n", " 'fba4c4a802904152bbeb6edb051e2607',\n", " '728acf0b65ad4108a1e7a72b146e338e',\n", " '0ff6d22870074917a6c014c33f4b7cf9',\n", " '29d7bb91ce6b4a74893c614f725c5178',\n", " 'b690e48b7dff464cb73dbdf1e6149309',\n", " 'd92a821e7fdc4276b4fc8201dadfef62',\n", " '7dcaeef50440471f83e8febf47b1049c',\n", " 'd06797d35176423f97a118fb2921bf35',\n", " '8601101c7fa9428c9624bce8ea4cee15',\n", " 'f45441e8e3264add88478323c93e2d38',\n", " '985d73938f0744fd9ca3e05d8ed4d99e',\n", " 'aec96ecb915d47eea4ae0dae9dc95446',\n", " '8120edfc5cf247babcaaf6e7bf59ebe1',\n", " '821cb61115ef45f3b24ecf7e7ffe5b27',\n", " 'e4dc1ed27a224c9294f648336d261c53',\n", " 'b90241dc87844d628f45401349ab9887',\n", " 'a255e1c4b31241058595c134f5de807a',\n", " '648c5ad4530442e5b50f28f40c386bb6',\n", " 'ffa4b3e9e4ae4289b0c404da4abadcbc',\n", " 'fa5af409bfe441d1bcee2d0b5e377678',\n", " '033a5c2bd8374ca6b8cb9b4d965954ab',\n", " '357babd6dd4947749142359d0fc0cdd3',\n", " 'e4853735544842de9790530cb56a3eba',\n", " '3698a84c5a8644d99072de0b3a6aa9f5',\n", " 'd9c6bac4f1dd41dcb3318843c7f79489',\n", " '3dd799500efb450f8004d4240e037b20',\n", " '686426621d894b2781e6fb48d4b16c8e',\n", " '85b807e001ff46fe985770fa6af9a534',\n", " '097971e5d47043a2b4d569d56634bb2e',\n", " '46bb945a79c94a7da13c0c1506e1c457',\n", " '5ef5cbe182d94b5d899d2dbc9595b3a8',\n", " '466985e3a3ce424ab38284db938f8d40',\n", " '3a87f5d24ee448649a1fb37e1572f0e4',\n", " 'e17801c99d6944b899da6568724826a9',\n", " '24d0a98cbe0d4450a1e3994c6dab3a15',\n", " '3fa0add1d0b642f296690f408b0372c8',\n", " '1d657774ca004dc79b7fcb36ac85e26e',\n", " '7e69a8dc03104c72bdfc8cb6c2fcf9ee',\n", " '40c85cbead07447a8ba67f4c279ffd8b',\n", " '7b595edf61784a549e64edfc1e18a497',\n", " 'a4d50cd2c1534c02b0a34458d25920d0',\n", " '318e51c677b949d09d9a61fb7a069082',\n", " 'a45a8c5d6a6a47ccbafb57a1bb45c4b0',\n", " 'a0c089e55b15476e8a89292b31b310fc',\n", " '41fcfb134a8a44c9931edccd36627ca2',\n", " '7e7687c8087a4174850cd19935c845bd',\n", " 'a812d2adb03546538480ad44b33fd2bb',\n", " '5c5779f29b93468ca603bf37687d068d',\n", " '50783f0c8c944ff19aa86f2e5ac781ac',\n", " '4a21f30d19f24fd5a331537371b46dce',\n", " 'adda51e0076048ca98142173498f3af7',\n", " '07fa53932ec041d5bcd71d77b273d8d2',\n", " '18768f62094043548f4d280627a9d3a9',\n", " 'd5fcc7eeeb154b179028b03beaf8f3f8',\n", " 'a6859c0e9eb74acfbfde7fefbc76d9b6',\n", " 'c2e90d49e7f14233acebd2ac10622efc',\n", " 'e0997f7ee41742ed8a0179f9805bf12e',\n", " '9b8dc4963513406d90a71935b05a7601',\n", " '1cfc3fe1c73f43f69d776c880641baa9',\n", " '34fc9ec282fe475d8314ea0a3a44b881',\n", " '0943841a14ef474b8844520d91bfee9c',\n", " 'ef27e818cad1462aa3bf7d9aaa19700f',\n", " '6e2fdf8601904078986ebb1c71bc8168',\n", " 'c18bf0d5ee5e45c8806cc1ea7d486bba',\n", " 'a87b5e1ac2dd42d488929580625996de',\n", " '8d384236c7e448439e9230a000d6aaf7',\n", " '6dabbe7ff4814ab4bdcb84715ab20af4',\n", " '943a13d4ef0d4bc2a81b16078b580e78',\n", " 'baa6fd13ff6f445a8adc6482e59eb411',\n", " '57b39b4eee4d4a0aac587d53ef68ff8d',\n", " 'fd07301c3d554f02a0793d5f7bb63f35',\n", " 'e99969118642471f887206e1c6a507e7',\n", " '584dc05d60844536bdaac7bb3c1b7cd4',\n", " 'f1ba0bf7f2b54612a3ffa4a66dd0989c',\n", " 'e254c679f27c4143bcdda32f15e846e1',\n", " '2d51c45ed9ca4a3ab95590cff7047d37',\n", " '3bc9228d85364bd1b1f5bb7d13af5a8c',\n", " 'b2de27d51d964d8c9e190f42d7ad9768',\n", " 'e08419ed7e154a7da62745b3bd5ebd78',\n", " '70a099122b2d43dbbe23d375432beaa1',\n", " '01c551bc80134225ad0391bb295b365c',\n", " 'c2388131208c4d8c868af7d7e6405cca',\n", " 'bb0e3a6a10cc4ac29a168fcee5042c17',\n", " '4b20b4b550da419d88ee62758c495138',\n", " '5ea09fb9fb074218b814dde11c1aed3f',\n", " 'ee62a59cd0784f4cbabb349725d7fe78',\n", " 'a58457312c084595ab30bd5c59d0b3cb',\n", " 'af5f60ae6a3a48129b15c89bebe493cf',\n", " 'e5fddd76079f492f83ab1582f8d46893',\n", " 'fd4e785b8d9c4d7bb8af493c54ee6870',\n", " 'd91c68bc847c4c5993e6fa53b657b504',\n", " '1bfd2c9fa301401a91ff49843b1c842c',\n", " '9684447b6a9044339ff355877d86f7d3',\n", " '6fcdfdbf98c64c8b89a3730f72f8268f',\n", " '6ccdf85fa1ce4bd19775b7fcf5a12ee7',\n", " 'e67bb1231dce4f35b3914e3a40bb9c12',\n", " '7a55571c1f654084844ec308bba0ba42',\n", " '38d650edd70742898f976233a7dfb85a',\n", " '496b340000dc4eb09780ce18b3ba5392',\n", " 'a79b3b9c6ea74599885fdb8d28d12cfd',\n", " 'f4de9dfe9b9c47f7aee86802c145d2d4',\n", " 'c5b4d1fed5874094acaadc869998174f',\n", " 'd4ca3a34bfeb41bc8065e682213eaaab',\n", " '40786361d73e4a58a173a099821b3020',\n", " 'c267c513aa0049168f2ab2e2444029c9',\n", " 'de47fda5d45340b58a4febf243c18c90',\n", " 'b8f6f44e9ca64d28956c159f9aa284bb',\n", " 'cb6cd0c5ffb743ce8d07d0c02ed2cbe3',\n", " 'f42720e5bcd94c4ab0f3880cf75dbb50',\n", " '3bd78527eff54c5db6ead2f0471d1b55',\n", " 'c5257e382fbc4f69a16aa0bc047dfee2',\n", " '22df791dc8fa45c9867e2cd4de171bd9',\n", " 'b3a0c0feba764bd0abfb446204a8239f',\n", " 'ba39d1b49cf840289c2d2d04e88948cd',\n", " '1a956280d4db49aea6007c9c1d0f698a',\n", " 'e7062f6c5dba476facf895b6faee99cd',\n", " '95f2febaef5a433f89c11d3e9741347f',\n", " 'fded3452cdbf42fa90f7fadfacd5dd63',\n", " '0a89bd45fc9d4148828cddb02a0921e7']" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Vector Store with recursive chunked documents\n", "\n", "recursive_collection_name = \"ai-safety-sr-arctic-embed-l-recursive\"\n", "\n", "recursive_qdrant_client = QdrantClient(url=qdrant_server,api_key=os.environ[\"QDRANT_API_KEY\"])\n", "# recursive_qdrant_client.create_collection(\n", "# collection_name=recursive_collection_name,\n", "# vectors_config=VectorParams(size=dimension, distance=Distance.COSINE),\n", "# )\n", "\n", "recursive_vector_store = QdrantVectorStore(\n", " client=recursive_qdrant_client,\n", " collection_name=recursive_collection_name,\n", " embedding=embedding_model,\n", ")\n", "\n", "recursive_vector_store.add_documents(recursive_chunked_docs)" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [], "source": [ "recursive_retriever = recursive_vector_store.as_retriever(search_type=\"similarity_score_threshold\",\n", " search_kwargs={'k':10,'score_threshold': 0.8})" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[Document(metadata={'source': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'file_path': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 11, 'total_pages': 73, 'format': 'PDF 1.6', 'title': 'Blueprint for an AI Bill of Rights', 'author': '', 'subject': '', 'keywords': '', 'creator': 'Adobe Illustrator 26.3 (Macintosh)', 'producer': 'iLovePDF', 'creationDate': \"D:20220920133035-04'00'\", 'modDate': \"D:20221003104118-04'00'\", 'trapped': '', '_id': '70fb8aa0-96a7-4d0a-9757-05ac44f08577', '_collection_name': 'ai-safety-sr-arctic-embed-l-recursive'}, page_content='FROM \\nPRINCIPLES \\nTO PRACTICE \\nA TECHINCAL COMPANION TO\\nTHE Blueprint for an \\nAI BILL OF RIGHTS\\n12'),\n", " Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 50, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': 'e254c679-f27c-4143-bcdd-a32f15e846e1', '_collection_name': 'ai-safety-sr-arctic-embed-l-recursive'}, page_content='• Accessibility and reasonable \\naccommodations \\n• AI actor credentials and qualifications \\n• Alignment to organizational values \\n• Auditing and assessment \\n• Change-management controls \\n• Commercial use \\n• Data provenance'),\n", " Document(metadata={'source': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'file_path': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 19, 'total_pages': 73, 'format': 'PDF 1.6', 'title': 'Blueprint for an AI Bill of Rights', 'author': '', 'subject': '', 'keywords': '', 'creator': 'Adobe Illustrator 26.3 (Macintosh)', 'producer': 'iLovePDF', 'creationDate': \"D:20220920133035-04'00'\", 'modDate': \"D:20221003104118-04'00'\", 'trapped': '', '_id': '2c07a476-9e85-425a-9a32-a053f1293ff2', '_collection_name': 'ai-safety-sr-arctic-embed-l-recursive'}, page_content='organization’s business processes or other activities, system goals, any human-run procedures that form a \\npart of the system, and specific performance expectations; a description of any data used to train machine \\nlearning models or for other purposes, including how data sources were processed and interpreted, a \\nsummary of what data might be missing, incomplete, or erroneous, and data relevancy justifications; the \\nresults of public consultation such as concerns raised and any decisions made due to these concerns; risk \\nidentification and management assessments and any steps taken to mitigate potential harms; the results of \\nperformance testing including, but not limited to, accuracy, differential demographic impact, resulting \\nerror rates (overall and per demographic group), and comparisons to previously deployed systems; \\nongoing monitoring procedures and regular performance testing reports, including monitoring frequency,'),\n", " Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 51, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': 'e08419ed-7e15-4a7d-a627-45b3bd5ebd78', '_collection_name': 'ai-safety-sr-arctic-embed-l-recursive'}, page_content='lifecycle and informed by representative AI Actors (see Figure 3 of the AI RMF). Until new and rigorous'),\n", " Document(metadata={'source': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'file_path': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 25, 'total_pages': 73, 'format': 'PDF 1.6', 'title': 'Blueprint for an AI Bill of Rights', 'author': '', 'subject': '', 'keywords': '', 'creator': 'Adobe Illustrator 26.3 (Macintosh)', 'producer': 'iLovePDF', 'creationDate': \"D:20220920133035-04'00'\", 'modDate': \"D:20221003104118-04'00'\", 'trapped': '', '_id': 'd9ebc600-4412-4f0a-890f-7836cb58f4a2', '_collection_name': 'ai-safety-sr-arctic-embed-l-recursive'}, page_content='for any resulting algorithmic discrimination. \\n26\\nAlgorithmic \\nDiscrimination \\nProtections'),\n", " Document(metadata={'source': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'file_path': 'https://www.whitehouse.gov/wp-content/uploads/2022/10/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 0, 'total_pages': 73, 'format': 'PDF 1.6', 'title': 'Blueprint for an AI Bill of Rights', 'author': '', 'subject': '', 'keywords': '', 'creator': 'Adobe Illustrator 26.3 (Macintosh)', 'producer': 'iLovePDF', 'creationDate': \"D:20220920133035-04'00'\", 'modDate': \"D:20221003104118-04'00'\", 'trapped': '', '_id': 'dd370438-231c-41db-b7b1-b4f1e7673cf7', '_collection_name': 'ai-safety-sr-arctic-embed-l-recursive'}, page_content='BLUEPRINT FOR AN \\nAI BILL OF \\nRIGHTS \\nMAKING AUTOMATED \\nSYSTEMS WORK FOR \\nTHE AMERICAN PEOPLE \\nOCTOBER 2022'),\n", " Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 38, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': '7b595edf-6178-4a54-9e64-edfc1e18a497', '_collection_name': 'ai-safety-sr-arctic-embed-l-recursive'}, page_content='guide the design of provenance data-tracking techniques. \\nHuman-AI Configuration; \\nInformation Integrity \\nMS-2.10-003 Verify deduplication of GAI training data samples, particularly regarding synthetic \\ndata. \\nHarmful Bias and Homogenization \\nAI Actor Tasks: AI Deployment, AI Impact Assessment, Domain Experts, End-Users, Operation and Monitoring, TEVV'),\n", " Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 59, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': 'b8f6f44e-9ca6-4d28-956c-159f9aa284bb', '_collection_name': 'ai-safety-sr-arctic-embed-l-recursive'}, page_content='https://www.bloomberg.com/graphics/2023-generative-ai-bias/. \\nNational Institute of Standards and Technology (2024) Adversarial Machine Learning: A Taxonomy and \\nTerminology of Attacks and Mitigations https://csrc.nist.gov/pubs/ai/100/2/e2023/final \\nNational Institute of Standards and Technology (2023) AI Risk Management Framework. \\nhttps://www.nist.gov/itl/ai-risk-management-framework \\nNational Institute of Standards and Technology (2023) AI Risk Management Framework, Chapter 3: AI \\nRisks and Trustworthiness. \\nhttps://airc.nist.gov/AI_RMF_Knowledge_Base/AI_RMF/Foundational_Information/3-sec-characteristics \\nNational Institute of Standards and Technology (2023) AI Risk Management Framework, Chapter 6: AI \\nRMF Profiles. https://airc.nist.gov/AI_RMF_Knowledge_Base/AI_RMF/Core_And_Profiles/6-sec-profile \\nNational Institute of Standards and Technology (2023) AI Risk Management Framework, Appendix A: \\nDescriptions of AI Actor Tasks.'),\n", " Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 57, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': '38d650ed-d707-4289-8f97-6233a7dfb85a', '_collection_name': 'ai-safety-sr-arctic-embed-l-recursive'}, page_content='54 \\nAppendix B. References \\nAcemoglu, D. (2024) The Simple Macroeconomics of AI https://www.nber.org/papers/w32487 \\nAI Incident Database. https://incidentdatabase.ai/ \\nAtherton, D. (2024) Deepfakes and Child Safety: A Survey and Analysis of 2023 Incidents and Responses. \\nAI Incident Database. https://incidentdatabase.ai/blog/deepfakes-and-child-safety/ \\nBadyal, N. et al. (2023) Intentional Biases in LLM Responses. arXiv. https://arxiv.org/pdf/2311.07611 \\nBing Chat: Data Exfiltration Exploit Explained. Embrace The Red. \\nhttps://embracethered.com/blog/posts/2023/bing-chat-data-exfiltration-poc-and-fix/ \\nBommasani, R. et al. (2022) Picking on the Same Person: Does Algorithmic Monoculture lead to Outcome \\nHomogenization? arXiv. https://arxiv.org/pdf/2211.13972 \\nBoyarskaya, M. et al. (2020) Overcoming Failures of Imagination in AI Infused System Development and \\nDeployment. arXiv. https://arxiv.org/pdf/2011.13416 \\nBrowne, D. et al. (2023) Securing the AI Pipeline. Mandiant.'),\n", " Document(metadata={'source': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'file_path': 'https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf', 'page': 12, 'total_pages': 64, 'format': 'PDF 1.6', 'title': 'Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile', 'author': 'National Institute of Standards and Technology', 'subject': '', 'keywords': '', 'creator': 'Acrobat PDFMaker 24 for Word', 'producer': 'Adobe PDF Library 24.2.159', 'creationDate': \"D:20240805141702-04'00'\", 'modDate': \"D:20240805143048-04'00'\", 'trapped': '', '_id': 'd5048b39-7eb0-4c6f-b97a-083b66aa6ac2', '_collection_name': 'ai-safety-sr-arctic-embed-l-recursive'}, page_content='Priorities Related to Information Integrity Research and Development.')]" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "recursive_retriever.invoke(\"What steps can organizations take to minimize bias in AI models?\")" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [], "source": [ "# Trying Compression retriver\n", "from langchain.retrievers import ContextualCompressionRetriever\n", "from langchain.retrievers.document_compressors import LLMChainExtractor\n", "from langchain_openai import ChatOpenAI\n", "\n", "base_retriever = recursive_retriever\n", "\n", "#Create a contextual compressor\n", "compressor_llm = ChatOpenAI(temperature=0, model_name=\"gpt-4o\", max_tokens=4000)\n", "compressor = LLMChainExtractor.from_llm(compressor_llm)\n", "\n", "#Combine the retriever with the compressor\n", "compression_retriever = ContextualCompressionRetriever(\n", " base_compressor=compressor,\n", " base_retriever=base_retriever\n", ")\n", "\n" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"Enter Your OpenAI API Key: \")" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "from langchain.prompts import ChatPromptTemplate\n", "\n", "RAG_PROMPT = \"\"\"\\\n", "Given a provided context and question, you must answer the question based only on context.\n", "\n", "If you cannot answer the question based on the context - you must say \"I don't know\".\n", "\n", "Context: {context}\n", "Question: {question}\n", "\"\"\"\n", "\n", "rag_prompt = ChatPromptTemplate.from_template(RAG_PROMPT)" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "from langchain_openai import ChatOpenAI\n", "\n", "# Using the same model used in the app.\n", "chat_model_name = \"gpt-4o\"\n", "llm = ChatOpenAI(model=chat_model_name)" ] }, { "cell_type": "code", "execution_count": 38, "metadata": {}, "outputs": [], "source": [ "from operator import itemgetter\n", "from langchain_core.runnables import RunnablePassthrough, RunnableParallel\n", "from langchain.schema import StrOutputParser\n", "\n", "ai_safety_rag_chain = (\n", " {\"context\": itemgetter(\"question\") | compression_retriever, \"question\": itemgetter(\"question\")}\n", " | rag_prompt | llm | StrOutputParser()\n", ")" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'Companies can ensure AI does not violate data privacy laws by incorporating built-in protections and ensuring that data collection conforms to reasonable expectations. They should collect only the data strictly necessary for the specific context and seek user permission, respecting their decisions regarding the collection, use, access, transfer, and deletion of data. If obtaining user permission is not possible, alternative privacy by design safeguards should be used. Additionally, systems should avoid user experience and design decisions that obfuscate user choice or burden users with privacy-invasive defaults.'" ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ai_safety_rag_chain.invoke({\"question\" : \"How can companies ensure AI does not violate data privacy laws?\"})" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\"I don't know.\"" ] }, "execution_count": 40, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ai_safety_rag_chain.invoke({\"question\" : \"What are the implications of using GAI systems for organizations in terms of risk management and compliance?\"})" ] } ], "metadata": { "kernelspec": { "display_name": "venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.9" } }, "nbformat": 4, "nbformat_minor": 2 }