app.py

import streamlit as st
import os
from streamlit_chat import message


from datasets import load_dataset

dataset = load_dataset("wikipedia", "20220301.en", split="train[240000:290000]")


wikidata = []

for record in dataset:
    wikidata.append(record["text"])

wikidata = list(set(wikidata))
# print("\n".join(wikidata[:5]))
# print(len(wikidata))

from sentence_transformers import SentenceTransformer
import torch

device = 'cuda' if torch.cuda.is_available() else 'cpu'

if device != 'cuda':
    print(f"you are using {device}. This is much slower than using "
    "a CUDA-enabled GPU. If on colab you can chnage this by "
    "clicking Runtime > change runtime type > GPU.")

model = SentenceTransformer("all-MiniLM-L6-v2", device=device)

# Creating a Index(Pinecone Vector Database)