Upload gradio_app.ipynb

gradio_app.ipynb

@@ -0,0 +1,455 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 63,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting sentencepiece\n",
+      "  Using cached sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.7 kB)\n",
+      "Downloading sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m49.7 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
+      "\u001b[?25hInstalling collected packages: sentencepiece\n",
+      "Successfully installed sentencepiece-0.2.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install sentencepiece"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 57,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting langsmith\n",
+      "  Downloading langsmith-0.1.104-py3-none-any.whl.metadata (13 kB)\n",
+      "Requirement already satisfied: httpx<1,>=0.23.0 in /home/sukhera/miniconda3/lib/python3.12/site-packages (from langsmith) (0.27.0)\n",
+      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /home/sukhera/miniconda3/lib/python3.12/site-packages (from langsmith) (3.10.7)\n",
+      "Requirement already satisfied: pydantic<3,>=1 in /home/sukhera/miniconda3/lib/python3.12/site-packages (from langsmith) (2.8.2)\n",
+      "Requirement already satisfied: requests<3,>=2 in /home/sukhera/miniconda3/lib/python3.12/site-packages (from langsmith) (2.32.2)\n",
+      "Requirement already satisfied: anyio in /home/sukhera/miniconda3/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith) (4.4.0)\n",
+      "Requirement already satisfied: certifi in /home/sukhera/miniconda3/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith) (2024.7.4)\n",
+      "Requirement already satisfied: httpcore==1.* in /home/sukhera/miniconda3/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith) (1.0.5)\n",
+      "Requirement already satisfied: idna in /home/sukhera/miniconda3/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith) (3.7)\n",
+      "Requirement already satisfied: sniffio in /home/sukhera/miniconda3/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith) (1.3.1)\n",
+      "Requirement already satisfied: h11<0.15,>=0.13 in /home/sukhera/miniconda3/lib/python3.12/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith) (0.14.0)\n",
+      "Requirement already satisfied: annotated-types>=0.4.0 in /home/sukhera/miniconda3/lib/python3.12/site-packages (from pydantic<3,>=1->langsmith) (0.7.0)\n",
+      "Requirement already satisfied: pydantic-core==2.20.1 in /home/sukhera/miniconda3/lib/python3.12/site-packages (from pydantic<3,>=1->langsmith) (2.20.1)\n",
+      "Requirement already satisfied: typing-extensions>=4.6.1 in /home/sukhera/miniconda3/lib/python3.12/site-packages (from pydantic<3,>=1->langsmith) (4.12.2)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/sukhera/miniconda3/lib/python3.12/site-packages (from requests<3,>=2->langsmith) (2.0.4)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/sukhera/miniconda3/lib/python3.12/site-packages (from requests<3,>=2->langsmith) (2.2.2)\n",
+      "Downloading langsmith-0.1.104-py3-none-any.whl (149 kB)\n",
+      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.1/149.1 kB\u001b[0m \u001b[31m22.4 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25hInstalling collected packages: langsmith\n",
+      "Successfully installed langsmith-0.1.104\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install -U langsmith"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/sukhera/miniconda3/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import PyPDF2\n",
+    "from transformers import BertTokenizer, BertModel\n",
+    "from transformers import LongformerModel, LongformerTokenizer\n",
+    "from transformers import BigBirdModel, BigBirdTokenizer\n",
+    "import numpy as np\n",
+    "from groq import Groq\n",
+    "import gradio as gr\n",
+    "from pathlib import Path\n",
+    "import torch\n",
+    "import json\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langsmith import Client\n",
+    "\n",
+    "# Initialize the LangSmith Client\n",
+    "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
+    "os.environ[\"LANGCHAIN_API_KEY\"] = \"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\"\n",
+    "client = Client()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/sukhera/miniconda3/lib/python3.12/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
+      "  warnings.warn(\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
+      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Load BERT tokenizer and model\n",
+    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
+    "model = BertModel.from_pretrained('bert-base-uncased')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load the BigBird model and tokenizer\n",
+    "tokenizer = BigBirdTokenizer.from_pretrained('google/bigbird-roberta-base')\n",
+    "model = BigBirdModel.from_pretrained('google/bigbird-roberta-base')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#longformer\n",
+    "# Load the Longformer model and tokenizer\n",
+    "tokenizer = LongformerTokenizer.from_pretrained('allenai/longformer-base-4096')\n",
+    "model = LongformerModel.from_pretrained('allenai/longformer-base-4096')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#longFormer\n",
+    "\n",
+    "def get_longformer_embedding(text):\n",
+    "    # Tokenize the text\n",
+    "    inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=4096)\n",
+    "    \n",
+    "    # Get the embeddings from Longformer\n",
+    "    with torch.no_grad():\n",
+    "        outputs = model(**inputs)\n",
+    "    \n",
+    "    # Use the [CLS] token's embedding as the aggregate representation\n",
+    "    cls_embedding = outputs.last_hidden_state[:, 0, :].numpy()\n",
+    "    \n",
+    "    return cls_embedding"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# BIGBIRD\n",
+    "def get_bigbird_embedding(text):\n",
+    "    # Tokenize the text\n",
+    "    inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=4096)\n",
+    "    \n",
+    "    # Get the embeddings from BigBird\n",
+    "    with torch.no_grad():\n",
+    "        outputs = model(**inputs)\n",
+    "    \n",
+    "    # Use the [CLS] token's embedding as the aggregate representation\n",
+    "    cls_embedding = outputs.last_hidden_state[:, 0, :].numpy()\n",
+    "    \n",
+    "    return cls_embedding"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_bert_embedding(text):\n",
+    "    # Tokenize the text\n",
+    "    inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=512)\n",
+    "    \n",
+    "    # Get the embeddings from BERT\n",
+    "    with torch.no_grad():\n",
+    "        outputs = model(**inputs)\n",
+    "    \n",
+    "    # Use the [CLS] token's embedding as the aggregate representation\n",
+    "    cls_embedding = outputs.last_hidden_state[:, 0, :].numpy()\n",
+    "    \n",
+    "    return cls_embedding\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def process_folder(file):\n",
+    "    folder_path = os.path.dirname(file.name)  # Get the directory of the selected file\n",
+    "    files = os.listdir(folder_path)  # List all files in the directory\n",
+    "    file_paths = [os.path.join(folder_path, f) for f in files]  # Get full paths of all files\n",
+    "    return f\"Files in folder: {', '.join(files)}\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Function to extract text from a PDF\n",
+    "def extract_text_from_pdf(pdf_file):\n",
+    "    text = ''\n",
+    "    with open(pdf_file, 'rb') as file:\n",
+    "        reader = PyPDF2.PdfReader(file)\n",
+    "        for page in reader.pages:\n",
+    "            text += page.extract_text() or ''\n",
+    "    return text\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cluster_emb={}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "def calculate_cosine(embedding1, embedding2):\n",
+    "    # Calculate the dot product and magnitudes of the embeddings\n",
+    "    dot_product = np.dot(embedding1, embedding2)\n",
+    "    magnitude1 = np.linalg.norm(embedding1)\n",
+    "    magnitude2 = np.linalg.norm(embedding2)\n",
+    "    \n",
+    "    # Calculate cosine similarity\n",
+    "    similarity = dot_product / (magnitude1 * magnitude2)\n",
+    "    return similarity"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def foo(files, JD):\n",
+    "    # Extract text and compute embeddings for job description using different models\n",
+    "    text_jd = extract_text_from_pdf(JD)    \n",
+    "    JD_embedding_bert = get_bert_embedding(text_jd).flatten()  # Flatten to match the dimension\n",
+    "    JD_embedding_longformer = get_longformer_embedding(text_jd).flatten()\n",
+    "    JD_embedding_bigbird = get_bigbird_embedding(text_jd).flatten()\n",
+    "\n",
+    "    sim = []\n",
+    "    \n",
+    "    for d in files:\n",
+    "        text = extract_text_from_pdf(d)\n",
+    "        # Compute embeddings for the resume using different models\n",
+    "        resume_embedding_bert = get_bert_embedding(text).flatten()  # Fixed function call\n",
+    "        resume_embedding_longformer = get_longformer_embedding(text).flatten()\n",
+    "        resume_embedding_bigbird = get_bigbird_embedding(text).flatten()\n",
+    "        # Calculate cosine similarity for each model\n",
+    "        similarity_bert = calculate_cosine(resume_embedding_bert, JD_embedding_bert)\n",
+    "        similarity_longformer = calculate_cosine(resume_embedding_longformer, JD_embedding_longformer)\n",
+    "        similarity_bigbird = calculate_cosine(resume_embedding_bigbird, JD_embedding_bigbird)\n",
+    "        # Append the results to the array\n",
+    "        sim.append(f\"\\nFile: {d.name:}\\n\"\n",
+    "                   f\"Bert Similarity: {similarity_bert:.4f}\\n\"\n",
+    "                   f\"Longformer Similarity: {similarity_longformer:.4f}\\n\"\n",
+    "                   f\"BigBird Similarity: {similarity_bigbird:.4f}\\n\")\n",
+    "        \n",
+    "    \n",
+    "    \n",
+    "    return \"\\n\".join(sim)  # Join the list into a single string for Gradio output\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/sukhera/miniconda3/lib/python3.12/site-packages/transformers/tokenization_utils_base.py:2888: UserWarning: `max_length` is ignored when `padding`=`True` and there is no truncation strategy. To pad to max length, use `padding='max_length'`.\n",
+      "  warnings.warn(\n",
+      "/home/sukhera/miniconda3/lib/python3.12/site-packages/transformers/tokenization_utils_base.py:2888: UserWarning: `max_length` is ignored when `padding`=`True` and there is no truncation strategy. To pad to max length, use `padding='max_length'`.\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "with gr.Blocks() as func:\n",
+    "    inputs = [gr.File(file_count=\"multiple\", label=\"Upload Resume Files\"), gr.File(label=\"Upload Job Description\")]\n",
+    "    outputs = gr.Textbox(label=\"Similarity Scores\")\n",
+    "    show = gr.Button(value=\"Calculate Similarity\")\n",
+    "    show.click(foo, inputs, outputs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Rerunning server... use `close()` to stop if you need to change `launch()` parameters.\n",
+      "----\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7862/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": []
+     },
+     "execution_count": 29,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "func.launch()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}