Upload 2 files

app.py

@@ -0,0 +1,97 @@
+from transformers import AutoModelForSequenceClassification
+from transformers import AutoTokenizer, AutoConfig
+from clean_data import cleaned_complaints
+import numpy as np
+from scipy.special import softmax
+import gradio as gr
+from transformers import PegasusForConditionalGeneration
+
+# Preprocess text (username and link placeholders)
+def preprocess(text):
+    new_text = []
+    for t in text.split(" "):
+        t = '@user' if t.startswith('@') and len(t) > 1 else t
+        t = 'http' if t.startswith('http') else t
+        new_text.append(t)
+    return " ".join(new_text)
+
+# load model
+MODEL = f"ThirdEyeData/Complaints_Roberta"
+#model = PegasusForConditionalGeneration.from_pretrained(MODEL)
+model = AutoModelForSequenceClassification.from_pretrained(MODEL)
+#model.save_pretrained(MODEL)
+
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL)
+config = AutoConfig.from_pretrained(MODEL)
+
+# create classifier function
+def classify_compliant(text):
+      text = cleaned_complaints(text)
+      if len(text)<3:
+          return "Cannot Categorize the Complaint"
+      else:
+          text = preprocess(text)
+          encoded_input = tokenizer(text, return_tensors='pt')
+          output = model(**encoded_input)
+          scores = output[0][0].detach().numpy()
+          scores = softmax(scores)
+
+          # Print labels and scores
+          probs = {}
+          ranking = np.argsort(scores)
+          ranking = ranking[::-1]
+
+  
+          l = config.id2label[ranking[0]]
+        #s = scores[ranking[i]]
+    #probs[l] = np.round(float(s), 4)
+          return l
+
+
+#build the Gradio app
+#Instructuction = "Write an imaginary review about a product or service you might be interested in."
+title="Customer Complaints Categorization"
+description = """
+This application uses fine-tuned Roberta to perform Customer Complaints Categorization. Roberta is a popular pre-trained language model that can be used for a variety of natural language processing tasks. This text classification model helps the company to categorize incoming support requests submitted by users and determine the appropriate course of action. The application can provide an opportunity for the service provider to resolve the customer’s problems on time and therefore, reduce dissatisfaction levels.
+
+
+Write a complaint on an insurance product or service and see how the machine learning model is able to Categorization your Complaint. Below is the type in which the complaints are segmented:
+1. Debt Collection
+2. False Claim or Statement
+3. Legal Issue
+4. Improper contact or sharing of info
+5. Follow Up Issue
+"""
+article = """
+            - Click submit button to test Consumer Complaint Segmentation
+            - Click the clear button to refresh the text
+            - This application has a linked model https://huggingface.co/ThirdEyeData/Complaints_Roberta
+           """
+
+demo = gr.Interface(classify_compliant,
+            inputs=gr.Textbox(lines =10,label = "Type your Complaint of our Product here or for a quick demo click on the examples provided below and output will automatically be populated in the output box ", max_lines = 20),
+            outputs = gr.Textbox(lines =5,label = "Complaint Category"),
+            title = title,
+            description = description,
+            #Instruction = Instructuction,
+            article = article,
+            #allow_flagging = "never",
+            live = False,
+            cache_example = False,
+            examples=[["""The day before my Salliemae student loan payment was due I contacted a rep to discuss the impact on my account of making my payment at the end of the month rather than the middle for just that one month. 
+            The rep indicated it would be no problem, but that I still may get a call  each day from Salliemae until I made my payment. I understood, requested my account be notated accordingly, and hung up. For two weeks I endured numerous calls per day ; 
+            I lost count at six calls one day, which was the norm for the number of calls Salliemae made in an effort to collect a debt that had a due date that had been arranged and had not come up yet. """],
+            ["""The representative told me the total amount due was {$2100.00} and that I can settle for half of that amount. Unfortunately, I was unable to accept the settlement but began to question the amount because my last statement was {$1800.00} and
+            there was nothing written in the contract for additional interest charges should my account go into collection. 
+            I told the representative that I will pay the amount actually owed and I want to make a payment arrangement. She told me I can't just do what I want, 
+            If I want to pay the original amount due, it has to be paid in full. I told her that that is not fair debt collection practice and that I am only contractually obligated to the {$1800.00} and we can set up an arrangement from that. """] ,
+             ["""This debt is beyond the Maryland Statute of Limitations. It is illegal for a debt collector to collect on an expired debt. They have taken illegal action by seizing my Maryland State Refund when the debt had already expired and beyond the Statute of Limitation which is 3 years in the state of Maryland"""],
+              ["""The company  has been calling my employer in an attempt to collect a debt. When I spoke with them and informed them that this was not an appropriate number to call. I asked what company they were calling from and a phone number so he told me the company name,  but the man on the phone would not give me his name or a phone number. 
+             I had mailed a letter requesting verification a few weeks ago and hadn't received anything back. In the letter I specifically requested that all communication be done through mail."""],
+             [""" I do n't think I chose the correct issue above, however I think it is closest to my issue. I have a record on my credit report that I have disputed through both the company and the credit bureaus. The dispute is marked as being disputed by me on my report, but it was not removed despite the creditor not sending me verification of this debt.
+             I do not even know what this debt is for.I have tried contacting the collection agency by mail to obtain verification with no response and they will not remove the item from my report."""]]
+                
+             )
+if __name__ == "__main__":
+    demo.launch()

clean_data.py

@@ -0,0 +1,86 @@
+import nltk
+from nltk.corpus import stopwords
+from nltk.stem import WordNetLemmatizer
+import warnings
+import re
+nltk.download("stopwords")
+nltk.download("wordnet")
+nltk.download("words")
+lemmatizer = WordNetLemmatizer()
+
+stop_words = set(stopwords.words('english'))
+
+contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not",
+                           "didn't": "did not",  "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not",
+                           "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is",
+                           "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would",
+                           "i'd've": "i would have", "i'll": "i will",  "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would",
+                           "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam",
+                           "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have",
+                           "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock",
+                           "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have",
+                           "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is",
+                           "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as",
+                           "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would",
+                           "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have",
+                           "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have",
+                           "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are",
+                           "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are",
+                           "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is",
+                           "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have",
+                           "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have",
+                           "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all",
+                           "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have",
+                           "you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have",
+                           "you're": "you are", "you've": "you have"}
+
+def cleaned_complaints(text):
+    import nltk
+    from nltk.corpus import stopwords
+    from nltk.stem import WordNetLemmatizer
+    from nltk.corpus import words
+    import warnings
+    import re
+
+    lemmatizer = WordNetLemmatizer()
+
+    stop_words = set(stopwords.words('english'))
+
+    contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not",
+                           "didn't": "did not",  "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not",
+                           "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is",
+                           "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would",
+                           "i'd've": "i would have", "i'll": "i will",  "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would",
+                           "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam",
+                           "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have",
+                           "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock",
+                           "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have",
+                           "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is",
+                           "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as",
+                           "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would",
+                           "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have",
+                           "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have",
+                           "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are",
+                           "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are",
+                           "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is",
+                           "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have",
+                           "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have",
+                           "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all",
+                           "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have",
+                           "you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have",
+                           "you're": "you are", "you've": "you have"}
+    
+    newString=re.sub(r'@[A-Za-z0-9]+','',text)                     #removing user mentions
+    newString=re.sub("#","",newString)                             #removing hashtag symbol
+    newString= ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in newString.split(" ")]) #contraction mapping
+    newString= re.sub(r'http\S+', '', newString)                   #removing links
+    newString= re.sub(r"'s\b","",newString)                        #removing 's
+    letters_only = re.sub("[^a-zA-Z]", " ", newString)             #Fetching out only letters
+    lower_case = letters_only.lower()                              #converting all words to lowercase
+    tokens = [w for w in lower_case.split() if not w in stop_words]#stopwords removal
+    tokens = [x for x in tokens if x in words.words()]
+#     tokens= lower_case.split()
+    newString=''
+    for i in tokens:                                                 
+        newString=newString+lemmatizer.lemmatize(i)+' '            #converting words using lemmatisation                               
+    return newString.strip()