Init

Deployment/Bert_medium.py

@@ -0,0 +1,17 @@
+
+from transformers import AutoModel
+from torch import nn
+import pytorch_lightning as pl
+
+
+class MediumBert(pl.LightningModule):
+    def __init__(self):
+        super().__init__()
+        self.bert_model = AutoModel.from_pretrained('asafaya/bert-medium-arabic')
+        self.fc = nn.Linear(512,18)
+    
+    def forward(self,input_ids,attention_mask):
+        out = self.bert_model(input_ids = input_ids, attention_mask =attention_mask)#inputs["input_ids"],inputs["token_type_ids"],inputs["attention_mask"])
+        pooler = out[1]
+        out = self.fc(pooler)
+        return out

Deployment/Dialect_Bert.py

@@ -0,0 +1,21 @@
+import torch.nn as nn
+from transformers import BertModel
+import pytorch_lightning as pl
+
+BERT_MODEL_NAME = 'alger-ia/dziribert'
+class Dialect_Detection(pl.LightningModule):
+    def __init__(self, n_classes):
+        super().__init__()
+        self.bert = BertModel.from_pretrained(BERT_MODEL_NAME)
+        self.classifier = nn.Linear(self.bert.config.hidden_size, n_classes)
+        self.criterion = nn.CrossEntropyLoss()
+
+    def forward(self, input_ids, attention_mask, labels=None):
+        output = self.bert(input_ids, attention_mask)
+        output = self.classifier(output.pooler_output)                    
+        # if provided with labels return loss and output
+        if labels is not None:
+            loss = self.criterion(output, labels)
+            return loss, output 
+
+        return output

Deployment/Offensive_Bert.py

@@ -0,0 +1,52 @@
+
+import torch.nn as nn
+from transformers import BertModel
+
+class BertClassifier(nn.Module):
+    """Bert Model for Classification Tasks.
+    """
+    def __init__(self, freeze_bert=False):
+        """
+        @param    bert: a BertModel object
+        @param    classifier: a torch.nn.Module classifier
+        @param    freeze_bert (bool): Set `False` to fine-tune the BERT model
+        """
+        super(BertClassifier, self).__init__()
+        #  hidden size of BERT, hidden size of our classifier, number of labels
+        D_in, H, D_out = 768, 50, 2
+
+        # Instantiate BERT model
+        self.bert = BertModel.from_pretrained('aubmindlab/bert-base-arabertv02')
+
+        # Instantiate an one-layer feed-forward classifier
+        self.classifier = nn.Sequential(
+            nn.Linear(D_in, H),
+            nn.ReLU(),
+            nn.Dropout(0.1),
+            nn.Linear(H, D_out)
+        )
+
+        # Freeze the BERT model
+        if freeze_bert:
+            for param in self.bert.parameters():
+                param.requires_grad = False
+        
+    def forward(self, input_ids, attention_mask):
+        """
+        Feed input to BERT and the classifier to compute logits.
+        @param    input_ids (torch.Tensor): an input tensor with shape (batch_size,
+                        max_length)
+        @param    attention_mask (torch.Tensor): a tensor that hold attention mask
+                        information with shape (batch_size, max_length)
+        @return   logits (torch.Tensor): an output tensor with shape (batch_size,
+                        num_labels)
+        """
+
+        outputs = self.bert(input_ids=input_ids,
+                            attention_mask=attention_mask)  
+        
+        # Extract the last hidden state of the token `[CLS]` for classification task and feed them to classifier to compute logits 
+        last_hidden_state_cls = outputs[0][:, 0, :]
+        logits = self.classifier(last_hidden_state_cls)
+
+        return logits

Deployment/app.py

@@ -0,0 +1,293 @@
+# Delete all objects from memory
+
+keys = list(globals().keys())
+
+for o in keys:
+    if not o.startswith('_'):
+        print(o)
+        del globals()[o]
+
+# Imort from a file called Bert-medium.py
+
+from Bert_medium import MediumBert
+from Offensive_Bert import BertClassifier
+from data_cleaning import cleaning_content
+from Dialect_Bert import Dialect_Detection
+
+import torch
+device = torch.device("cpu")
+
+
+from transformers import BertTokenizer, AutoTokenizer, BertTokenizerFast
+import streamlit as st
+
+# file path
+import os
+
+path_file = os.path.dirname(os.path.abspath(__file__))
+parent_path = os.path.dirname(path_file)
+
+##########################FUNCTIONS########################
+
+def predict_off(review_text,model,device,tokenizer):
+
+        encoded_review = tokenizer.encode_plus(
+        review_text,
+        max_length=256,
+        add_special_tokens=True,
+        return_token_type_ids=False,
+        padding='longest',
+        return_attention_mask=True,
+        return_tensors='pt',
+        )
+
+        input_ids = encoded_review['input_ids'].to(device)
+        attention_mask = encoded_review['attention_mask'].to(device)
+        output = model(input_ids, attention_mask)
+        _, prediction = torch.max(output, dim=1)
+        #print(f'Review text: {review_text}')
+        index = output.cpu().data.numpy().argmax()
+        #print(f'Sentiment  : {index}')
+        # decode the output of the model to get the predicted label
+        pred = index
+        
+        return pred
+#########################################""
+def predict_other(review_text,model,device,tokenizer):
+        
+        encoded_review = tokenizer.encode_plus(
+        review_text,
+        max_length=217,
+        add_special_tokens=True,
+        return_token_type_ids=False,
+        padding='longest',
+        return_attention_mask=True,
+        return_tensors='pt',
+        )
+
+        input_ids = encoded_review['input_ids'].to(device)
+        attention_mask = encoded_review['attention_mask'].to(device)
+        output = model(input_ids, attention_mask)
+        _, prediction = torch.max(output, dim=1)
+        #print(f'Review text: {review_text}')
+        index = output.cpu().data.numpy().argmax()
+        #print(f'Sentiment  : {index}')
+        # decode the output of the model to get the predicted label
+
+        return index
+#########################"##################
+
+def predict_dialect(review_text,model,device,tokenizer):
+        
+        encoded_review = tokenizer.encode_plus(
+        review_text,
+        max_length=123,
+        add_special_tokens=True,
+        return_token_type_ids=False,
+        padding='longest',
+        return_attention_mask=True,
+        return_tensors='pt',
+        )
+
+        input_ids = encoded_review['input_ids'].to(device)
+        attention_mask = encoded_review['attention_mask'].to(device)
+        output = model(input_ids, attention_mask)
+        _, prediction = torch.max(output, dim=1)
+        #print(f'Review text: {review_text}')
+        index = output.cpu().data.numpy().argmax()
+        #print(f'Sentiment  : {index}')
+        pred = index
+        return pred
+
+
+# Main prediction function
+
+def predict(text,device,offensive_model,offensive_tokenizer,racism_model,misogyny_model,verbalabuse_model,dialect_model,religionhate_model,tokenizer_dialect,other_tokenizer,off_dictionary,racism_dict,misogyny_dict,verbalabuse_dict,dialect_dict,religionhate_dict):
+        # clean text
+        text = cleaning_content(text)
+        
+        # predict using offensive model
+        off_pred = off_dictionary[predict_off(text,offensive_model,device,offensive_tokenizer)]
+
+        if off_pred == 'offensive':
+            # predict using racism model
+            rac_pred = racism_dict[predict_other(text,racism_model,device,other_tokenizer)]
+            # predict using misogyny model
+            misog_pred = misogyny_dict[predict_other(text,misogyny_model,device,other_tokenizer)]
+            # predict using verbal abuse model
+            ver_pred = verbalabuse_dict[predict_other(text,verbalabuse_model,device,other_tokenizer)]
+            # predict using dialect model
+            dialect_pred = dialect_dict[predict_dialect(text,dialect_model,device,tokenizer_dialect)]
+            # predict using religion hate model
+            Religion_Hate_pred = religionhate_dict[predict_other(text,religionhate_model,device,other_tokenizer)]
+            # return the prediction
+            return {"Offensiveness": off_pred, "Dialect": dialect_pred, "Misogyny": misog_pred, "Racism": rac_pred, "Verbal Abuse": ver_pred, "Religion Hate": Religion_Hate_pred}
+        
+        # predict using misogyny model
+        misog_pred = misogyny_dict[predict_other(text,misogyny_model,device,other_tokenizer)]
+        # predict using dialect model
+        dialect_pred = dialect_dict[predict_dialect(text,dialect_model,device,tokenizer_dialect)]
+        
+        # return the prediction  as a dataframe row
+        return {"Offensiveness": off_pred, "Dialect": dialect_pred, "Misogyny": misog_pred, "Racism": "Not_Racism", "Verbal Abuse": "Not Verbal Abuse", "Religion Hate": "Not Religion Hate"}
+###############################################
+
+from geopy.geocoders import Nominatim
+import numpy as np
+import pandas as pd
+import folium
+
+geolocator = Nominatim(user_agent="NLP")
+
+def geolocate(country):
+    try:
+        # Geolocate the center of the country
+        loc = geolocator.geocode(country)
+        # And return latitude and longitude
+        return (loc.latitude, loc.longitude)
+    except:
+        # Return missing value
+        return np.nan
+
+# Stream lit app
+
+st.title("Arabic Hate Speech Detection")
+
+st.write("This app detects hate speech in Arabic dialect text")
+
+st.write("Please enter your text below")
+
+
+# Session state
+if 'Loaded' not in st.session_state:
+    st.markdown('# Loading models')
+    st.session_state['Loaded'] = False
+else:
+    print('Model already loaded')
+    st.session_state['Loaded'] = True
+    
+
+if st.session_state['Loaded'] == False:
+
+    # Offensiveness detection model 
+
+    offensive_model = BertClassifier()
+    offensive_model.load_state_dict(torch.load(os.path.join(parent_path,'models\modelv3.pt')))
+    offensive_tokenizer = BertTokenizer.from_pretrained('aubmindlab/bert-base-arabertv02', do_lower_case=True)
+
+    #send model to device
+
+    offensive_model = offensive_model.to(device)
+    st.session_state['Offensive_model'] = offensive_model
+    st.session_state['Offensive_tokenizer'] = offensive_tokenizer
+    print('Offensive model loaded')
+    off_dictionary = {1: 'offensive', 0: 'non_offensive'}
+    st.session_state['Offensive_dictionary'] = off_dictionary
+
+    ##############################################################################################################################
+
+    # Other four models
+
+    other_tokenizer =  AutoTokenizer.from_pretrained("asafaya/bert-medium-arabic")
+    st.session_state['Other_tokenizer'] = other_tokenizer
+
+    racism_model,religionhate_model,verbalabuse_model,misogyny_model = MediumBert(),MediumBert(),MediumBert(),MediumBert()
+    ################################################################
+
+    racism_model.load_state_dict(torch.load(os.path.join(parent_path,'models\\racism\\racism_arabert.pt')))
+    racism_dict = {0: 'non_racist', 1: 'racist'}
+
+    racism_model = racism_model.to(device)
+
+    st.session_state['Racism_model'] = racism_model
+    st.session_state['Racism_dictionary'] = racism_dict
+
+    print('Racism model loaded')
+    ################################################################
+
+    religionhate_model.load_state_dict(torch.load(os.path.join(parent_path,'models\\religion_hate\\religion_hate_params.pt')))
+    religionhate_dict = {0: 'Religion Hate', 1: 'Not Religion Hate'}
+
+    religionhate_model = religionhate_model.to(device)
+
+    st.session_state['Religion_hate_model'] = religionhate_model
+    st.session_state['Religion_hate_dictionary'] = religionhate_dict
+
+    print('Religion Hate model loaded')
+    ################################################################
+
+    verbalabuse_model.load_state_dict(torch.load(os.path.join(parent_path,'models\\verbal_abuse\\verbal_abuse_arabert.pt')))
+    verbalabuse_dict = {0: 'Verbal Abuse', 1: 'Not Verbal Abuse'}
+
+    verbalabuse_model=verbalabuse_model.to(device)
+
+    st.session_state['Verbal_abuse_model'] = verbalabuse_model
+    st.session_state['Verbal_abuse_dictionary'] = verbalabuse_dict
+
+    print('Verbal Abuse model loaded')
+    ################################################################
+
+    misogyny_model.load_state_dict(torch.load(os.path.join(parent_path,'models\\misogyny\\misogyny.pt')))
+    misogyny_dict = {0: 'misogyny', 1: 'non_misogyny'}
+
+    misogyny_model=misogyny_model.to(device)
+
+    st.session_state['Misogyny_model'] = misogyny_model
+    st.session_state['Misogyny_dictionary'] = misogyny_dict
+
+
+    print('Misogyny model loaded')
+    ################################################################
+
+    # Dialect detection model
+
+    dialect_model = Dialect_Detection(10)
+    dialect_model.load_state_dict(torch.load(os.path.join(parent_path,'models\\dialect_classifier.pt')))
+
+    dialect_model = dialect_model.to(device)
+
+    st.session_state['Dialect_model'] = dialect_model
+
+    print('Dialect model loaded')
+
+    tokenizer_dialect = BertTokenizerFast.from_pretrained('alger-ia/dziribert')
+
+    st.session_state['Dialect_tokenizer'] = tokenizer_dialect
+
+    # load the model
+    dialect_dict = {0: 'lebanon', 1: 'egypt', 2: 'morocco', 3: 'tunisia', 4: 'algeria', 5: 'qatar', 6: 'iraq', 7: 'saudi arabia', 8: 'libya', 9: 'jordan'}
+
+    st.session_state['Dialect_dictionary'] = dialect_dict
+
+    st.session_state['Loaded'] = True
+
+text = st.text_area("Enter Text")
+
+if st.button("Predict") and text != '':
+    result = predict(text = text, device = device,
+                    offensive_model= st.session_state['Offensive_model'],
+                    offensive_tokenizer= st.session_state['Offensive_tokenizer'],
+                    racism_model= st.session_state['Racism_model'],
+                    misogyny_model=st.session_state['Misogyny_model'],
+                    verbalabuse_model= st.session_state['Verbal_abuse_model'],
+                    dialect_model=st.session_state['Dialect_model'],
+                    religionhate_model=st.session_state['Religion_hate_model'],
+                    tokenizer_dialect=st.session_state['Dialect_tokenizer'],
+                    other_tokenizer=st.session_state['Other_tokenizer'],
+                    off_dictionary=st.session_state['Offensive_dictionary'],
+                    racism_dict=st.session_state['Racism_dictionary'],
+                    misogyny_dict=st.session_state['Misogyny_dictionary'],
+                    verbalabuse_dict=st.session_state['Verbal_abuse_dictionary'],
+                    dialect_dict=st.session_state['Dialect_dictionary'],
+                    religionhate_dict=st.session_state['Religion_hate_dictionary'])
+
+    st.write(result)
+
+    location  = geolocate(result['Dialect'])
+
+    # map with contry highlited
+    location = pd.DataFrame({'lat': [location[0]], 'lon': [location[1]]})
+    st.map(data= location , zoom=5)
+    
+elif text == '':
+    st.write('Please enter text to predict')

Deployment/data_cleaning.py

@@ -0,0 +1,104 @@
+import re
+import string
+import nltk
+nltk.download('stopwords')
+
+
+arabic_stopwords = set(nltk.corpus.stopwords.words("arabic"))
+
+arabic_diacritics = re.compile("""
+                            ّ    | # Tashdid
+                            َ    | # Fatha
+                            ً    | # Tanwin Fath
+                            ُ    | # Damma
+                            ٌ    | # Tanwin Damm
+                            ِ    | # Kasra
+                            ٍ    | # Tanwin Kasr
+                            ْ    | # Sukun
+                            ـ     # Tatwil/Kashida
+                        """, re.VERBOSE)
+
+arabic_punctuations = '''`÷×؛<>_()*&^%][ـ،/:"؟.,'{}~¦+|!”…“–ـ'''
+english_punctuations = string.punctuation
+punctuations = arabic_punctuations + english_punctuations
+
+
+def remove_urls (text):
+    text = re.sub(r'(https|http)?:\/\/(\w|\.|\/|\?|\=|\&|\%)*\b', '', text, flags=re.MULTILINE)
+    return text
+
+
+def remove_emails(text):
+    text = re.sub(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)", "",  text, flags=re.MULTILINE)
+    return text
+
+# def remove_emoji(text):
+#     return emoji.get_emoji_regexp().sub(u'', text)
+
+def remove_emoji(data):
+    emoj = re.compile("["
+        u"\U0001F600-\U0001F64F"  # emoticons
+        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
+        u"\U0001F680-\U0001F6FF"  # transport & map symbols
+        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
+        u"\U00002500-\U00002BEF"  # chinese char
+        u"\U00002702-\U000027B0"
+        u"\U00002702-\U000027B0"
+        u"\U000024C2-\U0001F251"
+        u"\U0001f926-\U0001f937"
+        u"\U00010000-\U0010ffff"
+        u"\u2640-\u2642" 
+        u"\u2600-\u2B55"
+        u"\u200d"
+        u"\u23cf"
+        u"\u23e9"
+        u"\u231a"
+        u"\ufe0f"  # dingbats
+        u"\u3030"
+                    "]+", re.UNICODE)
+    return re.sub(emoj, '', data)
+
+def normalization(text):
+    text = re.sub("[إأآا]", "ا", text)
+    text = re.sub("ى", "ي", text)
+    text = re.sub("ؤ", "ء", text)
+    text = re.sub("ئ", "ء", text)
+    text = re.sub("ة", "ه", text)
+    text = re.sub("گ", "ك", text)
+    return text
+
+def remove_diacritics(text):
+    text = re.sub(arabic_diacritics, '', text)
+    return text
+
+def remove_stopwords(text):
+    filtered_sentence = [w for w in text.split() if not w in arabic_stopwords]
+    return ' '.join(filtered_sentence)
+
+def cleaning_content(line):
+    if (isinstance(line, float)):
+        return None
+    line.replace('\n', ' ')
+    line = remove_emails(line)
+    line = remove_urls(line)
+    line = remove_emoji(line)
+    nline = [w if '@' not in w else 'USERID' for w in line.split()]
+    line = ' '.join(nline)
+    line = line.replace('RT', '').replace('<LF>', '').replace('<br />','').replace('&quot;', '').replace('<url>', '').replace('USERID', '')
+
+
+    # add spaces between punc,
+    line = line.translate(str.maketrans({key: " {0} ".format(key) for key in punctuations}))
+
+    # then remove punc,
+    translator = str.maketrans('', '', punctuations)
+    line = line.translate(translator)
+
+    line = remove_stopwords(line)
+    line=remove_diacritics(normalization(line))
+
+    line = line.strip()
+    return line
+
+def hasDigits(s):
+    return any( 48 <= ord(char) <= 57  or 1632 <= ord(char) <= 1641 for char in s)

README.md

@@ -1,12 +1,12 @@
 ---
 title: NLP Project
-emoji: 🏃
-colorFrom: pink
-colorTo: gray
+emoji: 🚀
+colorFrom: red
+colorTo: purple
 sdk: streamlit
 sdk_version: 1.15.2
-app_file: app.py
+app_file: Deployment/app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

models/dialect_classifier.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8d9047effcbd6a8f914f074bdfa4aa3898969a22ea5ca0b75346dd5f20bbb66
+size 497881137

models/misogyny/label_encoder.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43bcfabf857ecea698a3d16884c09a138ef0ffdbae400b6dbe1bcb61153046a2
+size 540

models/misogyny/misogyny.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58cef97517696f0df0ffa7fdbd6492d299e0ecc35c5c3a1ba438cfb9da2e06a5
+size 168617261

models/modelv3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6f54084c52cfb8eb5370194b928a8ac19acffafd4c3b7ad13bb12c90667aeee
+size 541013353

models/offensive_dict.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22d8cb74c4ef29eb32ba61b0bfa173d9427fbb7348edae1e7ac7bcae7622cccb
+size 48

models/offensive_max_len.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d7d259e641cf037cbb2ed3449a27212d015e4481e113817ea8cabc26b65cabd
+size 15

models/racism/Racism_Detector.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8d5876d8d722a32b094fbaf85e008a31659ecf9c799a873f45097fb0156943a
+size 39546896

models/racism/racism_arabert.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8aeaa8895178ba5b65a65aa5fb7e1b6736dfd7d20441fa4a19e5ef722066d9a
+size 168617261

models/racism/racism_arabert_maxlen.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68d53d4b9a2d32c9ed1387fb638a991b45f38f07f8a1c2882018c62616998f21
+size 116

models/racism/racism_dict.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f27c423d7c2becfaecdb8029d1b7835605c622d62bca661099906506791b593
+size 42

models/racism/racismmaxlen.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfffe1ff1f9447c82a655b9d1140e64eb94a731d098c8b34db58d5488de9c0a7
+size 118

models/racism/racismtokenizer.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51d5d54e103e5c6d553e60725ad330ed02258bc1f7538dfb5dc3c00777e2953c
+size 365072

models/religion_hate/religion_hate_params.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfc0eb24605267d5bc8c3f74f96e4c77143510795adc6702b8d865db32a5f8a2
+size 168617261

models/verbal_abuse/verbal_abuse_arabert.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd98c5df09ec6070888c3731975566d5fc62c65d60204bbaa31831840b2f18f1
+size 168617261

requirements.txt

@@ -0,0 +1,7 @@
+pytorch-lightning == 1.8.6
+torch == 1.11.0+cu113
+transformers == 4.23.1
+numpy == 1.18.5
+pandas == 1.4.0
+nltk == 3.7
+geopy == 2.3.0