Update README.md

README.md

@@ -28,27 +28,139 @@ RTE focuses on evaluating the support or refutation of claims within a given tex
 
 ## Inference API Usage
 
-When using the Inference API, it is important to note that the input should be provided by pasting the text first, followed by the claim, without any spaces or separators. The model's tokenizer concatenates these inputs in the specified order. Interestingly, inverting the order of pasting (claim first, then text) seems to produce similar results, suggesting that the model generally captures coherence within a given text.
+When using the Inference API, it is important to note that the input should be provided by pasting the text first, followed by the claim, without any spaces or separators. The model's tokenizer concatenates these inputs in the specified order. Interestingly, inverting the order of pasting (claim first, then text) seems to produce similar results, suggesting that the model generally captures coherence within a given text (the label 0 indicates a coherent text, while the other label 1 signify an incoherent text).
 
 
 ## Training procedure
+The model was trained on Kaggle using as accelerator a GPU T4 x2. See the complete notebook here: <TODO>
+
+
+```python
+import json
+import numpy as np
+import os
+import pickle
+from IPython.display import clear_output
+import pandas as pd
+import tensorflow as tf
+import transformers
+from datasets import load_dataset
+from sklearn.metrics import confusion_matrix, classification_report
+from sklearn.model_selection import train_test_split
+from transformers import DistilBertTokenizer, TFDistilBertForSequenceClassification
+import warnings
+
+# Silence all warnings
+warnings.filterwarnings("ignore")
+
+
+# Try to create a directory named "models"
+try:
+    os.makedirs("models")
+except:
+    # If the directory already exists or if there's an error, do nothing (pass)
+    pass
+
+# Try to create a directory named "results"
+try:
+    os.makedirs("results")
+except:
+    # If the directory already exists or if there's an error, do nothing (pass)
+    pass
+
+# Try to create a directory named "history"
+try:
+    os.makedirs("history")
+except:
+    # If the directory already exists or if there's an error, do nothing (pass)
+    pass
+
+
+# Flag to determine if existing models and histories should be overwritten
+overwrite = True
+
+# Load dataset for the first fold
+data = load_dataset("raicrits/fever_folds", data_files="folds_en/1.json")['train']
+test = data['test'][0]
+val_set = data['val'][0]
+train_set = data['train'][0]
+
+# Define paths for model, results, and history
+model_path = 'models/DistilFEVERen_weights_0.h5'
+results_path = "results/DistilFEVERen_0.json"
+history_path = 'history/DistilFEVERen_0.pickle'
+
+# Load the tokenizer
+tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')
+
+# Preprocess the data
+test_encodings = tokenizer(test['text'], test['claim'], truncation=True, padding=True, max_length=256, return_tensors='tf')
+test_labels = tf.convert_to_tensor(test['label'])
+
+train_encodings = tokenizer(train_set['text'], train_set['claim'], truncation=True, padding=True, return_tensors='tf')
+val_encodings = tokenizer(val_set['text'], val_set['claim'], truncation=True, padding=True, return_tensors='tf')
+
+train_labels = tf.convert_to_tensor(train_set['label'])
+val_labels = tf.convert_to_tensor(val_set['label'])
+
+# Check if the model and history already exist for the first fold
+if not overwrite and os.path.exists(model_path):
+    print("Model and history already exist for fold {}. Loading...".format(0))
+    model = TFDistilBertForSequenceClassification.from_pretrained('distilbert-base-multilingual-cased', num_labels=3)
+    model.load_weights(model_path)
+    # with open(history_path, 'rb') as file_pi:
+    #    history = pickle.load(file_pi)
+else:
+    # Create a new model and define loss, optimizer, and callbacks
+    model = TFDistilBertForSequenceClassification.from_pretrained('distilbert-base-multilingual-cased', num_labels=3)
+    loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+    optimizer = tf.keras.optimizers.Adam(learning_rate=5e-5)
+    model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
+    model_checkpoint = tf.keras.callbacks.ModelCheckpoint(
+        model_path,
+        monitor='val_loss',
+        save_best_only=True,
+        mode='min',
+        save_weights_only=True
+    )
+    early_stopping = tf.keras.callbacks.EarlyStopping(
+        monitor='val_loss',
+        patience=1,
+        mode='min',
+        restore_best_weights=True
+    )
+
+    # Train the model for the first fold
+    clear_output(wait=True)
+    history = model.fit(
+        [train_encodings['input_ids'], train_encodings['attention_mask']], train_labels,
+        validation_data=([val_encodings['input_ids'], val_encodings['attention_mask']], val_labels),
+        batch_size=10,
+        epochs=100,
+        callbacks=[early_stopping, model_checkpoint]
+    )
+
+    # Save the training history
+    with open(history_path, 'wb') as file_pi:
+        pickle.dump(history.history, file_pi)
+```
 
 
-
-### Training hyperparameters
-
-The following hyperparameters were used during training:
-- optimizer: None
-- training_precision: float32
-
 ## Inference procedure
+```python
+def getPrediction(model,tokenizer,claim,text):
+    encodings = tokenizer([text], [claim], truncation=True, padding=True, max_length=256, return_tensors='tf')
+    preds = model.predict([encodings['input_ids'], encodings["attention_mask"]])
+    return preds
 
+text = "Soul Food is a 1997 American comedy-drama film produced by Kenneth `` Babyface '' Edmonds , Tracey Edmonds and Robert Teitel and released by Fox 2000 Pictures ."
+claim = 'Fox 2000 Pictures released the film Soul Food .'
+getPrediction(model,tokenizer,claim,text)
+```
 
 ### Evaluation results
 It achieves the following results on the evaluation set:
 
-## Inference procedure
-
 
 
 ### Framework versions
@@ -56,4 +168,5 @@ It achieves the following results on the evaluation set:
 - Transformers 4.35.0
 - TensorFlow 2.13.0
 - Datasets 2.1.0
-- Tokenizers 0.14.1
+- Tokenizers 0.14.1
+- Numpy 1.24.3