Upload fine_tune_json3.ipynb

fine_tune_json3.ipynb

@@ -0,0 +1,503 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\aft.AFREETECH\\Downloads\\model\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import json\n",
+    "import numpy as np\n",
+    "import torch\n",
+    "from transformers import GPT2LMHeadModel, GPT2Tokenizer, AdamW, get_linear_schedule_with_warmup\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Charger le dataset\n",
+    "with open('data/data_test.json', 'r') as f:\n",
+    "    data = json.load(f)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Séparer les données d'entrée et de sortie\n",
+    "inputs = [d['input'] for d in data]\n",
+    "outputs = [d['output'] for d in data]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Tokenizer\n",
+    "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Définir le token de padding\n",
+    "tokenizer.pad_token = tokenizer.eos_token  # Utiliser le token de fin comme token de padding\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Encoder les entrées et sorties avec padding\n",
+    "max_length = 128\n",
+    "input_ids = [tokenizer.encode(input_text, add_special_tokens=True, max_length=max_length, truncation=True, padding='max_length') for input_text in inputs]\n",
+    "output_ids = [tokenizer.encode(output_text, add_special_tokens=True, max_length=max_length, truncation=True, padding='max_length') for output_text in outputs]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Convertir en tenseurs numpy\n",
+    "input_ids = np.array(input_ids).astype(np.int64)  # Convertir en entiers longs\n",
+    "output_ids = np.array(output_ids).astype(np.int64)  # Convertir en entiers longs\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Charger le modèle pré-entraîné\n",
+    "model = GPT2LMHeadModel.from_pretrained('gpt2')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\aft.AFREETECH\\Downloads\\model\\.venv\\Lib\\site-packages\\transformers\\optimization.py:591: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Configurer l'entraînement\n",
+    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
+    "model.to(device)\n",
+    "optimizer = AdamW(model.parameters(), lr=1e-5)\n",
+    "scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=len(input_ids) * 10)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Configurer l'entraînement\n",
+    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
+    "model.to(device)\n",
+    "optimizer = AdamW(model.parameters(), lr=1e-5)\n",
+    "scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=len(input_ids) * 10)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\aft.AFREETECH\\AppData\\Local\\Temp\\ipykernel_3092\\3831564103.py:8: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ..\\torch\\csrc\\utils\\tensor_new.cpp:277.)\n",
+      "  input_ids_tensor = torch.tensor([input_ids[i]], device=device, dtype=torch.long)  # Convertir en tenseur Long\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch 1/10, Loss: 9.18336634202437\n",
+      "Epoch 2/10, Loss: 7.024676236239347\n",
+      "Epoch 3/10, Loss: 6.38383115421642\n",
+      "Epoch 4/10, Loss: 6.083545771512118\n",
+      "Epoch 5/10, Loss: 5.902632973410866\n",
+      "Epoch 6/10, Loss: 5.789845033125444\n",
+      "Epoch 7/10, Loss: 5.625866976651278\n",
+      "Epoch 8/10, Loss: 5.553086866031993\n",
+      "Epoch 9/10, Loss: 5.5227460861206055\n",
+      "Epoch 10/10, Loss: 5.475944844159213\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Entraîner le modèle\n",
+    "epochs = 10\n",
+    "for epoch in range(epochs):\n",
+    "    model.train()\n",
+    "    total_loss = 0\n",
+    "    for i in range(len(input_ids)):\n",
+    "        optimizer.zero_grad()\n",
+    "        input_ids_tensor = torch.tensor([input_ids[i]], device=device, dtype=torch.long)  # Convertir en tenseur Long\n",
+    "        output_ids_tensor = torch.tensor([output_ids[i]], device=device, dtype=torch.long)  # Convertir en tenseur Long\n",
+    "        outputs = model(input_ids_tensor, labels=output_ids_tensor)\n",
+    "        loss = outputs[0]\n",
+    "        loss.backward()\n",
+    "        optimizer.step()\n",
+    "        scheduler.step()\n",
+    "        total_loss += loss.item()\n",
+    "    print(f'Epoch {epoch+1}/{epochs}, Loss: {total_loss/len(input_ids)}')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Sauvegarder le modèle\n",
+    "model.save_pretrained('test_generator_model')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GPT2LMHeadModel(\n",
+       "  (transformer): GPT2Model(\n",
+       "    (wte): Embedding(50257, 768)\n",
+       "    (wpe): Embedding(1024, 768)\n",
+       "    (drop): Dropout(p=0.1, inplace=False)\n",
+       "    (h): ModuleList(\n",
+       "      (0-11): 12 x GPT2Block(\n",
+       "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (attn): GPT2SdpaAttention(\n",
+       "          (c_attn): Conv1D()\n",
+       "          (c_proj): Conv1D()\n",
+       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
+       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "        (mlp): GPT2MLP(\n",
+       "          (c_fc): Conv1D()\n",
+       "          (c_proj): Conv1D()\n",
+       "          (act): NewGELUActivation()\n",
+       "          (dropout): Dropout(p=0.1, inplace=False)\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "  )\n",
+       "  (lm_head): Linear(in_features=768, out_features=50257, bias=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## tome 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\aft.AFREETECH\\Downloads\\model\\.venv\\Lib\\site-packages\\transformers\\generation\\configuration_utils.py:567: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.7` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
+      "  warnings.warn(\n",
+      "c:\\Users\\aft.AFREETECH\\Downloads\\model\\.venv\\Lib\\site-packages\\transformers\\generation\\configuration_utils.py:572: UserWarning: `do_sample` is set to `False`. However, `top_p` is set to `0.95` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.\n",
+      "  warnings.warn(\n",
+      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Transform the following Python function into a pytest unit test function:\n",
+      "def mel2hz(mel):\n",
+      "    return 700*(10**(mel/2595.0)-1)\n",
+      "def mel3hz(mel):\n",
+      "   \n",
+      "def mel4hz(mel):\n",
+      "  \n",
+      "def mel5hz(mel):\n",
+      "  \n",
+      "def mel6hz(mel):\n",
+      "  \n",
+      "def mel7hz(mel):\n",
+      "  \n",
+      "def mel8hz(mel):\n",
+      "  \n",
+      "def mel9hz(mel):\n",
+      "  \n"
+     ]
+    }
+   ],
+   "source": [
+    "import inspect\n",
+    "import torch\n",
+    "from transformers import GPT2LMHeadModel, GPT2Tokenizer\n",
+    "\n",
+    "# Charger le modèle et le tokenizer\n",
+    "model = GPT2LMHeadModel.from_pretrained('test_generator_model')\n",
+    "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
+    "\n",
+    "# Définir la fonction Python à tester\n",
+    "def mel2hz(mel):\n",
+    "    return 700*(10**(mel/2595.0)-1)\n",
+    "\n",
+    "# Encoder la fonction en entrée\n",
+    "input_text = f\"Transform the following Python function into a pytest unit test function:\\n{inspect.getsource(mel2hz)}\"\n",
+    "input_ids = tokenizer.encode(input_text, return_tensors='pt')\n",
+    "\n",
+    "# Créer un attention mask\n",
+    "attention_mask = torch.ones(input_ids.shape, dtype=torch.long)\n",
+    "\n",
+    "# Générer le test unitaire avec des paramètres ajustés\n",
+    "output_ids = model.generate(\n",
+    "    input_ids,\n",
+    "    attention_mask=attention_mask,\n",
+    "    max_length=256,\n",
+    "    num_return_sequences=1,\n",
+    "    num_beams=5, \n",
+    "    temperature=0.7,\n",
+    "    top_k=50,  \n",
+    "    top_p=0.95,  \n",
+    "    early_stopping=True\n",
+    ")\n",
+    "\n",
+    "# Décoder et afficher le résultat\n",
+    "generated_test = tokenizer.decode(output_ids[0], skip_special_tokens=True)\n",
+    "print(generated_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import numpy as np\n",
+    "import torch\n",
+    "from transformers import T5ForConditionalGeneration, T5Tokenizer, AdamW, get_linear_schedule_with_warmup\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Charger le dataset\n",
+    "with open('data/data_test.json', 'r') as f:\n",
+    "    data = json.load(f)\n",
+    "\n",
+    "# Séparer les données d'entrée et de sortie\n",
+    "inputs = [d['input'] for d in data]\n",
+    "outputs = [d['output'] for d in data]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Tokenizer\n",
+    "tokenizer = T5Tokenizer.from_pretrained('t5-small')\n",
+    "\n",
+    "# Encoder les entrées et sorties\n",
+    "max_length = 128\n",
+    "input_ids = tokenizer(inputs, max_length=max_length, padding=True, truncation=True, return_tensors='pt').input_ids\n",
+    "output_ids = tokenizer(outputs, max_length=max_length, padding=True, truncation=True, return_tensors='pt').input_ids\n",
+    "\n",
+    "# Charger le modèle pré-entraîné\n",
+    "model = T5ForConditionalGeneration.from_pretrained('t5-small')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Configurer l'entraînement\n",
+    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
+    "model.to(device)\n",
+    "optimizer = AdamW(model.parameters(), lr=1e-5)\n",
+    "scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=len(input_ids) * 10)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Entraîner le modèle\n",
+    "epochs = 5\n",
+    "for epoch in range(epochs):\n",
+    "    model.train()\n",
+    "    total_loss = 0\n",
+    "    for i in range(len(input_ids)):\n",
+    "        optimizer.zero_grad()\n",
+    "        input_ids_tensor = input_ids[i].unsqueeze(0).to(device)  # Convertir en tenseur Long\n",
+    "        output_ids_tensor = output_ids[i].unsqueeze(0).to(device)  # Convertir en tenseur Long\n",
+    "        outputs = model(input_ids_tensor, labels=output_ids_tensor)\n",
+    "        loss = outputs.loss\n",
+    "        loss.backward()\n",
+    "        optimizer.step()\n",
+    "        scheduler.step()\n",
+    "        total_loss += loss.item()\n",
+    "    print(f'Epoch {epoch+1}/{epochs}, Loss: {total_loss/len(input_ids)}')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# Sauvegarder le modèle\n",
+    "model.save_pretrained('test_generator_model_t5')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import inspect\n",
+    "import torch\n",
+    "from transformers import T5ForConditionalGeneration, T5Tokenizer\n",
+    "\n",
+    "# Charger le modèle et le tokenizer\n",
+    "model = T5ForConditionalGeneration.from_pretrained('test_generator_model_t5')\n",
+    "tokenizer = T5Tokenizer.from_pretrained('t5-small')\n",
+    "\n",
+    "# Définir la fonction Python à tester\n",
+    "def mel2hz(mel):\n",
+    "    return 700 * (10 ** (mel / 2595.0) - 1)\n",
+    "\n",
+    "# Encoder la fonction en entrée\n",
+    "input_text = f\"Please write a pytest unit test for the following Python function:\\n{inspect.getsource(mel2hz)}\"\n",
+    "input_ids = tokenizer.encode(input_text, return_tensors='pt')\n",
+    "\n",
+    "# Créer un attention mask\n",
+    "attention_mask = torch.ones(input_ids.shape, dtype=torch.long)\n",
+    "\n",
+    "# Générer le test unitaire avec des paramètres ajustés\n",
+    "output_ids = model.generate(\n",
+    "    input_ids,\n",
+    "    attention_mask=attention_mask,\n",
+    "    max_length=256,\n",
+    "    num_return_sequences=1,\n",
+    "    num_beams=5,\n",
+    "    temperature=0.7,  # Ajustement de la température\n",
+    "    top_k=50,\n",
+    "    top_p=0.95,\n",
+    "    early_stopping=True\n",
+    ")\n",
+    "\n",
+    "# Décoder et afficher le résultat\n",
+    "generated_test = tokenizer.decode(output_ids[0], skip_special_tokens=True)\n",
+    "print(generated_test)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}