fix(camembert): camembert fine-tuning

camemBERT_finetuning.ipynb

@@ -11,30 +11,39 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 86,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
+      "To disable this warning, you can either:\n",
+      "\t- Avoid using `tokenizers` before the fork if possible\n",
+      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
+     ]
+    },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Requirement already satisfied: transformers in ./venv/lib/python3.12/site-packages (4.46.3)\n",
+      "Requirement already satisfied: transformers in ./venv/lib/python3.12/site-packages (4.47.1)\n",
       "Requirement already satisfied: tf-keras in ./venv/lib/python3.12/site-packages (2.18.0)\n",
-      "Collecting focal-loss\n",
-      "  Downloading focal_loss-0.0.7-py3-none-any.whl.metadata (5.1 kB)\n",
+      "Requirement already satisfied: focal-loss in ./venv/lib/python3.12/site-packages (0.0.7)\n",
       "Requirement already satisfied: filelock in ./venv/lib/python3.12/site-packages (from transformers) (3.16.1)\n",
-      "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in ./venv/lib/python3.12/site-packages (from transformers) (0.26.3)\n",
+      "Requirement already satisfied: huggingface-hub<1.0,>=0.24.0 in ./venv/lib/python3.12/site-packages (from transformers) (0.26.5)\n",
       "Requirement already satisfied: numpy>=1.17 in ./venv/lib/python3.12/site-packages (from transformers) (1.26.4)\n",
-      "Requirement already satisfied: packaging>=20.0 in ./venv/lib/python3.12/site-packages (from transformers) (24.1)\n",
+      "Requirement already satisfied: packaging>=20.0 in ./venv/lib/python3.12/site-packages (from transformers) (24.2)\n",
       "Requirement already satisfied: pyyaml>=5.1 in ./venv/lib/python3.12/site-packages (from transformers) (6.0.2)\n",
-      "Requirement already satisfied: regex!=2019.12.17 in ./venv/lib/python3.12/site-packages (from transformers) (2024.9.11)\n",
+      "Requirement already satisfied: regex!=2019.12.17 in ./venv/lib/python3.12/site-packages (from transformers) (2024.11.6)\n",
       "Requirement already satisfied: requests in ./venv/lib/python3.12/site-packages (from transformers) (2.32.3)\n",
-      "Requirement already satisfied: tokenizers<0.21,>=0.20 in ./venv/lib/python3.12/site-packages (from transformers) (0.20.3)\n",
+      "Requirement already satisfied: tokenizers<0.22,>=0.21 in ./venv/lib/python3.12/site-packages (from transformers) (0.21.0)\n",
       "Requirement already satisfied: safetensors>=0.4.1 in ./venv/lib/python3.12/site-packages (from transformers) (0.4.5)\n",
       "Requirement already satisfied: tqdm>=4.27 in ./venv/lib/python3.12/site-packages (from transformers) (4.66.5)\n",
       "Requirement already satisfied: tensorflow<2.19,>=2.18 in ./venv/lib/python3.12/site-packages (from tf-keras) (2.18.0)\n",
-      "Requirement already satisfied: fsspec>=2023.5.0 in ./venv/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (2024.10.0)\n",
-      "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./venv/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (4.12.2)\n",
+      "Requirement already satisfied: fsspec>=2023.5.0 in ./venv/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.24.0->transformers) (2024.10.0)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./venv/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.24.0->transformers) (4.12.2)\n",
       "Requirement already satisfied: absl-py>=1.0.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (2.1.0)\n",
       "Requirement already satisfied: astunparse>=1.6.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.6.3)\n",
       "Requirement already satisfied: flatbuffers>=24.3.25 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (24.3.25)\n",
@@ -43,11 +52,11 @@
       "Requirement already satisfied: libclang>=13.0.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (18.1.1)\n",
       "Requirement already satisfied: opt-einsum>=2.3.2 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (3.4.0)\n",
       "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0dev,>=3.20.3 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (4.25.5)\n",
-      "Requirement already satisfied: setuptools in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (75.2.0)\n",
-      "Requirement already satisfied: six>=1.12.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.16.0)\n",
+      "Requirement already satisfied: setuptools in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (75.6.0)\n",
+      "Requirement already satisfied: six>=1.12.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.17.0)\n",
       "Requirement already satisfied: termcolor>=1.1.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (2.5.0)\n",
-      "Requirement already satisfied: wrapt>=1.11.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.16.0)\n",
-      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.67.0)\n",
+      "Requirement already satisfied: wrapt>=1.11.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.17.0)\n",
+      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.68.1)\n",
       "Requirement already satisfied: tensorboard<2.19,>=2.18 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (2.18.0)\n",
       "Requirement already satisfied: keras>=3.5.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (3.7.0)\n",
       "Requirement already satisfied: h5py>=3.11.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (3.12.1)\n",
@@ -56,23 +65,17 @@
       "Requirement already satisfied: idna<4,>=2.5 in ./venv/lib/python3.12/site-packages (from requests->transformers) (3.10)\n",
       "Requirement already satisfied: urllib3<3,>=1.21.1 in ./venv/lib/python3.12/site-packages (from requests->transformers) (2.2.3)\n",
       "Requirement already satisfied: certifi>=2017.4.17 in ./venv/lib/python3.12/site-packages (from requests->transformers) (2024.8.30)\n",
-      "Requirement already satisfied: wheel<1.0,>=0.23.0 in ./venv/lib/python3.12/site-packages (from astunparse>=1.6.0->tensorflow<2.19,>=2.18->tf-keras) (0.44.0)\n",
-      "Requirement already satisfied: rich in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (13.9.2)\n",
+      "Requirement already satisfied: wheel<1.0,>=0.23.0 in ./venv/lib/python3.12/site-packages (from astunparse>=1.6.0->tensorflow<2.19,>=2.18->tf-keras) (0.45.1)\n",
+      "Requirement already satisfied: rich in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (13.9.4)\n",
       "Requirement already satisfied: namex in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (0.0.8)\n",
-      "Requirement already satisfied: optree in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (0.13.0)\n",
+      "Requirement already satisfied: optree in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (0.13.1)\n",
       "Requirement already satisfied: markdown>=2.6.8 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf-keras) (3.7)\n",
       "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf-keras) (0.7.2)\n",
-      "Requirement already satisfied: werkzeug>=1.0.1 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf-keras) (3.0.4)\n",
+      "Requirement already satisfied: werkzeug>=1.0.1 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf-keras) (3.1.3)\n",
       "Requirement already satisfied: MarkupSafe>=2.1.1 in ./venv/lib/python3.12/site-packages (from werkzeug>=1.0.1->tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf-keras) (3.0.2)\n",
       "Requirement already satisfied: markdown-it-py>=2.2.0 in ./venv/lib/python3.12/site-packages (from rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (3.0.0)\n",
       "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in ./venv/lib/python3.12/site-packages (from rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (2.18.0)\n",
-      "Requirement already satisfied: mdurl~=0.1 in ./venv/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (0.1.2)\n",
-      "Downloading focal_loss-0.0.7-py3-none-any.whl (19 kB)\n",
-      "Installing collected packages: focal-loss\n",
-      "Successfully installed focal-loss-0.0.7\n",
-      "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
+      "Requirement already satisfied: mdurl~=0.1 in ./venv/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (0.1.2)\n"
      ]
     }
    ],
@@ -82,7 +85,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 87,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -93,7 +96,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 88,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -102,7 +105,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 89,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -113,46 +116,41 @@
     ")\n",
     "\n",
     "# To avoid overfitting the model on sentences that don't have any labels\n",
-    "# lambda_sentences, lambda_labels, _, __ = dp.from_bio_file_to_examples(\n",
-    "#     \"./data/bio/fr.bio/1k_train_unlabeled_samples.bio\"\n",
-    "# )\n",
-    "\n",
-    "large_sentences, large_labels, _, __ = dp.from_bio_file_to_examples(\n",
-    "    \"./data/bio/fr.bio/1k_train_large_samples.bio\"\n",
+    "lambda_sentences, lambda_labels, _, __ = dp.from_bio_file_to_examples(\n",
+    "    \"./data/bio/fr.bio/1k_train_unlabeled_samples.bio\"\n",
     ")\n",
     "\n",
-    "sentences = sentences + large_sentences\n",
-    "labels = labels + large_labels"
+    "long_sentences, long_labels, _, __ = dp.from_bio_file_to_examples(\n",
+    "    \"./data/bio/fr.bio/1k_train_large_samples.bio\"\n",
+    ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 90,
    "metadata": {},
    "outputs": [],
    "source": [
-    "import app.travel_resolver.libs.nlp.data_processing as dp\n",
-    "\n",
-    "processed_sentences, processed_labels = dp.process_sentences_and_labels(\n",
-    "    sentences, labels, return_tokens=True, stemming=False\n",
-    ")"
+    "sentences = sentences + lambda_sentences + long_sentences\n",
+    "labels = labels + lambda_labels + long_labels"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 91,
    "metadata": {},
    "outputs": [],
    "source": [
-    "for i in range(len(processed_sentences)):\n",
-    "    for j in range(len(processed_sentences[i])):\n",
-    "        if processed_labels[i][j] > 0:\n",
-    "            processed_sentences[i][j] = processed_sentences[i][j].title()"
+    "import app.travel_resolver.libs.nlp.data_processing as dp\n",
+    "\n",
+    "processed_sentences, processed_labels = dp.process_sentences_and_labels(\n",
+    "    sentences, labels, return_tokens=True, stemming=False\n",
+    ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 92,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -161,12 +159,12 @@
     "  as well as the embedding size\n",
     "\"\"\"\n",
     "\n",
-    "MAX_LEN = 100"
+    "MAX_LEN = 150"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 93,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -177,25 +175,26 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 94,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from transformers import TFAutoModelForTokenClassification, CamembertTokenizer\n",
+    "from transformers import TFAutoModelForTokenClassification, CamembertTokenizerFast\n",
     "import numpy as np\n",
     "\n",
-    "tokenizer = CamembertTokenizer.from_pretrained(\"camembert-base\")"
+    "tokenizer = CamembertTokenizerFast.from_pretrained(\"cmarkea/distilcamembert-base\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 95,
    "metadata": {},
    "outputs": [],
    "source": [
     "tokenized_sentences = tokenizer(\n",
     "    processed_sentences,\n",
     "    is_split_into_words=True,\n",
+    "    return_offsets_mapping=True,\n",
     "    truncation=True,\n",
     "    padding=\"max_length\",\n",
     "    max_length=MAX_LEN,\n",
@@ -204,7 +203,62 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 96,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def align_labels_with_tokens(encodings, labels):\n",
+    "    \"\"\"\n",
+    "    Aligns the labels to match the tokenized outputs.\n",
+    "\n",
+    "    Args:\n",
+    "        encodings (BatchEncoding): Tokenized outputs from the Hugging Face tokenizer (must use a fast tokenizer).\n",
+    "        labels (List[List[int]]): Original labels for each sentence before tokenization. Each inner list corresponds to one sentence.\n",
+    "\n",
+    "    Returns:\n",
+    "        List[List[int]]: Aligned labels, where each inner list corresponds to the aligned labels for the tokenized sentence.\n",
+    "                         Special tokens and padding are assigned a value of -100.\n",
+    "    \"\"\"\n",
+    "    adapted_labels = []\n",
+    "\n",
+    "    for i, label in enumerate(labels):\n",
+    "        word_ids = encodings.word_ids(\n",
+    "            batch_index=i\n",
+    "        )  # Get word IDs for the i-th sentence\n",
+    "        aligned_labels = []\n",
+    "        previous_word_id = None\n",
+    "\n",
+    "        for word_id in word_ids:\n",
+    "            if word_id is None:\n",
+    "                # Special tokens (e.g., [CLS], [SEP], or padding)\n",
+    "                aligned_labels.append(-100)\n",
+    "            elif word_id != previous_word_id:\n",
+    "                # New word\n",
+    "                aligned_labels.append(label[word_id])\n",
+    "            else:\n",
+    "                # Subword token (same word)\n",
+    "                aligned_labels.append(\n",
+    "                    label[word_id]\n",
+    "                )  # Or append -100 to ignore subwords\n",
+    "            previous_word_id = word_id\n",
+    "\n",
+    "        adapted_labels.append(aligned_labels)\n",
+    "\n",
+    "    return adapted_labels"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 97,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "readapted_labels = align_labels_with_tokens(tokenized_sentences, padded_labels)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 98,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -220,14 +274,14 @@
     ") = train_test_split(\n",
     "    tokenized_sentences[\"input_ids\"],\n",
     "    tokenized_sentences[\"attention_mask\"],\n",
-    "    padded_labels,\n",
+    "    readapted_labels,\n",
     "    test_size=0.2,\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 99,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -254,7 +308,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 40,
+   "execution_count": 100,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -290,37 +344,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "class_weights = {0: 0.1, 1: 20.0, 2: 20.0}\n",
-    "\n",
-    "\n",
-    "def weighted_loss(y_true, y_pred):\n",
-    "    weights = tf.constant(\n",
-    "        [class_weights[i] for i in range(len(class_weights))], dtype=tf.float32\n",
-    "    )\n",
-    "    weights = tf.gather(\n",
-    "        weights, tf.cast(y_true, tf.int32)\n",
-    "    )  # Get weights for true labels\n",
-    "    loss = tf.keras.losses.sparse_categorical_crossentropy(\n",
-    "        y_true, y_pred, from_logits=True\n",
-    "    )\n",
-    "    return loss * weights"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 61,
+   "execution_count": 101,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "All PyTorch model weights were used when initializing TFCamembertForTokenClassification.\n",
-      "\n",
+      "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFCamembertForTokenClassification: ['roberta.embeddings.position_ids']\n",
+      "- This IS expected if you are initializing TFCamembertForTokenClassification from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing TFCamembertForTokenClassification from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
       "Some weights or buffers of the TF 2.0 model TFCamembertForTokenClassification were not initialized from the PyTorch model and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
       "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
      ]
@@ -330,73 +363,77 @@
     "from focal_loss import SparseCategoricalFocalLoss\n",
     "\n",
     "camembert = TFAutoModelForTokenClassification.from_pretrained(\n",
-    "    \"camembert-base\", num_labels=len(unique_labels)\n",
+    "    \"cmarkea/distilcamembert-base\", num_labels=len(unique_labels)\n",
     ")\n",
     "\n",
     "loss_func = SparseCategoricalFocalLoss(\n",
-    "    gamma=2, class_weight=[0.1, 2, 2], from_logits=True\n",
+    "    gamma=2, class_weight=[1, 10, 10], from_logits=True\n",
     ")\n",
     "\n",
     "camembert.compile(\n",
-    "    optimizer=tf.keras.optimizers.legacy.Adam(5e-5),\n",
-    "    loss=loss_func,\n",
-    "    metrics=[\"accuracy\", entity_accuracy],\n",
+    "    optimizer=tf.keras.optimizers.legacy.Adam(8e-4),\n",
+    "    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
+    "    metrics=[entity_accuracy],\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 46,
+   "execution_count": 102,
    "metadata": {},
    "outputs": [],
    "source": [
-    "train_dataset = train_dataset.batch(32)\n",
-    "test_dataset = test_dataset.batch(32)"
+    "train_dataset = train_dataset.batch(64)\n",
+    "test_dataset = test_dataset.batch(64)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 62,
+   "execution_count": 103,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Epoch 1/4\n",
-      "272/272 [==============================] - 1596s 6s/step - loss: 0.0124 - accuracy: 0.9677 - entity_accuracy: 0.8099 - val_loss: 0.0038 - val_accuracy: 0.9799 - val_entity_accuracy: 0.9682\n",
-      "Epoch 2/4\n",
-      "272/272 [==============================] - 1560s 6s/step - loss: 0.0031 - accuracy: 0.9852 - entity_accuracy: 0.9684 - val_loss: 0.0019 - val_accuracy: 0.9885 - val_entity_accuracy: 0.9820\n",
-      "Epoch 3/4\n",
-      "272/272 [==============================] - 1560s 6s/step - loss: 0.0020 - accuracy: 0.9907 - entity_accuracy: 0.9767 - val_loss: 0.0016 - val_accuracy: 0.9941 - val_entity_accuracy: 0.9775\n",
-      "Epoch 4/4\n",
-      "272/272 [==============================] - 1605s 6s/step - loss: 0.0016 - accuracy: 0.9923 - entity_accuracy: 0.9789 - val_loss: 0.0017 - val_accuracy: 0.9920 - val_entity_accuracy: 0.9831\n"
+      "Epoch 1/10\n"
      ]
     },
     {
-     "data": {
-      "text/plain": [
-       "<tf_keras.src.callbacks.History at 0x2dab031a0>"
-      ]
-     },
-     "execution_count": 62,
-     "metadata": {},
-     "output_type": "execute_result"
+     "ename": "TypeError",
+     "evalue": "in user code:\n\n    File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/engine/training.py\", line 1398, in train_function  *\n        return step_function(self, iterator)\n    File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/engine/training.py\", line 1381, in step_function  **\n        outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/engine/training.py\", line 1370, in run_step  **\n        outputs = model.train_step(data)\n    File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/transformers/modeling_tf_utils.py\", line 1672, in train_step\n        y_pred = self(x, training=True)\n    File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/utils/traceback_utils.py\", line 70, in error_handler\n        raise e.with_traceback(filtered_tb) from None\n    File \"/var/folders/3h/5n6s9rcj3sx0gpncsxbq_99m0000gn/T/__autograph_generated_filepc984rni.py\", line 40, in tf__run_call_with_unpacked_inputs\n        raise\n\n    TypeError: Exception encountered when calling layer 'tf_camembert_for_token_classification_5' (type TFCamembertForTokenClassification).\n    \n    in user code:\n    \n        File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/transformers/modeling_tf_utils.py\", line 1393, in run_call_with_unpacked_inputs  *\n            return func(self, **unpacked_inputs)\n    \n        TypeError: outer_factory.<locals>.inner_factory.<locals>.tf__call() got an unexpected keyword argument 'offset_mapping'\n    \n    \n    Call arguments received by layer 'tf_camembert_for_token_classification_5' (type TFCamembertForTokenClassification):\n      • input_ids={'input_ids': 'tf.Tensor(shape=(None, 150), dtype=int32)', 'attention_mask': 'tf.Tensor(shape=(None, 150), dtype=int32)', 'offset_mapping': 'tf.Tensor(shape=(None, 150, 2), dtype=int32)'}\n      • attention_mask=None\n      • token_type_ids=None\n      • position_ids=None\n      • head_mask=None\n      • inputs_embeds=None\n      • output_attentions=None\n      • output_hidden_states=None\n      • return_dict=None\n      • labels=None\n      • training=True\n",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[103], line 7\u001b[0m\n\u001b[1;32m      1\u001b[0m early_stopping \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mkeras\u001b[38;5;241m.\u001b[39mcallbacks\u001b[38;5;241m.\u001b[39mEarlyStopping(\n\u001b[1;32m      2\u001b[0m     monitor\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mval_loss\u001b[39m\u001b[38;5;124m\"\u001b[39m, min_delta\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.001\u001b[39m, patience\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m, restore_best_weights\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m      3\u001b[0m )\n\u001b[1;32m      5\u001b[0m csv_logger \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mkeras\u001b[38;5;241m.\u001b[39mcallbacks\u001b[38;5;241m.\u001b[39mCSVLogger(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtraining.log\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 7\u001b[0m \u001b[43mcamembert\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m      8\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtrain_dataset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      9\u001b[0m \u001b[43m    \u001b[49m\u001b[43mvalidation_data\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtest_dataset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     10\u001b[0m \u001b[43m    \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     11\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43mearly_stopping\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcsv_logger\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     12\u001b[0m \u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/transformers/modeling_tf_utils.py:1229\u001b[0m, in \u001b[0;36mTFPreTrainedModel.fit\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1226\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(keras\u001b[38;5;241m.\u001b[39mModel\u001b[38;5;241m.\u001b[39mfit)\n\u001b[1;32m   1227\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfit\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m   1228\u001b[0m     args, kwargs \u001b[38;5;241m=\u001b[39m convert_batch_encoding(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m-> 1229\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/utils/traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[1;32m     68\u001b[0m     \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[1;32m     69\u001b[0m     \u001b[38;5;66;03m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[0;32m---> 70\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     71\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     72\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",
+      "File \u001b[0;32m/var/folders/3h/5n6s9rcj3sx0gpncsxbq_99m0000gn/T/__autograph_generated_filelw03dryu.py:15\u001b[0m, in \u001b[0;36mouter_factory.<locals>.inner_factory.<locals>.tf__train_function\u001b[0;34m(iterator)\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     14\u001b[0m     do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m---> 15\u001b[0m     retval_ \u001b[38;5;241m=\u001b[39m ag__\u001b[38;5;241m.\u001b[39mconverted_call(ag__\u001b[38;5;241m.\u001b[39mld(step_function), (ag__\u001b[38;5;241m.\u001b[39mld(\u001b[38;5;28mself\u001b[39m), ag__\u001b[38;5;241m.\u001b[39mld(iterator)), \u001b[38;5;28;01mNone\u001b[39;00m, fscope)\n\u001b[1;32m     16\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[1;32m     17\u001b[0m     do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n",
+      "File \u001b[0;32m~/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/transformers/modeling_tf_utils.py:1672\u001b[0m, in \u001b[0;36mTFPreTrainedModel.train_step\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m   1670\u001b[0m     y_pred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m(x, training\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, return_loss\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m   1671\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1672\u001b[0m     y_pred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtraining\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m   1673\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_using_dummy_loss:\n\u001b[1;32m   1674\u001b[0m     loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcompiled_loss(y_pred\u001b[38;5;241m.\u001b[39mloss, y_pred\u001b[38;5;241m.\u001b[39mloss, sample_weight, regularization_losses\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlosses)\n",
+      "File \u001b[0;32m/var/folders/3h/5n6s9rcj3sx0gpncsxbq_99m0000gn/T/__autograph_generated_filepc984rni.py:37\u001b[0m, in \u001b[0;36mouter_factory.<locals>.inner_factory.<locals>.tf__run_call_with_unpacked_inputs\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     36\u001b[0m     do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m---> 37\u001b[0m     retval_ \u001b[38;5;241m=\u001b[39m ag__\u001b[38;5;241m.\u001b[39mconverted_call(ag__\u001b[38;5;241m.\u001b[39mld(func), (ag__\u001b[38;5;241m.\u001b[39mld(\u001b[38;5;28mself\u001b[39m),), \u001b[38;5;28mdict\u001b[39m(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mag__\u001b[38;5;241m.\u001b[39mld(unpacked_inputs)), fscope)\n\u001b[1;32m     38\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[1;32m     39\u001b[0m     do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n",
+      "\u001b[0;31mTypeError\u001b[0m: in user code:\n\n    File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/engine/training.py\", line 1398, in train_function  *\n        return step_function(self, iterator)\n    File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/engine/training.py\", line 1381, in step_function  **\n        outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/engine/training.py\", line 1370, in run_step  **\n        outputs = model.train_step(data)\n    File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/transformers/modeling_tf_utils.py\", line 1672, in train_step\n        y_pred = self(x, training=True)\n    File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/utils/traceback_utils.py\", line 70, in error_handler\n        raise e.with_traceback(filtered_tb) from None\n    File \"/var/folders/3h/5n6s9rcj3sx0gpncsxbq_99m0000gn/T/__autograph_generated_filepc984rni.py\", line 40, in tf__run_call_with_unpacked_inputs\n        raise\n\n    TypeError: Exception encountered when calling layer 'tf_camembert_for_token_classification_5' (type TFCamembertForTokenClassification).\n    \n    in user code:\n    \n        File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/transformers/modeling_tf_utils.py\", line 1393, in run_call_with_unpacked_inputs  *\n            return func(self, **unpacked_inputs)\n    \n        TypeError: outer_factory.<locals>.inner_factory.<locals>.tf__call() got an unexpected keyword argument 'offset_mapping'\n    \n    \n    Call arguments received by layer 'tf_camembert_for_token_classification_5' (type TFCamembertForTokenClassification):\n      • input_ids={'input_ids': 'tf.Tensor(shape=(None, 150), dtype=int32)', 'attention_mask': 'tf.Tensor(shape=(None, 150), dtype=int32)', 'offset_mapping': 'tf.Tensor(shape=(None, 150, 2), dtype=int32)'}\n      • attention_mask=None\n      • token_type_ids=None\n      • position_ids=None\n      • head_mask=None\n      • inputs_embeds=None\n      • output_attentions=None\n      • output_hidden_states=None\n      • return_dict=None\n      • labels=None\n      • training=True\n"
+     ]
     }
    ],
    "source": [
-    "callback = tf.keras.callbacks.EarlyStopping(\n",
-    "    monitor=\"val_loss\", patience=0, restore_best_weights=True\n",
+    "early_stopping = tf.keras.callbacks.EarlyStopping(\n",
+    "    monitor=\"val_loss\", min_delta=0.001, patience=0, restore_best_weights=True\n",
     ")\n",
     "\n",
+    "csv_logger = tf.keras.callbacks.CSVLogger(\"training.log\")\n",
+    "\n",
     "camembert.fit(\n",
-    "    train_dataset, validation_data=test_dataset, epochs=4, callbacks=[callback]\n",
+    "    train_dataset,\n",
+    "    validation_data=test_dataset,\n",
+    "    epochs=10,\n",
+    "    callbacks=[early_stopping, csv_logger],\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 76,
    "metadata": {},
    "outputs": [
     {
@@ -405,7 +442,7 @@
        "<tf.Tensor: shape=(), dtype=float32, numpy=0.1186538115143776>"
       ]
      },
-     "execution_count": 54,
+     "execution_count": 76,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -421,26 +458,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 63,
+   "execution_count": 79,
    "metadata": {},
    "outputs": [],
    "source": [
-    "camembert.save_pretrained(\"./models/camembert\")"
+    "camembert.save_pretrained(\"./models/distilcamembert-base-ner-cross-entropy-11\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 78,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "tf_model.h5: 100%|██████████| 440M/440M [00:20<00:00, 21.8MB/s] \n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# camembert.push_to_hub(\"CamemBERT-NER-Travel\")"
    ]
@@ -462,7 +491,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.4"
+   "version": "3.12.8"
   }
  },
  "nbformat": 4,

requirements.txt

@@ -13,4 +13,6 @@ hmmlearn # for hidden markov models
 ipykernel
 tabulate
 transformers
-sentencepiece
+sentencepiece
+stanza
+pydot