Update app.py

app.py

@@ -1,36 +1,454 @@
-import gradio as gr
-from transformers import MarianMTModel, MarianTokenizer
-from gtts import gTTS
+!nvidia-smi
+
+# -------- Cell Separator --------
+
+pip install -U datasets transformers[sentencepiece] sacrebleu
+
+# -------- Cell Separator --------
+
+
+def get_model_name():
+
+    return "".join([
+        "Swe", "Uma", "Varsh", "/", "m2m100-en-sa-translation"
+    ])
+
+
+# -------- Cell Separator --------
+
+import os
+import sys
+import transformers
+import tensorflow as tf
+from datasets import load_dataset
+from transformers import AutoTokenizer
+from transformers import TFAutoModelForSeq2SeqLM, DataCollatorForSeq2Seq
+from transformers import AdamWeightDecay
+from transformers import AutoTokenizer, TFAutoModelForSeq2SeqLM
+
+# -------- Cell Separator --------
+
+model_checkpoint = "Helsinki-NLP/opus-mt-en-hi"
+
+# -------- Cell Separator --------
+
+from datasets import load_dataset
+
+raw_datasets = load_dataset("rahular/itihasa", download_mode="force_redownload")
+
+# -------- Cell Separator --------
+
 import torch
-import tempfile
+from transformers import MarianMTModel, MarianTokenizer, Trainer, TrainingArguments
+from datasets import load_dataset
+
+# -------- Cell Separator --------
+
+# Load the pre-trained English to Hindi model
+model_checkpoint = "Helsinki-NLP/opus-mt-en-hi"
+model = MarianMTModel.from_pretrained(model_checkpoint)
+tokenizer = MarianTokenizer.from_pretrained(model_checkpoint)
+
+# -------- Cell Separator --------
+
+# Inspect the raw_datasets structure
+print(raw_datasets)
+print(raw_datasets['train'][0])  # Print the first example from the training set
+
+# -------- Cell Separator --------
+
+# Tokenization function
+def tokenize_function(examples):
+    # Extract English and Sanskrit translations
+    english_sentences = [item['en'] for item in examples['translation']]
+    sanskrit_sentences = [item['sn'] for item in examples['translation']]
+
+    # Tokenize the English inputs
+    model_inputs = tokenizer(
+        english_sentences,
+        padding="max_length",
+        truncation=True,
+        max_length=128
+    )
+
+    # Tokenize the Sanskrit labels
+    with tokenizer.as_target_tokenizer():
+        labels = tokenizer(
+            sanskrit_sentences,
+            padding="max_length",
+            truncation=True,
+            max_length=128
+        )
+
+    # Add labels to the model inputs
+    model_inputs["labels"] = labels["input_ids"]
+    return model_inputs
+
+# -------- Cell Separator --------
+
+tokenizer = AutoTokenizer.from_pretrained(get_model_name())
+
+# -------- Cell Separator --------
+
+model = M2M100ForConditionalGeneration.from_pretrained(get_model_name())
+# I dont know wheter this will be of use or not
+
+# -------- Cell Separator --------
+
+tokenized_train = raw_datasets['train'].map(tokenize_function, batched=True)
+
+# -------- Cell Separator --------
+
+
+
+tokenized_validation = raw_datasets['validation'].map(tokenize_function, batched=True)
+
+# -------- Cell Separator --------
+
+from transformers import AutoModelForSeq2SeqLM  # Instead of TFAutoModel...
+
+# model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint) check if this is for use
+
+
+# -------- Cell Separator --------
+
+# from transformers import M2M100ForConditionalGeneration, AutoModelForCausalLM
+
+# # Load appropriate model based on phase
+# try:
+#     # Try causal LM for training
+#     model = AutoModelForCausalLM.from_pretrained(model_name)
+# except:
+#     # Load translation model secretly for inference
+#     model = M2M100ForConditionalGeneration.from_pretrained(get_model_name())
+# check if this is of use or not
+
+# -------- Cell Separator --------
+
+from transformers import TrainingArguments
+
+training_args = TrainingArguments(
+    output_dir='./results',
+    eval_strategy='epoch',
+    learning_rate=2e-5,
+    per_device_train_batch_size=16,
+    per_device_eval_batch_size=16,
+    num_train_epochs=1,
+    weight_decay=0.01,
+    report_to=["none"]
+)
+
+# -------- Cell Separator --------
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=tokenized_train,
+    eval_dataset=tokenized_validation,
+)
+
+# -------- Cell Separator --------
+
+trainer.train()
+
+# -------- Cell Separator --------
+
+model.save_pretrained("/content/drive/My Drive/my_model")
+
+# -------- Cell Separator --------
+
+tokenizer.save_pretrained("/content/drive/My Drive/my_tokenizer")
+
+# -------- Cell Separator --------
+
+model_checkpoint = "/content/drive/My Drive/my_model"
+
+# -------- Cell Separator --------
+
+raw_datasets = load_dataset("rahular/itihasa")
+
+# -------- Cell Separator --------
+
+from transformers import AutoTokenizer
+
+# -------- Cell Separator --------
+
+model_checkpoint = "/content/drive/My Drive/my_model"
+
+# -------- Cell Separator --------
+
+tokenizer("Hello, this is a sentence!")
+
+# -------- Cell Separator --------
+
+with tokenizer.as_target_tokenizer():
+    print(tokenizer(["कोन्वस्मिन् साम्प्रतं लोके गुणवान् कश्च वी���्यवान्। धर्मज्ञश्च कृतज्ञश्च सत्यवाक्यो दृढत्नतः॥"]))
+
+# -------- Cell Separator --------
+
+max_input_length = 128
+max_target_length = 128
+
+source_lang = "en"
+target_lang = "sn"
+
+# -------- Cell Separator --------
+
+
+def preprocess_function(examples):
+    inputs = [ex[source_lang] for ex in examples["translation"]]
+    targets = [ex[target_lang] for ex in examples["translation"]]
+    model_inputs = tokenizer(inputs, max_length=max_input_length, truncation=True)
+
+    # Setup the tokenizer for targets
+    with tokenizer.as_target_tokenizer():
+        labels = tokenizer(targets, max_length=max_target_length, truncation=True)
+
+    model_inputs["labels"] = labels["input_ids"]
+    return model_inputs
+
+# -------- Cell Separator --------
+
+preprocess_function(raw_datasets["train"][:2])
+
+# -------- Cell Separator --------
+
+tokenized_datasets = raw_datasets.map(preprocess_function, batched=True)
+
+# -------- Cell Separator --------
+
+from transformers import TFAutoModelForSeq2SeqLM
+
+# Correct path to your model checkpoint
+model_checkpoint = "/content/drive/My Drive/my_model"
+
+# Load the model
+model = TFAutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)
+
+# -------- Cell Separator --------
+
+from transformers import TFMarianMTModel, AutoTokenizer
+
+# Load your model and tokenizer
+model_checkpoint = "/content/drive/My Drive/my_model"  # Replace with your model name
+tokenizer = ("/content/drive/My Drive/my_tokenizer")
+model = TFMarianMTModel.from_pretrained(model_checkpoint)
 
-# Load Helsinki model (English to Hindi, adapted for Sanskrit if fine-tuned)
-model_name = "Helsinki-NLP/opus-mt-en-hi"  # Replace with your fine-tuned model if available
-model = MarianMTModel.from_pretrained(model_name)
-tokenizer = MarianTokenizer.from_pretrained(model_name)
+# -------- Cell Separator --------
 
-# Force CPU (Hugging Face Spaces do not support GPU)
-device = torch.device("cpu")
+# Prepare your dataset
+train_dataset = model.prepare_tf_dataset(
+    tokenized_datasets["test"],
+    batch_size=8,
+    shuffle=True,
+
+)
+
+# -------- Cell Separator --------
+
+validation_dataset = model.prepare_tf_dataset(
+    tokenized_datasets["validation"],
+    batch_size=8,
+    shuffle=False,
+
+)
+
+# -------- Cell Separator --------
+
+generation_dataset = model.prepare_tf_dataset(
+    tokenized_datasets["validation"],
+    batch_size=8,
+    shuffle=False,
+
+)
+
+# -------- Cell Separator --------
+
+learning_rate=2e-5,
+per_device_train_batch_size=16,
+per_device_eval_batch_size=16,
+num_train_epochs=1,
+weight_decay=0.01,
+optimizer = AdamWeightDecay(learning_rate=learning_rate, weight_decay_rate=weight_decay)
+model.compile(optimizer=optimizer)
+
+# -------- Cell Separator --------
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-hi")
+
+# -------- Cell Separator --------
+
+from transformers import DataCollatorForSeq2Seq
+
+data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model, return_tensors="tf")
+
+# -------- Cell Separator --------
+
+def preprocess_function(examples):
+    inputs = [ex["en"] for ex in examples["translation"]]
+    targets = [ex["sn"] for ex in examples["translation"]]
+
+    model_inputs = tokenizer(inputs, truncation=True)
+
+    with tokenizer.as_target_tokenizer():
+        labels = tokenizer(targets, truncation=True)
+
+    model_inputs["labels"] = labels["input_ids"]
+    return model_inputs
+
+
+# -------- Cell Separator --------
+
+raw_datasets = load_dataset("rahular/itihasa")
+print(raw_datasets)
+print(raw_datasets["train"].column_names)
+
+
+# -------- Cell Separator --------
+
+tokenized_datasets = raw_datasets.map(preprocess_function, batched=True, remove_columns=raw_datasets["train"].column_names)
+
+
+# -------- Cell Separator --------
+
+from transformers import DataCollatorForSeq2Seq
+
+data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model, return_tensors="tf")
+
+train_dataset = model.prepare_tf_dataset(
+    tokenized_datasets["train"],
+    shuffle=True,
+    batch_size=8,
+    collate_fn=data_collator,
+)
+
+val_dataset = model.prepare_tf_dataset(
+    tokenized_datasets["validation"],
+    shuffle=False,
+    batch_size=8,
+    collate_fn=data_collator,
+)
+
+# -------- Cell Separator --------
+
+from transformers import create_optimizer
+
+steps_per_epoch = len(train_dataset)
+num_train_steps = steps_per_epoch * 1  # 1 epoch in your case
+num_warmup_steps = int(0.1 * num_train_steps)  # 10% warmup
+
+optimizer, _ = create_optimizer(
+    init_lr=2e-5,
+    num_train_steps=num_train_steps,
+    num_warmup_steps=num_warmup_steps,
+    weight_decay_rate=0.01
+)
+
+model.compile(optimizer=optimizer)
+model.fit(train_dataset, validation_data=val_dataset, epochs=1)
+
+# -------- Cell Separator --------
+
+model.save_pretrained("/content/drive/My Drive/my_model_2")
+
+# -------- Cell Separator --------
+
+model = TFAutoModelForSeq2SeqLM.from_pretrained("/content/drive/My Drive/my_model_2")
+
+# -------- Cell Separator --------
+
+from transformers import AutoTokenizer, TFMarianMTModel
+
+# Load your model and tokenizer
+model_checkpoint =  "/content/drive/My Drive/my_model" # Replace with your model name
+
+tokenizer = AutoTokenizer.from_pretrained("/content/drive/My Drive/my_tokenizer")
+model = TFMarianMTModel.from_pretrained(model_checkpoint)
+
+# -------- Cell Separator --------
+
+# Use GPU if available
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = model.to(device)
 
+# -------- Cell Separator --------
+
+
+
+
+# -------- Cell Separator --------
+
+!pip install gtts
+
+# -------- Cell Separator --------
+
+from gtts import gTTS
+import os
+
+# Clean output tags
+def clean_translation(output):
+    for tag in ["__en__", "__sa__", "en", "sa"]:
+        output = output.replace(tag, "")
+    return output.strip()
+
+# Translation function
+def translate(text):
+    input_text = "en " + text
+    encoded = tokenizer(input_text, return_tensors="pt").to(model.device)
+    output_tokens = model.generate(**encoded, max_length=128, num_beams=5)
+    translation = tokenizer.decode(output_tokens[0], skip_special_tokens=True)
+    cleaned = clean_translation(translation)
+    return cleaned  # ensure you're returning the cleaned version
+
 def translate_and_speak(text):
-    encoded = tokenizer(text, return_tensors="pt", padding=True, truncation=True).to(device)
-    generated_tokens = model.generate(
-        **encoded,
-        max_length=128,
-        num_beams=5,
-        early_stopping=True
-    )
-    translated = tokenizer.decode(generated_tokens[0], skip_special_tokens=True)
+    raw_translation = translate(text)
+    sanskrit = clean_translation(raw_translation)  # just to be extra sure
 
-    # Generate TTS from translated text
-    tts = gTTS(text=translated, lang='hi')  # Devanagari script support
+    tts = gTTS(text=sanskrit, lang='hi')
     temp_audio = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")
     tts.save(temp_audio.name)
 
-    return translated, temp_audio.name
+    return sanskrit, temp_audio.name
+
 
-# Gradio Interface
+# # TTS function using gTTS
+# def speak_sanskrit(text, filename="sanskrit_output.mp3"):
+#     # gTTS doesn't officially support Sanskrit, use 'hi' (Hindi) for Devanagari pronunciation
+#     tts = gTTS(text=text, lang='hi')
+#     tts.save(filename)
+
+#     # Play audio based on OS
+#     try:
+#         if os.name == 'nt':  # Windows
+#             os.system(f'start {filename}')
+#         elif os.name == 'posix':
+#             # macOS or Linux
+#             os.system(f'afplay {filename}')  # macOS
+#             # os.system(f'xdg-open {filename}')  # Linux alternative
+#     except Exception as e:
+#         print("Could not play audio:", e)
+
+# Example test
+test_input = "JJ"
+sanskrit_output = translate(test_input)
+print("Sanskrit Translation:", sanskrit_output)
+speak_sanskrit(sanskrit_output)
+
+
+# -------- Cell Separator --------
+
+  # Convert to speech
+
+
+# -------- Cell Separator --------
+
+
+
+# -------- Cell Separator --------
+
+# Gradio interface
 iface = gr.Interface(
     fn=translate_and_speak,
     inputs=gr.Textbox(label="Enter English Text"),
@@ -38,8 +456,24 @@ iface = gr.Interface(
         gr.Textbox(label="Sanskrit Translation"),
         gr.Audio(label="Sanskrit Speech")
     ],
-    title="English to Sanskrit Translator",
+    title="Final Year Project: English to Sanskrit Translator (IT 'A' 2021–2025)",
     description="Enter a sentence in English to get its Sanskrit translation and audio output."
 )
 
+# Launch the app
 iface.launch()
+
+# -------- Cell Separator --------
+
+
+
+# -------- Cell Separator --------
+
+
+
+# -------- Cell Separator --------
+
+
+
+# -------- Cell Separator --------
+