app.py

import os
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, Dataset
from torch.optim import AdamW
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import time
from tqdm import tqdm
from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments
from diffusers import DiffusionPipeline
from huggingface_hub import login, HfApi, Repository
from dotenv import load_dotenv
import gradio as gr

# Cargar variables de entorno
load_dotenv()

class UnifiedModel(nn.Module):
    def __init__(self, models):
        super(UnifiedModel, self).__init__()
        self.models = nn.ModuleList(models)
        self.classifier = nn.Linear(sum([model.config.hidden_size for model in models if hasattr(model, 'config')]), 2)

    def forward(self, inputs):
        hidden_states = []
        for model in self.models:
            if isinstance(model, nn.Module):
                outputs = model(**inputs)
                hidden_states.append(outputs.last_hidden_state[:, 0, :])
            elif isinstance(model, DiffusionPipeline):
                outputs = model(**inputs)
                hidden_states.append(torch.tensor(outputs).float())
        concatenated_hidden_states = torch.cat(hidden_states, dim=-1)
        logits = self.classifier(concatenated_hidden_states)
        return logits

class SyntheticDataset(Dataset):
    def __init__(self, tokenizers, size=100):
        self.tokenizers = tokenizers
        self.size = size
        self.data = self._generate_data()

    def _generate_data(self):
        data = []
        for _ in range(self.size):
            text = "This is a sample sentence for testing purposes."
            label = torch.tensor(0)  # Sample label
            item = {"text": text, "label": label}
            for name, tokenizer in self.tokenizers.items():
                tokenized = tokenizer(text, padding="max_length", truncation=True, max_length=128)
                item[f"input_ids_{name}"] = torch.tensor(tokenized["input_ids"])
                item[f"attention_mask_{name}"] = torch.tensor(tokenized["attention_mask"])
            data.append(item)
        return data

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        return self.data[idx]

def push_to_hub(local_dir, repo_name):
    try:
        repo_url = HfApi().create_repo(repo_name, exist_ok=True)
        repo = Repository(local_dir, clone_from=repo_url)

        if not os.path.exists(os.path.join(local_dir, ".git")):
            os.system(f"cd {local_dir} && git init && git remote add origin {repo_url} && git pull origin main")

        repo.git_add(auto_lfs_track=True)
        repo.git_commit("Add model and tokenizer files")

        json_files = ["config.json", "generation_config.json", "special_tokens_map.json", "tokenizer.json", "tokenizer.model", "tokenizer_config.json"]
        for json_file in json_files:
            json_file_path = os.path.join(local_dir, json_file)
            if os.path.exists(json_file_path):
                repo.git_add(json_file_path)

        repo.git_push()
        print(f"Pushed model and tokenizer to {repo_url}")
    except Exception as e:
        print(f"Error pushing to Hugging Face Hub: {e}")

def load_model(model_name):
    tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
    model = AutoModelForCausalLM.from_pretrained(model_name)
    return tokenizer, model

def train(model, train_loader, eval_loader, args):
    model.train()
    epoch = 0
    total_steps = len(train_loader)
    for step, batch in enumerate(train_loader):
        start_time = time.time()
        input_ids = [batch[f"input_ids_{name}"].to("cpu") for name in tokenizers.keys()]
        attention_mask = [batch[f"attention_mask_{name}"].to("cpu") for name in tokenizers.keys()]
        labels = batch["label"].to("cpu")
        optimizer.zero_grad()
        outputs = model(input_ids)
        loss = nn.CrossEntropyLoss()(outputs, labels)
        loss.backward()
        optimizer.step()

        elapsed_time = time.time() - start_time
        estimated_total_time = total_steps * (elapsed_time / (step + 1))
        estimated_remaining_time = estimated_total_time - elapsed_time

        if step % args.logging_steps == 0:
            train_losses.append(loss.item())
            print(f"Step {step}/{total_steps}, Loss: {loss.item()}, Estimated remaining time: {estimated_remaining_time:.2f} seconds")

    epoch += 1
    model.eval()
    eval_loss = 0
    with torch.no_grad():
        for batch in eval_loader:
            input_ids = [batch[f"input_ids_{name}"].to("cpu") for name in tokenizers.keys()]
            attention_mask = [batch[f"attention_mask_{name}"].to("cpu") for name in tokenizers.keys()]
            labels = batch["label"].to("cpu")
            outputs = model(input_ids)
            loss = nn.CrossEntropyLoss()(outputs, labels)
            eval_loss += loss.item()

    eval_loss /= len(eval_loader)
    eval_losses.append(eval_loss)
    print(f"Epoch {epoch}/{args.num_train_epochs}, Evaluation Loss: {eval_loss}")

def gradio_interface(input_text):
    # Define the Gradio interface function
    tokenized_inputs = {name: tokenizer.encode(input_text, return_tensors="pt") for name, tokenizer in tokenizers.items()}
    model_output = unified_model(tokenized_inputs)
    return model_output

def main():
    while True:
        try:
            os.system("git config --global credential.helper store")
            login(token=os.getenv("HUGGINGFACE_TOKEN"), add_to_git_credential=True)

            # Definir los modelos que se van a utilizar
            models_to_train = [
                "openai-community/gpt2-xl",
                "google/gemma-2-9b-it",
                "google/gemma-2-9b",
                "meta-llama/Meta-Llama-3.1-8B-Instruct",
                "meta-llama/Meta-Llama-3.1-8B",
                "openbmb/MiniCPM-V-2_6",
                "bigcode/starcoder",
                "WizardLMTeam/WizardCoder-Python-34B-V1.0",
                "Qwen/Qwen2-72B-Instruct",
                "google/gemma-2-2b-it",
                "facebook/bart-large-cnn",
                "Falconsai/text_summarization",
                "microsoft/speecht5_tts",
                "Groq/Llama-3-Groq-70B-Tool-Use",
                "Groq/Llama-3-Groq-8B-Tool-Use",
                "facebook/musicgen-large",
                "facebook/musicgen-melody",
                "black-forest-labs/FLUX.1-schnell",
                "facebook/musicgen-small",
                "stabilityai/stable-video-diffusion-img2vid-xt-1-1",
                "openai/whisper-small",
                "black-forest-labs/FLUX.1-dev",
                "stabilityai/stable-diffusion-2-1"
            ]

            # Inicializar los modelos y tokenizadores
            tokenizers = {}
            models = []
            for model_name in models_to_train:
                tokenizer, model = load_model(model_name)
                tokenizers[model_name] = tokenizer
                models.append(model)

            # Crear un dataset sintético para entrenamiento y evaluación
            synthetic_dataset = SyntheticDataset(tokenizers, size=100)

            # Dividir el dataset en entrenamiento y evaluación
            train_size = int(0.8 * len(synthetic_dataset))
            val_size = len(synthetic_dataset) - train_size
            train_dataset, val_dataset = torch.utils.data.random_split(synthetic_dataset, [train_size, val_size])

            # Crear DataLoaders para entrenamiento y evaluación
            train_loader = DataLoader(train_dataset, batch_size=2, shuffle=True)
            eval_loader = DataLoader(val_dataset, batch_size=16)

            # Unificar los modelos en uno solo
            unified_model = UnifiedModel(models)
            unified_model.to(torch.device("cpu"))

            # Mostrar la cantidad de parámetros totales a entrenar
            total_params = sum(p.numel() for p in unified_model.parameters())
            print(f"Total parameters to train: {total_params}")

            # Definir los argumentos de entrenamiento
            training_args = TrainingArguments(
                per_device_train_batch_size=2,
                per_device_eval_batch_size=16,
                num_train_epochs=1,
                logging_steps=10,
                save_steps=10,
                evaluation_strategy="steps"
            )

            # Definir el optimizador
            optimizer = AdamW(unified_model.parameters(), lr=5e-5)

            # Listas para almacenar las pérdidas
            train_losses = []
            eval_losses = []

            # Entrenar el modelo
            train(unified_model, train_loader, eval_loader, training_args)

            # Visualizar pérdidas
            fig, ax = plt.subplots()
            ax.set_xlabel("Epochs")
            ax.set_ylabel("Loss")
            ax.plot(train_losses, label="Training Loss")
            ax.plot(eval_losses, label="Evaluation Loss")
            ax.legend()

            def animate(i):
                ax.clear()
                ax.plot(train_losses, label="Training Loss")
                ax.plot(eval_losses, label="Evaluation Loss")
                ax.set_xlabel("Epochs")
                ax.set_ylabel("Loss")
                ax.legend()

            ani = animation.FuncAnimation(fig, animate, interval=1000)
            plt.show()

            # Guardar el modelo y el tokenizador unificados
            if not os.path.exists("./outputs/unified_model"):
                os.makedirs("./outputs/unified_model")

            # Guardar el modelo unificado en un directorio local
            local_dir = "./outputs/unified_model"
            torch.save(unified_model.state_dict(), os.path.join(local_dir, "pytorch_model.bin"))

            # Guardar el tokenizador en un directorio local
            for name, tokenizer in tokenizers.items():
                tokenizer.save_pretrained(local_dir)

            # Subir el modelo y el tokenizador a Hugging Face
            push_to_hub(local_dir, repo_name="Ffftdtd5dtft/my_model")

            # Configurar y lanzar la interfaz Gradio
            interface = gr.Interface(fn=gradio_interface, inputs="text", outputs="text")
            interface.launch()

            break
        except Exception as e:
            print(f"Error: {e}")
            time.sleep(2)

if __name__ == "__main__":
    main()