Create t5-base-qg-hl finetuned

valhalla/t5-base-qg-hl finetuned

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""Untitled6.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/11megvyfcr49Oy4FGK7kteQ2iMdxZYp4L
+"""
+
+pip install transformers datasets sentence-transformers evaluate scikit-learn
+
+from google.colab import files
+uploaded = files.upload()
+
+from google.colab import files
+uploaded = files.upload()
+
+from transformers import T5Tokenizer, T5ForConditionalGeneration, Trainer, TrainingArguments, DataCollatorForSeq2Seq
+from datasets import load_dataset, DatasetDict, load_metric
+from sentence_transformers import SentenceTransformer, util
+from sklearn.metrics.pairwise import cosine_similarity
+import pandas as pd
+import torch
+import numpy as np
+
+def load_csv_datasets(train_path, eval_path):
+    train_df = pd.read_csv(train_path)
+    eval_df = pd.read_csv(eval_path)
+
+    dataset = DatasetDict({
+        'train': Dataset.from_pandas(train_df),
+        'eval': Dataset.from_pandas(eval_df)
+    })
+    return dataset
+
+def preprocess(example):
+    input_text = example['input']
+    target_text = example['target']
+    model_inputs = tokenizer(input_text, max_length=512, padding='max_length', truncation=True)
+    labels = tokenizer(target_text, max_length=64, padding='max_length', truncation=True)
+    model_inputs["labels"] = labels["input_ids"]
+    return model_inputs
+
+model_name = "valhalla/t5-base-qg-hl"
+tokenizer = T5Tokenizer.from_pretrained(model_name)
+model = T5ForConditionalGeneration.from_pretrained(model_name)
+
+dataset = load_csv_datasets("train.csv", "eval.csv")
+tokenized_dataset = dataset.map(preprocess, batched=True)
+data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)
+
+training_args = TrainingArguments(
+    output_dir="./qg_finetuned",
+    eval_strategy="epoch",
+    save_strategy="epoch",
+    per_device_train_batch_size=4,
+    per_device_eval_batch_size=4,
+    num_train_epochs=3,
+    logging_dir='./logs',
+    logging_steps=10,
+    save_total_limit=1,
+    load_best_model_at_end=True,
+    metric_for_best_model="cosine",
+    greater_is_better=True
+)
+
+def compute_metrics(eval_pred):
+    predictions, labels = eval_pred
+    # Ensure predictions is a list of lists of integers for batch_decode
+    if isinstance(predictions, tuple):
+      predictions = predictions[0]
+
+    # Replace -100 in labels as we can't decode them.
+    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
+
+    # Filter out invalid token IDs from predictions
+    valid_vocab_size = tokenizer.vocab_size
+    filtered_predictions = []
+    for pred_seq in predictions.tolist():
+        filtered_seq = [token_id for token_id in pred_seq[0] if 0 <= token_id < valid_vocab_size]
+        filtered_predictions.append(filtered_seq)
+
+
+    decoded_preds = tokenizer.batch_decode(filtered_predictions, skip_special_tokens=True)
+    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
+
+    # Using SentenceTransformer for cosine similarity
+    embedder = SentenceTransformer('all-MiniLM-L6-v2')
+    embeddings_pred = embedder.encode(decoded_preds, convert_to_tensor=True)
+    embeddings_label = embedder.encode(decoded_labels, convert_to_tensor=True)
+
+    cosine_scores = util.cos_sim(embeddings_pred, embeddings_label).diagonal()
+    avg_cosine = cosine_scores.mean().item()
+
+    return {"cosine": avg_cosine}
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=tokenized_dataset["train"],
+    eval_dataset=tokenized_dataset["eval"],
+    tokenizer=tokenizer,
+    data_collator=data_collator,
+    compute_metrics=compute_metrics,
+)
+trainer.train()
+
+results = trainer.evaluate()
+print("Evaluation Results:", results)