![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}
"
]
},
{
"cell_type": "markdown",
"source": [
"# Fine-Tuning mT5 for English → German Translation\n",
"\n",
"This notebook fine-tunes **`google/mt5-small`** on the **WMT16 En→De** dataset using Hugging Face Transformers.\n",
"\n",
"- Model: Multilingual T5 (mT5) – pre-trained on 101 languages **without supervised translation**\n",
"- Task: Teach it **high-quality English to German translation** via fine-tuning\n",
"- Dataset: WMT16 (via `datasets` library)\n",
"- Framework: `transformers` + `seq2seq` Trainer\n",
"\n",
"---"
],
"metadata": {
"id": "XnxaVxCe8gjQ"
}
},
{
"cell_type": "markdown",
"source": [
"## 1. Install Dependencies"
],
"metadata": {
"id": "P05w2JE68gjR"
}
},
{
"cell_type": "code",
"source": [
"from google.colab import drive\n",
"drive.mount('/content/drive')\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "3-EOS45xALKJ",
"outputId": "7b790ea5-c852-4688-9189-91a725c6be72"
},
"execution_count": 1,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Mounted at /content/drive\n"
]
}
]
},
{
"cell_type": "code",
"execution_count": 2,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.8/51.8 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m104.1/104.1 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h"
]
}
],
"source": [
"!pip install -q transformers datasets sentencepiece sacrebleu accelerate evaluate\n",
"!pip install -q torch --index-url https://download.pytorch.org/whl/cu118"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "tBzChq5L8gjR",
"outputId": "264ac5a0-a9ef-4f49-a5be-6045a15181f6"
}
},
{
"cell_type": "code",
"source": [
"!pip install pandas"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "1O0iGKJ7A6yW",
"outputId": "4194b26b-9090-4444-866f-b86e7058179b"
},
"execution_count": 5,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Requirement already satisfied: pandas in /usr/local/lib/python3.12/dist-packages (2.2.2)\n",
"Requirement already satisfied: numpy>=1.26.0 in /usr/local/lib/python3.12/dist-packages (from pandas) (2.0.2)\n",
"Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas) (2.9.0.post0)\n",
"Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas) (2025.2)\n",
"Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas) (2025.2)\n",
"Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## 2. Load Dataset (WMT16 En→De)"
],
"metadata": {
"id": "_niqDW2e8gjS"
}
},
{
"cell_type": "code",
"source": [
"import gdown\n",
"import json\n",
"\n",
"data_path = \"/content/drive/MyDrive/llm-translator/parallel_corpus.json\" # Adjust path\n",
"try:\n",
" with open(data_path, \"r\", encoding=\"utf-8\") as file:\n",
" data = json.load(file)\n",
" print(f\"Loaded {len(data)} entries from parallel_corpus.json\")\n",
"except FileNotFoundError:\n",
" print(f\"Error: The file '{data_path}' was not found.\")\n",
" exit(1)\n",
"except json.JSONDecodeError:\n",
" print(\"Error: Failed to decode JSON from the file.\")\n",
" exit(1)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "ds1obOiM8gjT",
"outputId": "25f14e99-d3fb-491b-c9c8-7807e8d2b6f5"
},
"execution_count": 3,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Loaded 31192 entries from parallel_corpus.json\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"import string\n",
"import pandas as pd\n",
"from datasets import Dataset, DatasetDict\n",
"\n",
"\n",
"# Build the three language pairs exactly as you did\n",
"def is_valid(t): return bool(t and t.strip() and t.strip() not in string.punctuation)\n",
"df = pd.DataFrame(data)\n",
"breton_df = df[df.apply(lambda r: is_valid(r[\"niv_text\"]) and is_valid(r[\"koad21_text\"]), axis=1)][[\"niv_text\",\"koad21_text\"]].rename(columns={\"niv_text\":\"en\",\"koad21_text\":\"target\"})\n",
"breton_df[\"language\"] = \"br\"\n",
"cornish_df = df[df.apply(lambda r: is_valid(r[\"niv_text\"]) and is_valid(r[\"abk_text\"]), axis=1)][[\"niv_text\",\"abk_text\"]].rename(columns={\"niv_text\":\"en\",\"abk_text\":\"target\"})\n",
"cornish_df[\"language\"] = \"abk\"\n",
"welsh_df = df[df.apply(lambda r: is_valid(r[\"niv_text\"]) and is_valid(r[\"bcnda_text\"]), axis=1)][[\"niv_text\",\"bcnda_text\"]].rename(columns={\"niv_text\":\"en\",\"bcnda_text\":\"target\"})\n",
"welsh_df[\"language\"] = \"cy\"\n",
"\n",
"\n",
"combined_df = pd.concat([breton_df, cornish_df, welsh_df], ignore_index=True)\n",
"print(combined_df.head(5))\n",
"dataset = Dataset.from_pandas(combined_df).train_test_split(test_size=0.2, seed=42)\n",
"print(f\"Combined dataset size: {len(combined_df)} pairs (Breton: {len(breton_df)}, Cornish: {len(cornish_df)}, Welsh: {len(welsh_df)})\")\n",
"\n",
"raw_datasets = DatasetDict({\n",
" \"train\": dataset[\"train\"],\n",
" \"test\" : dataset[\"test\"]\n",
"})\n",
"print(f\"Train: {len(raw_datasets['train'])}, Test: {len(raw_datasets['test'])}\")\n",
"print(raw_datasets['train'])"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "jbv3mVHMAdZT",
"outputId": "f42a3f59-4be9-46f3-fc9d-80f4614cbf3f"
},
"execution_count": 10,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
" en \\\n",
"0 The Lord called to Moses and spoke to him from... \n",
"1 “Speak to the Israelites and say to them: ‘Whe... \n",
"2 “ ‘If the offering is a burnt offering from th... \n",
"3 You are to lay your hand on the head of the bu... \n",
"4 You are to slaughter the young bull before the... \n",
"\n",
" target language \n",
"0 An AOTROU a c’halvas Moizez hag a gomzas dezha... br \n",
"1 Komz da vibien Israel ha lavar: Pa raio unan b... br \n",
"2 Mar d-eo e brof ul loskaberzh a loened bras, e... br \n",
"3 Lakaat a raio e zorn war benn al loskaberzh, a... br \n",
"4 Lazhañ a raio ar c’hole dirak an AOTROU ; an a... br \n",
"Combined dataset size: 93233 pairs (Breton: 31077, Cornish: 31086, Welsh: 31070)\n",
"Train: 74586, Test: 18647\n",
"Dataset({\n",
" features: ['en', 'target', 'language'],\n",
" num_rows: 74586\n",
"})\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## 3. Load Model & Tokenizer"
],
"metadata": {
"id": "920INjs88gjT"
}
},
{
"cell_type": "code",
"source": [
"from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n",
"\n",
"model_name = \"t5-small\"\n",
"tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)\n",
"model = AutoModelForSeq2SeqLM.from_pretrained(model_name)\n",
"\n",
"# mT5 uses SentencePiece – no fast tokenizer available"
],
"metadata": {
"id": "7XvJ2Sl38gjT"
},
"execution_count": 11,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## 4. Preprocess Function"
],
"metadata": {
"id": "H-cPZfJj8gjU"
}
},
{
"cell_type": "code",
"source": [
"max_input_length = 128\n",
"max_target_length = 128\n",
"\n",
"def preprocess(examples):\n",
" inputs = [f\"translate English to {lang}: {en}\"\n",
" for lang, en in zip(examples[\"language\"], examples[\"en\"])]\n",
" targets = examples[\"target\"]\n",
" model_inputs = tokenizer(inputs, max_length=max_input_length,\n",
" truncation=True, padding=\"max_length\")\n",
" labels = tokenizer(targets, max_length=max_target_length,\n",
" truncation=True, padding=\"max_length\").input_ids\n",
" model_inputs[\"labels\"] = labels\n",
" return model_inputs\n",
"\n"
],
"metadata": {
"id": "eY7-UjaE8gjU"
},
"execution_count": 13,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# Apply preprocessing\n",
"print(\"Tokenising …\")\n",
"print(raw_datasets)\n",
"tokenized_datasets = raw_datasets.map(preprocess,\n",
" batched=True,\n",
" remove_columns=raw_datasets[\"train\"].column_names)\n",
"\n",
"print(tokenized_datasets)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 456,
"referenced_widgets": [
"8a49fb7918744d23bf64237c2674fc6b",
"7a64e72d88084b6897d53c344aad4358",
"85b34cfea65c4a8dbdae30a8eec22a98",
"500b7fb2607c4eebbb5bea0bfc4e33ff",
"b53f4a68898e45098d6801e4987e4c10",
"008567f59c13464b82ca3946ccf0058c",
"6ac91c5649bc4b6fa0d3860e71cdb01b",
"90bfdbd03ac44d5ca3df23ebcc02d863",
"6618b65302df46098bd2ad029ed668eb",
"a04d20d2b4934f35a7890f24546023d7",
"5f5df2985c6f49d3816888ec4ebd9add",
"419e345a9ff34dce965af89ad6569ff1",
"2f1473b2871b49f5b916298ab9866bd8",
"171e066c98944ad2abcad3f61d11144c",
"67c666a7cc8e4ae5a5c6c7442fdb8484",
"3eb3a35b6b2b414fa725a49e002da0e4",
"12e03166c7854af59078eb601478338a",
"9310394718a64134b5d17290db28f880",
"c851e5be2363465981fa92285c6a9579",
"3d2bdb8969cd45499c33a49ab73cab39",
"5e584348fea147fb95649323af691ea9",
"20af10029189476d8f84badf52743b26"
]
},
"id": "HVO9qHXZ8gjV",
"outputId": "0a5edd95-149e-4596-abc3-2668b500f7b1"
},
"execution_count": 14,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Tokenising …\n",
"DatasetDict({\n",
" train: Dataset({\n",
" features: ['en', 'target', 'language'],\n",
" num_rows: 74586\n",
" })\n",
" test: Dataset({\n",
" features: ['en', 'target', 'language'],\n",
" num_rows: 18647\n",
" })\n",
"})\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"Map: 0%| | 0/74586 [00:00"
],
"text/html": [
"\n",
" | Epoch | \n", "Training Loss | \n", "Validation Loss | \n", "
|---|
"
]
},
"metadata": {}
},
{
"output_type": "error",
"ename": "KeyboardInterrupt",
"evalue": "",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m/tmp/ipython-input-4032920361.py\u001b[0m in \u001b[0;36m