Create app.py

app.py

@@ -0,0 +1,141 @@
+# train.py
+#!/usr/bin/env python
+import os
+import json
+import string
+import pandas as pd
+import evaluate
+import numpy as np
+from datasets import load_dataset, DatasetDict
+from transformers import (
+    AutoTokenizer, AutoModelForSeq2SeqLM,
+    Seq2SeqTrainingArguments, Seq2SeqTrainer,
+    DataCollatorForSeq2Seq
+)
+from huggingface_hub import login
+
+# -------------------------------------------------
+# 0. HF login (set HF_TOKEN in Secrets)
+# -------------------------------------------------
+login()   # reads HF_TOKEN from environment
+
+# -------------------------------------------------
+# 1. Load dataset from Hub
+# -------------------------------------------------
+dataset = load_dataset("your-username/celtic-parallel")
+data = json.loads(dataset["train"][0]["parallel_corpus.json"])  # dummy – we load the file directly
+# Actually we load the JSON file that was uploaded:
+raw = load_dataset("your-username/celtic-parallel", data_files="parallel_corpus.json")["train"]
+df = pd.DataFrame(json.loads(raw[0]["parallel_corpus.json"]))
+
+# -------------------------------------------------
+# 2. Build English → {br, abk, cy}
+# -------------------------------------------------
+def is_valid(t):
+    return bool(t and t.strip() and t.strip() not in string.punctuation)
+
+br = df[df.apply(lambda r: is_valid(r["niv_text"]) and is_valid(r["koad21_text"]), axis=1)][["niv_text","koad21_text"]].rename(columns={"niv_text":"en","koad21_text":"target"})
+br["language"] = "br"
+
+abk = df[df.apply(lambda r: is_valid(r["niv_text"]) and is_valid(r["abk_text"]), axis=1)][["niv_text","abk_text"]].rename(columns={"niv_text":"en","abk_text":"target"})
+abk["language"] = "abk"
+
+cy = df[df.apply(lambda r: is_valid(r["niv_text"]) and is_valid(r["bcnda_text"]), axis=1)][["niv_text","bcnda_text"]].rename(columns={"niv_text":"en","bcnda_text":"target"})
+cy["language"] = "cy"
+
+combined = pd.concat([br, abk, cy], ignore_index=True)
+print(f"Total examples: {len(combined)} (br:{len(br)}, abk:{len(abk)}, cy:{len(cy)})")
+
+# -------------------------------------------------
+# 3. Train / test split
+# -------------------------------------------------
+from datasets import Dataset
+ds = Dataset.from_pandas(combined).train_test_split(test_size=0.2, seed=42)
+raw_datasets = DatasetDict({"train": ds["train"], "test": ds["test"]})
+
+# -------------------------------------------------
+# 4. Tokenizer & Model
+# -------------------------------------------------
+model_name = "google/mt5-small"
+tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)
+model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+
+# -------------------------------------------------
+# 5. Pre-process
+# -------------------------------------------------
+MAX_LEN = 96
+
+def preprocess(examples):
+    inputs = [f"translate English to {lang}: {en}"
+              for lang, en in zip(examples["language"], examples["en"])]
+    targets = examples["target"]
+    model_inputs = tokenizer(inputs, max_length=MAX_LEN, truncation=True, padding="max_length")
+    labels = tokenizer(targets, max_length=MAX_LEN, truncation=True, padding="max_length").input_ids
+    model_inputs["labels"] = labels
+    return model_inputs
+
+tokenized = raw_datasets.map(preprocess, batched=True, remove_columns=raw_datasets["train"].column_names)
+
+# -------------------------------------------------
+# 6. Data collator & metric
+# -------------------------------------------------
+data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
+
+metric = evaluate.load("sacrebleu")
+
+def compute_metrics(eval_preds):
+    preds, labels = eval_preds
+    if isinstance(preds, tuple):
+        preds = preds[0]
+    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
+    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
+    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
+    decoded_preds = [p.strip() for p in decoded_preds]
+    decoded_labels = [[l.strip()] for l in decoded_labels]
+    result = metric.compute(predictions=decoded_preds, references=decoded_labels)
+    return {"bleu": result["score"]}
+
+# -------------------------------------------------
+# 7. Training args
+# -------------------------------------------------
+training_args = Seq2SeqTrainingArguments(
+    output_dir="mt5-celtic-finetuned",
+    eval_strategy="epoch",
+    save_strategy="epoch",
+    learning_rate=3e-4,
+    per_device_train_batch_size=16,
+    per_device_eval_batch_size=16,
+    weight_decay=0.01,
+    num_train_epochs=3,
+    predict_with_generate=True,
+    fp16=True,          # GPU
+    bf16=True,          # TPU (auto-enabled if on TPU)
+    logging_steps=100,
+    report_to="wandb",  # optional
+    push_to_hub=True,
+    hub_model_id="your-username/mt5-celtic-en-br-abk-cy",
+    hub_strategy="end",
+    load_best_model_at_end=True,
+    metric_for_best_model="bleu",
+)
+
+# -------------------------------------------------
+# 8. Trainer
+# -------------------------------------------------
+trainer = Seq2SeqTrainer(
+    model=model,
+    args=training_args,
+    train_dataset=tokenized["train"],
+    eval_dataset=tokenized["test"],
+    tokenizer=tokenizer,
+    data_collator=data_collator,
+    compute_metrics=compute_metrics,
+)
+
+trainer.train()
+
+# -------------------------------------------------
+# 9. Final push
+# -------------------------------------------------
+trainer.push_to_hub("mt5-celtic-en-br-abk-cy")
+print("Model pushed to Hub!")