Update app.py

app.py

@@ -1,158 +1,62 @@
-# train.py
-#!/usr/bin/env python
-import os
-import json
-import string
-import pandas as pd
-import evaluate
-import numpy as np
-from datasets import load_dataset, DatasetDict
-from transformers import (
-    AutoTokenizer, AutoModelForSeq2SeqLM,
-    Seq2SeqTrainingArguments, Seq2SeqTrainer,
-    DataCollatorForSeq2Seq
-)
-from huggingface_hub import login
-
-# -------------------------------------------------
-# 0. HF login (set HF_TOKEN in Secrets)
-# -------------------------------------------------
+# 1. Install
+!pip install -q pandas datasets huggingface_hub
 
-if "HF_TOKEN" in os.environ:
-    login(token=os.environ["HF_TOKEN"])
-    print("Logged in to Hugging Face via HF_TOKEN")
-else:
-    raise ValueError("HF_TOKEN not found! Add it in Space Secrets."
-)
-
-# -------------------------------------------------
-# 1. Load dataset from Hub
-# -------------------------------------------------
-dataset = load_dataset("Jendersen/bible_welsh_cornish_breton_english")
+# 2. Load your CSV directly from the dataset repo
+from datasets import load_dataset
+import pandas as pd
 
-# Actually we load the JSON file that was uploaded:
-raw = load_dataset(
+# Your CSV is in the dataset repo
+ds = load_dataset(
     "Jendersen/bible_welsh_cornish_breton_english",
-    data_files="parallel_corpus.json"
+    data_files="parallel_corpus.csv"
 )["train"]
 
-# Debug: See what columns we actually have
-print("Columns in dataset:", raw.column_names)
-print("First example:", raw[0])
-
-# Convert directly to DataFrame — no json.loads needed!
-df = pd.DataFrame(raw)
+df = ds.to_pandas()
+print(f"Loaded {len(df)} rows, columns: {list(df.columns)}")
 
-print(f"Loaded {len(df)} verses with columns: {list(df.columns)}")
-
-# -------------------------------------------------
-# 2. Build English → {br, abk, cy}
-# -------------------------------------------------
+# 3. Helper: keep only non-empty, non-punctuation
+import string
 def is_valid(t):
     return bool(t and t.strip() and t.strip() not in string.punctuation)
 
-br = df[df.apply(lambda r: is_valid(r["niv_text"]) and is_valid(r["koad21_text"]), axis=1)][["niv_text","koad21_text"]].rename(columns={"niv_text":"en","koad21_text":"target"})
-br["language"] = "br"
-
-abk = df[df.apply(lambda r: is_valid(r["niv_text"]) and is_valid(r["abk_text"]), axis=1)][["niv_text","abk_text"]].rename(columns={"niv_text":"en","abk_text":"target"})
-abk["language"] = "abk"
-
-cy = df[df.apply(lambda r: is_valid(r["niv_text"]) and is_valid(r["bcnda_text"]), axis=1)][["niv_text","bcnda_text"]].rename(columns={"niv_text":"en","bcnda_text":"target"})
-cy["language"] = "cy"
-
-combined = pd.concat([br, abk, cy], ignore_index=True)
-print(f"Total examples: {len(combined)} (br:{len(br)}, abk:{len(abk)}, cy:{len(cy)})")
-
-# -------------------------------------------------
-# 3. Train / test split
-# -------------------------------------------------
-from datasets import Dataset
-ds = Dataset.from_pandas(combined).train_test_split(test_size=0.2, seed=42)
-raw_datasets = DatasetDict({"train": ds["train"], "test": ds["test"]})
-
-# -------------------------------------------------
-# 4. Tokenizer & Model
-# -------------------------------------------------
-model_name = "t5-small"
-tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)
-model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
-
-# -------------------------------------------------
-# 5. Pre-process
-# -------------------------------------------------
-MAX_LEN = 96
-
-def preprocess(examples):
-    inputs = [f"translate English to {lang}: {en}"
-              for lang, en in zip(examples["language"], examples["en"])]
-    targets = examples["target"]
-    model_inputs = tokenizer(inputs, max_length=MAX_LEN, truncation=True, padding="max_length")
-    labels = tokenizer(targets, max_length=MAX_LEN, truncation=True, padding="max_length").input_ids
-    model_inputs["labels"] = labels
-    return model_inputs
-
-tokenized = raw_datasets.map(preprocess, batched=True, remove_columns=raw_datasets["train"].column_names)
-
-# -------------------------------------------------
-# 6. Data collator & metric
-# -------------------------------------------------
-data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
-
-metric = evaluate.load("sacrebleu")
-
-def compute_metrics(eval_preds):
-    preds, labels = eval_preds
-    if isinstance(preds, tuple):
-        preds = preds[0]
-    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
-    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
-    decoded_preds = [p.strip() for p in decoded_preds]
-    decoded_labels = [[l.strip()] for l in decoded_labels]
-    result = metric.compute(predictions=decoded_preds, references=decoded_labels)
-    return {"bleu": result["score"]}
-
-# -------------------------------------------------
-# 7. Training args
-# -------------------------------------------------
-training_args = Seq2SeqTrainingArguments(
-    output_dir="mt5-celtic-finetuned",
-    eval_strategy="epoch",
-    save_strategy="epoch",
-    learning_rate=3e-4,
-    per_device_train_batch_size=16,
-    per_device_eval_batch_size=16,
-    weight_decay=0.01,
-    num_train_epochs=3,
-    predict_with_generate=True,
-    fp16=False,          # GPU
-    #bf16=True,          # TPU (auto-enabled if on TPU)
-    logging_steps=100,
-    report_to="wandb",  # optional
-    push_to_hub=True,
-    hub_model_id="Jendersen/mt5-celtic-en-br-kw-cy",
-    hub_strategy="end",
-    load_best_model_at_end=True,
-    metric_for_best_model="bleu",
-)
-
-# -------------------------------------------------
-# 8. Trainer
-# -------------------------------------------------
-trainer = Seq2SeqTrainer(
-    model=model,
-    args=training_args,
-    train_dataset=tokenized["train"],
-    eval_dataset=tokenized["test"],
-    tokenizer=tokenizer,
-    data_collator=data_collator,
-    compute_metrics=compute_metrics,
-)
-
-trainer.train()
-
-# -------------------------------------------------
-# 9. Final push
-# -------------------------------------------------
-trainer.push_to_hub("doubleyooz/mt5-celtic-en-br-kw-cy")
-print("Model pushed to Hub!")
+# 4. Build pairs (same logic as your script)
+pairs = []
+
+# Breton (br)
+br = df[df.apply(lambda r: is_valid(r["niv_text"]) and is_valid(r["koad21_text"]), axis=1)]
+for _, r in br.iterrows():
+    pairs.append({
+        "text": f"translate English to br: {r['niv_text']}",
+        "target": r["koad21_text"]
+    })
+
+# Cornish (kw) – column is "abk_text"
+kw = df[df.apply(lambda r: is_valid(r["niv_text"]) and is_valid(r["abk_text"]), axis=1)]
+for _, r in kw.iterrows():
+    pairs.append({
+        "text": f"translate English to kw: {r['niv_text']}",
+        "target": r["abk_text"]
+    })
+
+# Welsh (cy)
+cy = df[df.apply(lambda r: is_valid(r["niv_text"]) and is_valid(r["bcnda_text"]), axis=1)]
+for _, r in cy.iterrows():
+    pairs.append({
+        "text": f"translate English to cy: {r['niv_text']}",
+        "target": r["bcnda_text"]
+    })
+
+final = pd.DataFrame(pairs)
+print(f"Total valid pairs: {len(final)}")
+
+# 5. Train / validation split
+final = final.sample(frac=1, random_state=42).reset_index(drop=True)
+split = int(0.8 * len(final))
+train_df = final.iloc[:split]
+valid_df = final.iloc[split:]
+
+# 6. Save
+train_df.to_csv("train.csv", index=False)
+valid_df.to_csv("valid.csv", index=False)
+
+print(f"Saved train.csv ({len(train_df)} rows) and valid.csv ({len(valid_df)} rows)")