try to recreate the code

app.py

@@ -1,118 +1,128 @@
 import gradio as gr
-import random
-import re
-import difflib
-import torch
-from functools import lru_cache
-from transformers import pipeline
-
-# ------------------- Sentence Bank (customize freely) -------------------
-SENTENCE_BANK = [
-    "The quick brown fox jumps over the lazy dog.",
-    "I promise to speak clearly and at a steady pace.",
-    "Open source makes AI more transparent and inclusive.",
-    "Hugging Face Spaces make demos easy to share.",
-    "Today the weather in Berlin is pleasantly cool.",
-    "Privacy and transparency should go hand in hand.",
-    "Please generate a new sentence for me to read.",
-    "Machine learning can amplify or reduce inequality.",
-    "Responsible AI requires participation from everyone.",
-    "This microphone test checks my pronunciation accuracy.",
-]
-
-# ------------------- Utilities -------------------
-def normalize_text(t: str) -> str:
-    # English-only normalization: lowercase, keep letters/digits/' and -
-    t = t.lower()
-    t = re.sub(r"[^a-z0-9'\-]+", " ", t)
-    t = re.sub(r"\s+", " ", t).strip()
-    return t
-
-def similarity_and_diff(ref: str, hyp: str):
-    """Return similarity ratio (0..1) and HTML diff highlighting changes."""
-    ref_tokens = ref.split()
-    hyp_tokens = hyp.split()
-    sm = difflib.SequenceMatcher(a=ref_tokens, b=hyp_tokens)
-    ratio = sm.ratio()
 
+import src.generate as generate
+import src.process as process
+import src.tts as tts
+
+
+# ------------------- UI printing functions -------------------
+def clear_all():
+    # target, user_transcript, score_html, diff_html, result_html,
+    # tts_text, clone_status, tts_audio
+    return "", "", "", "", "", "", "", None
+
+
+def make_result_html(pass_threshold, passed, ratio):
+    """Returns summary and score label."""
+    summary = (
+        f"✅ Correct (≥ {int(pass_threshold * 100)}%)"
+        if passed else
+        f"❌ Not a match (need ≥ {int(pass_threshold * 100)}%)"
+    )
+    score = f"Similarity: {ratio * 100:.1f}%"
+    return summary, score
+
+
+def make_alignment_html(ref_tokens, hyp_tokens, alignments):
+    """Returns HTML showing alignment between target and recognized user audio."""
     out = []
-    for op, i1, i2, j1, j2 in sm.get_opcodes():
+    no_match_html = ' <span style="background:#ffe0e0;text-decoration:line-through;">'
+    match_html = ' <span style="background:#e0ffe0;">'
+    for span in alignments:
+        op, i1, i2, j1, j2 = span
+        ref_string = " ".join(ref_tokens[i1:i2])
+        hyp_string = " ".join(hyp_tokens[j1:j2])
         if op == "equal":
-            out.append(" " + " ".join(ref_tokens[i1:i2]))
+            out.append(" " + ref_string)
         elif op == "delete":
-            out.append(
-                ' <span style="background:#ffe0e0;text-decoration:line-through;">'
-                + " ".join(ref_tokens[i1:i2]) + "</span>"
-            )
+            out.append(no_match_html + ref_string + "</span>")
         elif op == "insert":
-            out.append(
-                ' <span style="background:#e0ffe0;">'
-                + " ".join(hyp_tokens[j1:j2]) + "</span>"
-            )
+            out.append(match_html + hyp_string + "</span>")
         elif op == "replace":
-            out.append(
-                ' <span style="background:#ffe0e0;text-decoration:line-through;">'
-                + " ".join(ref_tokens[i1:i2]) + "</span>"
-            )
-            out.append(
-                ' <span style="background:#e0ffe0;">'
-                + " ".join(hyp_tokens[j1:j2]) + "</span>"
-            )
+            out.append(no_match_html + ref_string + "</span>")
+            out.append(match_html + hyp_string + "</span>")
     html = '<div style="line-height:1.6;font-size:1rem;">' + "".join(out).strip() + "</div>"
-    return ratio, html
-
-@lru_cache(maxsize=2)
-def get_asr(model_id: str, device_preference: str):
-    """Cache an ASR pipeline. device_preference: 'auto'|'cpu'|'cuda'."""
-    if device_preference == "cuda" and torch.cuda.is_available():
-        device = 0
-    elif device_preference == "auto":
-        device = 0 if torch.cuda.is_available() else -1
-    else:
-        device = -1
-    return pipeline(
-        "automatic-speech-recognition",
-        model=model_id,           # use English-only Whisper models (.en)
-        device=device,
-        chunk_length_s=30,
-        return_timestamps=False,
-    )
+    return html
 
-def gen_sentence():
-    return random.choice(SENTENCE_BANK)
 
-def clear_all():
-    # target, hyp_out, score_out, diff_out, summary_out
-    return "", "", "", "", ""
+def make_html(sentence_match):
+    """Build diff + results HTML."""
+    diff_html = make_alignment_html(sentence_match.target_tokens,
+                                    sentence_match.user_tokens,
+                                    sentence_match.alignments)
+    result_html, score_html = make_result_html(sentence_match.pass_threshold,
+                                               sentence_match.passed,
+                                               sentence_match.ratio)
+    return score_html, result_html, diff_html
+
 
 # ------------------- Core Check (English-only) -------------------
-def check_pronunciation(audio_path, target_sentence, model_id, device_pref, pass_threshold):
+def get_user_transcript(audio_path: gr.Audio, target_sentence: str, model_id: str, device_pref: str) -> (str, str):
+    """ASR for the input audio and basic validation."""
     if not target_sentence:
-        return "", "", "", "Please generate a sentence first."
-
-    asr = get_asr(model_id, device_pref)
-
-    try:
-        # IMPORTANT: For English-only Whisper (.en), do NOT pass language/task args.
-        result = asr(audio_path)
-        hyp_raw = result["text"].strip()
-    except Exception as e:
-        return "", "", "", f"Transcription failed: {e}"
-
-    ref_norm = normalize_text(target_sentence)
-    hyp_norm = normalize_text(hyp_raw)
-
-    ratio, diff_html = similarity_and_diff(ref_norm, hyp_norm)
-    passed = ratio >= pass_threshold
+        return "Please generate a sentence first.", ""
+    if audio_path is None:
+        return "Please start, record, then stop the audio recording before trying to transcribe.", ""
+
+    user_transcript = process.run_asr(audio_path, model_id, device_pref)
+    if isinstance(user_transcript, Exception):
+        return f"Transcription failed: {user_transcript}", ""
+    return "", user_transcript
+
+
+def transcribe_check(audio_path, target_sentence, model_id, device_pref, pass_threshold):
+    """Transcribe user audio, compute match, and render results."""
+    error_msg, user_transcript = get_user_transcript(audio_path, target_sentence, model_id, device_pref)
+    if error_msg:
+        score_html = ""
+        diff_html = ""
+        result_html = error_msg
+    else:
+        sentence_match = process.SentenceMatcher(target_sentence, user_transcript, pass_threshold)
+        score_html, result_html, diff_html = make_html(sentence_match)
+    return user_transcript, score_html, result_html, diff_html
+
+
+# ------------------- Voice cloning gate -------------------
+def clone_if_pass(
+    audio_path,              # ref voice (the same recorded clip)
+    target_sentence,         # sentence user was supposed to say
+    user_transcript,         # what ASR heard
+    tts_text,                # what we want to synthesize (in cloned voice)
+    pass_threshold,          # must meet or exceed this
+    tts_model_id,            # e.g., "coqui/XTTS-v2"
+    tts_language,            # e.g., "en"
+):
+    """
+    If user correctly read the target (>= threshold), clone their voice from the
+    recorded audio and speak 'tts_text'. Otherwise, refuse.
+    """
+    # Basic validations
+    if audio_path is None:
+        return None, "Record audio first (reference voice is required)."
+    if not target_sentence:
+        return None, "Generate a target sentence first."
+    if not user_transcript:
+        return None, "Transcribe first to verify the sentence."
+    if not tts_text:
+        return None, "Enter the sentence to synthesize."
+
+    # Recompute pass/fail to avoid relying on UI state
+    sm = process.SentenceMatcher(target_sentence, user_transcript, pass_threshold)
+    if not sm.passed:
+        return None, (
+            f"❌ Cloning blocked: your reading did not reach the threshold "
+            f"({sm.ratio*100:.1f}% < {int(pass_threshold*100)}%)."
+        )
 
-    summary = (
-        f"✅ Correct (≥ {int(pass_threshold*100)}%)"
-        if passed else
-        f"❌ Not a match (need ≥ {int(pass_threshold*100)}%)"
-    )
-    score = f"Similarity: {ratio*100:.1f}%"
+    # Run zero-shot cloning
+    out = tts.run_tts_clone(audio_path, tts_text, model_id=tts_model_id, language=tts_language)
+    if isinstance(out, Exception):
+        return None, f"Voice cloning failed: {out}"
+    sr, wav = out
+    # Gradio Audio can take a tuple (sr, np.array)
+    return (sr, wav), f"✅ Cloned and synthesized with {tts_model_id} ({tts_language})."
 
-    return hyp_raw, score, diff_html, summary
 
 # ------------------- UI -------------------
 with gr.Blocks(title="Say the Sentence (English)") as demo:
@@ -122,25 +132,28 @@ with gr.Blocks(title="Say the Sentence (English)") as demo:
         1) Generate a sentence.  
         2) Record yourself reading it.  
         3) Transcribe & check your accuracy.  
+        4) If matched, clone your voice to speak any sentence you enter.
         """
     )
 
     with gr.Row():
-        target = gr.Textbox(label="Target sentence", interactive=False, placeholder="Click 'Generate sentence'")
+        target = gr.Textbox(label="Target sentence", interactive=False,
+                            placeholder="Click 'Generate sentence'")
 
     with gr.Row():
         btn_gen = gr.Button("🎲 Generate sentence", variant="primary")
         btn_clear = gr.Button("🧹 Clear")
 
     with gr.Row():
-        audio = gr.Audio(sources=["microphone"], type="filepath", label="Record your voice")
+        audio = gr.Audio(sources=["microphone"], type="filepath",
+                         label="Record your voice")
 
     with gr.Accordion("Advanced settings", open=False):
         model_id = gr.Dropdown(
             choices=[
-                "openai/whisper-tiny.en",        # fastest (CPU-friendly)
-                "openai/whisper-base.en",        # better accuracy, a bit slower
-                "distil-whisper/distil-small.en" # optional distil English model
+                "openai/whisper-tiny.en",
+                "openai/whisper-base.en",
+                "distil-whisper/distil-small.en",
             ],
             value="openai/whisper-tiny.en",
             label="ASR model (English only)",
@@ -150,11 +163,11 @@ with gr.Blocks(title="Say the Sentence (English)") as demo:
             value="auto",
             label="Device preference"
         )
-        pass_threshold = gr.Slider(0.50, 1.00, value=0.85, step=0.01, label="Match threshold")
+        pass_threshold = gr.Slider(0.50, 1.00, value=0.85, step=0.01,
+                                   label="Match threshold")
 
     with gr.Row():
         btn_check = gr.Button("✅ Transcribe & Check", variant="primary")
-<<<<<<< HEAD
     with gr.Row():
         user_transcript = gr.Textbox(label="Transcription", interactive=False)
     with gr.Row():
@@ -172,10 +185,10 @@ with gr.Blocks(title="Say the Sentence (English)") as demo:
     with gr.Row():
         tts_model_id = gr.Dropdown(
             choices=[
-                "tts_models/multilingual/multi-dataset/xtts_v2",
-                # add others if you like, e.g. "myshell-ai/MeloTTS"
+                "coqui/XTTS-v2",
+                # add others if you like, e.g., "myshell-ai/MeloTTS"
             ],
-            value="tts_models/multilingual/multi-dataset/xtts_v2",
+            value="coqui/XTTS-v2",
             label="TTS (voice cloning) model",
         )
         tts_language = gr.Dropdown(
@@ -183,24 +196,32 @@ with gr.Blocks(title="Say the Sentence (English)") as demo:
             value="en",
             label="Language",
         )
-=======
->>>>>>> parent of c5d4931 (add audio cloning functionality (test))
 
     with gr.Row():
-        hyp_out = gr.Textbox(label="Transcription", interactive=False)
+        btn_clone = gr.Button("🔁 Clone voice (if passed)", variant="secondary")
     with gr.Row():
-        score_out = gr.Label(label="Score")
-        summary_out = gr.Label(label="Result")
-    diff_out = gr.HTML(label="Word-level diff (red = expected but missing / green = extra or replacement)")
+        tts_audio = gr.Audio(label="Cloned speech output", interactive=False)
+        clone_status = gr.Label(label="Cloning status")
+
+    # -------- Events --------
+    btn_gen.click(fn=generate.gen_sentence_set, outputs=target)
+
+    btn_clear.click(
+        fn=clear_all,
+        outputs=[target, user_transcript, score_html, result_html, diff_html, tts_text, clone_status, tts_audio]
+    )
 
-    # Events
-    btn_gen.click(fn=gen_sentence, outputs=target)
-    btn_clear.click(fn=clear_all, outputs=[target, hyp_out, score_out, diff_out, summary_out])
     btn_check.click(
-        fn=check_pronunciation,
+        fn=transcribe_check,
         inputs=[audio, target, model_id, device_pref, pass_threshold],
-        outputs=[hyp_out, score_out, diff_out, summary_out]
+        outputs=[user_transcript, score_html, result_html, diff_html]
+    )
+
+    btn_clone.click(
+        fn=clone_if_pass,
+        inputs=[audio, target, user_transcript, tts_text, pass_threshold, tts_model_id, tts_language],
+        outputs=[tts_audio, clone_status],
     )
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()

requirements.txt

@@ -3,12 +3,7 @@ transformers>=4.44.0
 torch>=2.2.0
 accelerate>=0.33.0
 sentencepiece>=0.2.0
-<<<<<<< HEAD
 numpy
 TTS>=0.22.0
 ffmpeg-python
 torchaudio
-
-
-=======
->>>>>>> parent of c5d4931 (add audio cloning functionality (test))

src/generate.py

@@ -0,0 +1,45 @@
+import random
+
+from transformers import pipeline, AutoTokenizer
+
+import src.process as process
+
+# You can choose to use either:
+# (1) a list of pre-specified sentences, in SENTENCE_BANK
+# (2) an LLM-generated sentence.
+# SENTENCE_BANK is used in the gen_sentence_set function.
+# LLM generation is used in the gen_sentence_llm function.
+
+# ------------------- Sentence Bank (customize freely) -------------------
+SENTENCE_BANK = [
+    "The quick brown fox jumps over the lazy dog.",
+    "I promise to speak clearly and at a steady pace.",
+    "Open source makes AI more transparent and inclusive.",
+    "Hugging Face Spaces make demos easy to share.",
+    "Today the weather in Berlin is pleasantly cool.",
+    "Privacy and transparency should go hand in hand.",
+    "Please generate a new sentence for me to read.",
+    "Machine learning can amplify or reduce inequality.",
+    "Responsible AI requires participation from everyone.",
+    "This microphone test checks my pronunciation accuracy.",
+]
+
+
+def gen_sentence_llm():
+    """Generates a sentence using an LLM.
+    Returns:
+        Normalized text string to display in the UI.
+    """
+    prompt = ""
+    tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2")
+    generator = pipeline('text-generation', model='gpt2')
+    result = generator(prompt, stop_strings=[".", ], num_return_sequences=1,
+                       tokenizer=tokenizer, pad_token_id=tokenizer.eos_token_id)
+    display_text = process.normalize_text(result[0]["generated_text"],
+                                          lower=False)
+    return display_text
+
+
+def gen_sentence_set():
+    """Returns a sentence for the user to say using a prespecified set of options."""
+    return random.choice(SENTENCE_BANK)

src/process.py

@@ -1,74 +1,87 @@
 import difflib
-import os
 import re
 from functools import lru_cache
 
+import gradio.components.audio as gr_audio
 import torch
 from transformers import pipeline
 
+
 # ------------------- Utilities -------------------
 def normalize_text(t: str, lower: bool = True) -> str:
+    """For normalizing LLM-generated and human-generated strings.
+    For LLMs, this removes extraneous quote marks and spaces."""
+    # English-only normalization: lowercase, keep letters/digits/' and -
     if lower:
         t = t.lower()
+    # TODO: Previously was re.sub(r"[^a-z0-9'\-]+", " ", t); discuss normalizing for LLMs too.
     t = re.sub(r"[^a-zA-Z0-9'\-.,]+", " ", t)
     t = re.sub(r"\s+", " ", t).strip()
     return t
 
-def _pick_device(pref: str) -> int:
-    if pref == "cuda" and torch.cuda.is_available():
-        return 0
-    if pref == "auto":
-        return 0 if torch.cuda.is_available() else -1
-    return -1
 
 @lru_cache(maxsize=2)
-def get_asr_pipeline(model_id: str, device_preference: str):
-    device = _pick_device(device_preference)
-    # IMPORTANT: For .en models do NOT set language/task
+def get_asr_pipeline(model_id: str, device_preference: str) -> pipeline:
+    """Cache an ASR pipeline.
+    Parameters:
+        model_id: String of desired ASR model.
+        device_preference: String of desired device for ASR processing, "cuda", "cpu", or "auto".
+    Returns:
+        transformers.pipeline ASR component.
+    """
+    if device_preference == "cuda" and torch.cuda.is_available():
+        device = 0
+    elif device_preference == "auto":
+        device = 0 if torch.cuda.is_available() else -1
+    else:
+        device = -1
     return pipeline(
         "automatic-speech-recognition",
-        model=model_id,
+        model=model_id,           # use English-only Whisper models (.en)
         device=device,
         chunk_length_s=30,
         return_timestamps=False,
     )
 
-def _validate_audio_path(p: str) -> None:
-    if not isinstance(p, str):
-        raise ValueError("Audio input is not a file path (expected type='filepath').")
-    if not os.path.exists(p):
-        raise FileNotFoundError(f"Recorded audio file not found: {p}")
-    if os.path.getsize(p) < 1024:
-        raise ValueError("Recorded audio seems empty or too short (<1KB). Try again.")
-
-def run_asr(audio_path, model_id: str, device_pref: str):
-    """
-    Returns the recognized text or an Exception (do NOT raise).
+def run_asr(audio_path: gr_audio, model_id: str, device_pref: str) -> str | Exception:
+    """Returns the recognized user utterance from the input audio stream.
+    Parameters:
+        audio_path: gradio.Audio component.
+        model_id: String of desired ASR model.
+        device_preference: String of desired device for ASR processing, "cuda", "cpu", or "auto".
+    Returns:
+        hyp_raw: Recognized user utterance.
     """
+    asr = get_asr_pipeline(model_id, device_pref)
     try:
-        _validate_audio_path(audio_path)
-        asr = get_asr_pipeline(model_id, device_pref)
+        # IMPORTANT: For English-only Whisper (.en), do NOT pass language/task args.
         result = asr(audio_path)
-        # transformers ASR returns {"text": "...", ...}
-        hyp_raw = result.get("text", "").strip()
-        if not hyp_raw:
-            raise RuntimeError("ASR returned empty text.")
-        return hyp_raw
+        hyp_raw = result["text"].strip()
     except Exception as e:
-        # Return the real, descriptive error back to the UI
         return e
+    return hyp_raw
 
-# -------------- diff + matching (unchanged) --------------
-def similarity_and_diff(ref_tokens: list, hyp_tokens: list):
+def similarity_and_diff(ref_tokens: list, hyp_tokens: list) -> (float, list[str, int, int, int]):
+    """
+    Returns:
+        ratio: Similarity ratio (0..1).
+        opcodes: List of differences between target and recognized user utterance.
+    """
     sm = difflib.SequenceMatcher(a=ref_tokens, b=hyp_tokens)
-    return sm.ratio(), sm.get_opcodes()
+    ratio = sm.ratio()
+    opcodes = sm.get_opcodes()
+    return ratio, opcodes
 
 class SentenceMatcher:
+    """Class for keeping track of (target sentence, user utterance) match features."""
     def __init__(self, target_sentence, user_transcript, pass_threshold):
-        self.target_sentence = target_sentence
-        self.user_transcript = user_transcript
-        self.pass_threshold = pass_threshold
-        self.target_tokens = normalize_text(target_sentence).split()
-        self.user_tokens = normalize_text(user_transcript).split()
-        self.ratio, self.alignments = similarity_and_diff(self.target_tokens, self.user_tokens)
-        self.passed = self.ratio >= self.pass_threshold
+        self.target_sentence: str = target_sentence
+        self.user_transcript: str = user_transcript
+        self.pass_threshold: float = pass_threshold
+        self.target_tokens: list = normalize_text(target_sentence).split()
+        self.user_tokens: list = normalize_text(user_transcript).split()
+        self.ratio: float
+        self.alignments: list
+        self.ratio, self.alignments = similarity_and_diff(self.target_tokens,
+                                                          self.user_tokens)
+        self.passed: bool = self.ratio >= self.pass_threshold

src/tts.py

@@ -1,32 +1,42 @@
-# src/tts.py
 from __future__ import annotations
 from typing import Tuple, Union
+
 import numpy as np
-from TTS.api import TTS  # ← from the Coqui TTS package, not transformers
+from transformers import pipeline
+
+# We use the text-to-speech pipeline with XTTS v2 (zero-shot cloning)
+# Example forward params: {"speaker_wav": "/path/to/ref.wav", "language": "en"}
+
+def get_tts_pipeline(model_id: str):
+    """
+    Create a TTS pipeline for the given model.
+    XTTS v2 works well for zero-shot cloning and is available on the Hub.
+    """
+    # NOTE: Add device selection similar to ASR if needed
+    return pipeline("text-to-speech", model=model_id)
 
 def run_tts_clone(
     ref_audio_path: str,
     text_to_speak: str,
-    model_id: str = "tts_models/multilingual/multi-dataset/xtts_v2",
+    model_id: str = "coqui/XTTS-v2",
     language: str = "en",
 ) -> Union[Tuple[int, np.ndarray], Exception]:
     """
-    Synthesize `text_to_speak` in the cloned voice from `ref_audio_path`.
+    Synthesize 'text_to_speak' in the cloned voice from 'ref_audio_path'.
 
     Returns:
         (sampling_rate, waveform) on success, or Exception on failure.
     """
     try:
-        try:
-            tts = TTS(model_name=model_id, progress_bar=False, gpu=False)
-        except KeyError as ke:
-            # Typical message shows just 'xtts_v2' → old Coqui package
-            return RuntimeError(
-                f"Coqui TTS cannot find '{model_id}'. "
-                "Please upgrade the TTS package (e.g., `pip install -U TTS>=0.22.0`)."
-            )
-
-        wav = tts.tts(text=text_to_speak, speaker_wav=ref_audio_path, language=language)
-        return 24000, np.asarray(wav, dtype=np.float32)
+        tts = get_tts_pipeline(model_id)
+        result = tts(
+            text_to_speak,
+            forward_params={"speaker_wav": ref_audio_path, "language": language},
+        )
+        # transformers TTS returns dict like: {"audio": {"array": np.ndarray, "sampling_rate": 24000}}
+        audio = result["audio"]
+        sr = int(audio["sampling_rate"])
+        wav = audio["array"].astype(np.float32)
+        return sr, wav
     except Exception as e:
         return e