replace all iso1_to_name with iso1toall
Browse files
app.py
CHANGED
|
@@ -12,13 +12,13 @@ favourite_langs = {"Romanian": "ro", "German": "de", "English": "en", "-----": "
|
|
| 12 |
df = pl.read_parquet("isolanguages.parquet")
|
| 13 |
non_empty_isos = df.slice(1).filter(pl.col("ISO639-1") != "").rows()
|
| 14 |
all_langs = {iso[0]: (iso[1], iso[2], iso[3]) for iso in non_empty_isos} # {'Romanian': ('ro', 'rum', 'ron')}
|
|
|
|
| 15 |
name_to_iso1 = {iso[0]: iso[1] for iso in non_empty_isos} # {'Romanian': 'ro', 'German': 'de'}
|
| 16 |
# langs = ["German", "Romanian", "English", "French", "Spanish", "Italian"]
|
| 17 |
langs = list(favourite_langs.keys())
|
| 18 |
langs.extend(list(all_langs.keys())) # Language options as list, add favourite languages first
|
| 19 |
# all_langs = languagecodes.iso_languages_byname
|
| 20 |
# iso1_to_name = {codes[0]: lang for entry in all_langs for lang, codes in entry.items()} # {'ro': 'Romanian', 'de': 'German'}
|
| 21 |
-
iso1_to_name = {iso[1]: iso[0] for iso in non_empty_isos} # {'ro': 'Romanian', 'de': 'German'}
|
| 22 |
|
| 23 |
def timer(func):
|
| 24 |
import time
|
|
@@ -28,7 +28,7 @@ def timer(func):
|
|
| 28 |
end_time = time.time()
|
| 29 |
execution_time = end_time - start_time
|
| 30 |
# print(f"Function {func.__name__!r} executed in {execution_time:.4f} seconds.")
|
| 31 |
-
message_text = f'{message_text[:-1]} in {execution_time:.4f}!' if message_text.endswith(('.', '!')) else f'{message_text} in {execution_time:.4f}!'
|
| 32 |
return translated_text, message_text
|
| 33 |
return wrapper
|
| 34 |
|
|
@@ -78,9 +78,9 @@ class Translators:
|
|
| 78 |
pipe = pipeline("translation", model=hplt_model, device=self.device)
|
| 79 |
translation = pipe(self.input_text)
|
| 80 |
translated_text = translation[0]['translation_text']
|
| 81 |
-
message = f'Translated from {
|
| 82 |
else:
|
| 83 |
-
translated_text = f'HPLT model from {
|
| 84 |
message = f"Available models: {', '.join(hplt_models)}"
|
| 85 |
return translated_text, message
|
| 86 |
|
|
@@ -126,7 +126,7 @@ class Translators:
|
|
| 126 |
if f"{self.sl}-{self.tl}" in quickmt_models:
|
| 127 |
model_path = Translators.quickmtdownload(model_name)
|
| 128 |
translated_text = Translators.quickmttranslate(model_path, self.input_text)
|
| 129 |
-
message = f'Translated from {
|
| 130 |
# Pivot language English
|
| 131 |
elif self.sl in available_languages and self.tl in available_languages:
|
| 132 |
model_name = f"quickmt-{self.sl}-en"
|
|
@@ -135,9 +135,9 @@ class Translators:
|
|
| 135 |
model_name = f"quickmt-en-{self.tl}"
|
| 136 |
model_path = Translators.quickmtdownload(model_name)
|
| 137 |
translated_text = Translators.quickmttranslate(model_path, entranslation)
|
| 138 |
-
message = f'Translated from {
|
| 139 |
else:
|
| 140 |
-
translated_text = f'No Quickmt model available for translation from {
|
| 141 |
message = f"Available models: {', '.join(quickmt_models)}"
|
| 142 |
return translated_text, message
|
| 143 |
|
|
@@ -206,7 +206,7 @@ class Translators:
|
|
| 206 |
try:
|
| 207 |
pipe = pipeline("translation", model=self.model_name, device=self.device)
|
| 208 |
translation = pipe(self.input_text)
|
| 209 |
-
message = f'Translated from {
|
| 210 |
return translation[0]['translation_text'], message
|
| 211 |
except Exception as error:
|
| 212 |
return f"Error translating with model: {self.model_name}! Try other available language combination or model.", error
|
|
@@ -222,7 +222,7 @@ class Translators:
|
|
| 222 |
iso1to3 = {iso[1]: iso[3] for iso in non_empty_isos} # {'ro': 'ron'}
|
| 223 |
iso3tl = iso1to3.get(self.tl) # 'deu', 'ron', 'eng', 'fra'
|
| 224 |
translation = pipe(f'>>{iso3tl}<< {self.input_text}')
|
| 225 |
-
return translation[0]['translation_text'], f'Translated from {
|
| 226 |
except Exception as error:
|
| 227 |
return f"Error translating with model: {self.model_name}! Try other available language combination.", error
|
| 228 |
|
|
@@ -231,13 +231,13 @@ class Translators:
|
|
| 231 |
model_name = f"Helsinki-NLP/opus-mt-{self.sl}-{self.tl}"
|
| 232 |
pipe = pipeline("translation", model=model_name, device=self.device)
|
| 233 |
translation = pipe(self.input_text)
|
| 234 |
-
return translation[0]['translation_text'], f'Translated from {
|
| 235 |
except EnvironmentError:
|
| 236 |
try: # Tatoeba models
|
| 237 |
model_name = f"Helsinki-NLP/opus-tatoeba-{self.sl}-{self.tl}"
|
| 238 |
pipe = pipeline("translation", model=model_name, device=self.device)
|
| 239 |
translation = pipe(self.input_text)
|
| 240 |
-
return translation[0]['translation_text'], f'Translated from {
|
| 241 |
except EnvironmentError as error:
|
| 242 |
self.model_name = "Helsinki-NLP/opus-mt-tc-bible-big-mul-mul" # Last resort: try multi to multi
|
| 243 |
return self.HelsinkiNLP_mulroa()
|
|
|
|
| 12 |
df = pl.read_parquet("isolanguages.parquet")
|
| 13 |
non_empty_isos = df.slice(1).filter(pl.col("ISO639-1") != "").rows()
|
| 14 |
all_langs = {iso[0]: (iso[1], iso[2], iso[3]) for iso in non_empty_isos} # {'Romanian': ('ro', 'rum', 'ron')}
|
| 15 |
+
iso1toall = {iso[1]: (iso[0], iso[2], iso[3]) for iso in non_empty_isos} # {'ro': ('Romanian', 'rum', 'ron')}
|
| 16 |
name_to_iso1 = {iso[0]: iso[1] for iso in non_empty_isos} # {'Romanian': 'ro', 'German': 'de'}
|
| 17 |
# langs = ["German", "Romanian", "English", "French", "Spanish", "Italian"]
|
| 18 |
langs = list(favourite_langs.keys())
|
| 19 |
langs.extend(list(all_langs.keys())) # Language options as list, add favourite languages first
|
| 20 |
# all_langs = languagecodes.iso_languages_byname
|
| 21 |
# iso1_to_name = {codes[0]: lang for entry in all_langs for lang, codes in entry.items()} # {'ro': 'Romanian', 'de': 'German'}
|
|
|
|
| 22 |
|
| 23 |
def timer(func):
|
| 24 |
import time
|
|
|
|
| 28 |
end_time = time.time()
|
| 29 |
execution_time = end_time - start_time
|
| 30 |
# print(f"Function {func.__name__!r} executed in {execution_time:.4f} seconds.")
|
| 31 |
+
message_text = f'{message_text[:-1]} in {execution_time:.4f} seconds!' if message_text.endswith(('.', '!')) else f'{message_text} in {execution_time:.4f} seconds!'
|
| 32 |
return translated_text, message_text
|
| 33 |
return wrapper
|
| 34 |
|
|
|
|
| 78 |
pipe = pipeline("translation", model=hplt_model, device=self.device)
|
| 79 |
translation = pipe(self.input_text)
|
| 80 |
translated_text = translation[0]['translation_text']
|
| 81 |
+
message = f'Translated from {iso1toall[self.sl][0]} to {iso1toall[self.tl][0]} with {hplt_model}.'
|
| 82 |
else:
|
| 83 |
+
translated_text = f'HPLT model from {iso1toall[self.sl][0]} to {iso1toall[self.tl][0]} not available!'
|
| 84 |
message = f"Available models: {', '.join(hplt_models)}"
|
| 85 |
return translated_text, message
|
| 86 |
|
|
|
|
| 126 |
if f"{self.sl}-{self.tl}" in quickmt_models:
|
| 127 |
model_path = Translators.quickmtdownload(model_name)
|
| 128 |
translated_text = Translators.quickmttranslate(model_path, self.input_text)
|
| 129 |
+
message = f'Translated from {iso1toall[self.sl][0]} to {iso1toall[self.tl][0]} with {model_name}.'
|
| 130 |
# Pivot language English
|
| 131 |
elif self.sl in available_languages and self.tl in available_languages:
|
| 132 |
model_name = f"quickmt-{self.sl}-en"
|
|
|
|
| 135 |
model_name = f"quickmt-en-{self.tl}"
|
| 136 |
model_path = Translators.quickmtdownload(model_name)
|
| 137 |
translated_text = Translators.quickmttranslate(model_path, entranslation)
|
| 138 |
+
message = f'Translated from {iso1toall[self.sl][0]} to {iso1toall[self.tl][0]} with Quickmt using pivot language English.'
|
| 139 |
else:
|
| 140 |
+
translated_text = f'No Quickmt model available for translation from {iso1toall[self.sl][0]} to {iso1toall[self.tl][0]}!'
|
| 141 |
message = f"Available models: {', '.join(quickmt_models)}"
|
| 142 |
return translated_text, message
|
| 143 |
|
|
|
|
| 206 |
try:
|
| 207 |
pipe = pipeline("translation", model=self.model_name, device=self.device)
|
| 208 |
translation = pipe(self.input_text)
|
| 209 |
+
message = f'Translated from {iso1toall[self.sl][0]} to {iso1toall[self.tl][0]} with {self.model_name}.'
|
| 210 |
return translation[0]['translation_text'], message
|
| 211 |
except Exception as error:
|
| 212 |
return f"Error translating with model: {self.model_name}! Try other available language combination or model.", error
|
|
|
|
| 222 |
iso1to3 = {iso[1]: iso[3] for iso in non_empty_isos} # {'ro': 'ron'}
|
| 223 |
iso3tl = iso1to3.get(self.tl) # 'deu', 'ron', 'eng', 'fra'
|
| 224 |
translation = pipe(f'>>{iso3tl}<< {self.input_text}')
|
| 225 |
+
return translation[0]['translation_text'], f'Translated from {iso1toall[self.sl][0]} to {iso1toall[self.tl][0]} with {self.model_name}.'
|
| 226 |
except Exception as error:
|
| 227 |
return f"Error translating with model: {self.model_name}! Try other available language combination.", error
|
| 228 |
|
|
|
|
| 231 |
model_name = f"Helsinki-NLP/opus-mt-{self.sl}-{self.tl}"
|
| 232 |
pipe = pipeline("translation", model=model_name, device=self.device)
|
| 233 |
translation = pipe(self.input_text)
|
| 234 |
+
return translation[0]['translation_text'], f'Translated from {iso1toall[self.sl][0]} to {iso1toall[self.tl][0]} with {model_name}.'
|
| 235 |
except EnvironmentError:
|
| 236 |
try: # Tatoeba models
|
| 237 |
model_name = f"Helsinki-NLP/opus-tatoeba-{self.sl}-{self.tl}"
|
| 238 |
pipe = pipeline("translation", model=model_name, device=self.device)
|
| 239 |
translation = pipe(self.input_text)
|
| 240 |
+
return translation[0]['translation_text'], f'Translated from {iso1toall[self.sl][0]} to {iso1toall[self.tl][0]} with {model_name}.'
|
| 241 |
except EnvironmentError as error:
|
| 242 |
self.model_name = "Helsinki-NLP/opus-mt-tc-bible-big-mul-mul" # Last resort: try multi to multi
|
| 243 |
return self.HelsinkiNLP_mulroa()
|