Update utils.py

utils.py

@@ -18,7 +18,7 @@ def get_tokenizer(id):
 def get_tokenization(tokenizer, text):
     logging.debug(f"Tokenizing text: {text}")
     ids = tokenizer.encode(text).ids
-    string_tokens = tokenizer.decode(ids)
+    string_tokens =  [tokenizer.id_to_token(id) for id in ids]
     logging.debug(f"Tokens: {string_tokens}")
     return string_tokens