Upload tokenization_hy.py with huggingface_hub
Browse files- tokenization_hy.py +1 -1
tokenization_hy.py
CHANGED
|
@@ -287,7 +287,7 @@ class HYTokenizer(PreTrainedTokenizer):
|
|
| 287 |
|
| 288 |
# tests
|
| 289 |
if __name__ == "__main__":
|
| 290 |
-
tokenizer = HYTokenizer.from_pretrained('/hy')
|
| 291 |
text = '你好,世界'
|
| 292 |
tokens = tokenizer.tokenize(text)
|
| 293 |
print(tokens)
|
|
|
|
| 287 |
|
| 288 |
# tests
|
| 289 |
if __name__ == "__main__":
|
| 290 |
+
tokenizer = HYTokenizer.from_pretrained('./other_tokenizer_vocab/hy')
|
| 291 |
text = '你好,世界'
|
| 292 |
tokens = tokenizer.tokenize(text)
|
| 293 |
print(tokens)
|