Commit
·
66875f9
1
Parent(s):
364ead3
Update tokenization_baichuan.py
Browse files- tokenization_baichuan.py +1 -0
tokenization_baichuan.py
CHANGED
|
@@ -72,6 +72,7 @@ class BaichuanTokenizer(PreTrainedTokenizer):
|
|
| 72 |
eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
|
| 73 |
unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
|
| 74 |
pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
|
|
|
|
| 75 |
self.vocab_file = vocab_file
|
| 76 |
self.add_bos_token = add_bos_token
|
| 77 |
self.add_eos_token = add_eos_token
|
|
|
|
| 72 |
eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
|
| 73 |
unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
|
| 74 |
pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
|
| 75 |
+
|
| 76 |
self.vocab_file = vocab_file
|
| 77 |
self.add_bos_token = add_bos_token
|
| 78 |
self.add_eos_token = add_eos_token
|