Allow passing string to `save_tokenizer_config`
Browse filesSee https://huggingface.co/openGPT-X/Teuken-7B-instruct-research-v0.4/discussions/7.
Need to implement this "upstream" in the tokenizer repo as well.
- gptx_tokenizer.py +3 -1
gptx_tokenizer.py
CHANGED
|
@@ -395,7 +395,9 @@ class HFGPTXTokenizer(PreTrainedTokenizer):
|
|
| 395 |
old_tokenizer_file_path.replace(new_tokenizer_file_path)
|
| 396 |
return str(new_tokenizer_file_path)
|
| 397 |
|
| 398 |
-
def save_tokenizer_config(self, save_dir: Path) -> None:
|
|
|
|
|
|
|
| 399 |
# convert Path to str
|
| 400 |
for k in self.tokenizer_config:
|
| 401 |
if isinstance(self.tokenizer_config[k], Path):
|
|
|
|
| 395 |
old_tokenizer_file_path.replace(new_tokenizer_file_path)
|
| 396 |
return str(new_tokenizer_file_path)
|
| 397 |
|
| 398 |
+
def save_tokenizer_config(self, save_dir: Union[str, Path]) -> None:
|
| 399 |
+
save_dir = Path(save_dir)
|
| 400 |
+
|
| 401 |
# convert Path to str
|
| 402 |
for k in self.tokenizer_config:
|
| 403 |
if isinstance(self.tokenizer_config[k], Path):
|