Valentin De Matos
commited on
fix: handles AutoTokenizer from untrusted source
Browse filesSet trust_remote_code param depending of cfg.trust_remote_code when calling AutoTokenizer.from_pretrained
src/axolotl/utils/models.py
CHANGED
|
@@ -167,7 +167,7 @@ def load_model(
|
|
| 167 |
else:
|
| 168 |
tokenizer = getattr(transformers, tokenizer_type).from_pretrained(model)
|
| 169 |
except:
|
| 170 |
-
tokenizer = AutoTokenizer.from_pretrained(base_model_config)
|
| 171 |
|
| 172 |
logging.debug(f"EOS: {tokenizer.eos_token_id} / {tokenizer.eos_token}")
|
| 173 |
logging.debug(f"BOS: {tokenizer.bos_token_id} / {tokenizer.bos_token}")
|
|
|
|
| 167 |
else:
|
| 168 |
tokenizer = getattr(transformers, tokenizer_type).from_pretrained(model)
|
| 169 |
except:
|
| 170 |
+
tokenizer = AutoTokenizer.from_pretrained(base_model_config, trust_remote_code=True if cfg.trust_remote_code is True else False)
|
| 171 |
|
| 172 |
logging.debug(f"EOS: {tokenizer.eos_token_id} / {tokenizer.eos_token}")
|
| 173 |
logging.debug(f"BOS: {tokenizer.bos_token_id} / {tokenizer.bos_token}")
|