async function loadTokenizer(pretrained_model_name_or_path, options) {

    const info = await Promise.all([
        getModelJSON(pretrained_model_name_or_path, 'tokenizer.json', true, options),
        getModelJSON(pretrained_model_name_or_path, 'tokenizer_config.json', true, options),
    ])

    // Override legacy option if `options.legacy` is not null
    if (options.legacy !== null) {
        info[1].legacy = options.legacy;
    }
    return info;
}
Loads a tokenizer from the specified path.
@param {string} pretrained_model_name_or_path The path to the tokenizer directory.
@param {PretrainedTokenizerOptions} options Additional options for loading the tokenizer.
@returns {Promise<any[]>} A promise that resolves with information about the loaded tokenizer.