constructor(config) {
        super(config);

        /** @type {Map<string, number>} */
        this.tokens_to_ids = objectToMap(config.vocab);

        this.unk_token_id = this.tokens_to_ids.get(config.unk_token);
        this.unk_token = config.unk_token;

        this.vocab = new Array(this.tokens_to_ids.size);
        for (const [key, value] of this.tokens_to_ids) {
            this.vocab[value] = key;
        }

        // Tokenizers >= 0.20.0 serializes BPE merges as a [string, string][] instead of a string[],
        // which resolves the ambiguity for merges containing spaces.
        const use_new_merge_format = Array.isArray(config.merges[0]);

        /** @type {[string, string][]} */
        this.merges = use_new_merge_format
            ? /** @type {[string, string][]} */(config.merges)
            : (/** @type {string[]} */(config.merges)).map(x => /** @type {[string, string]} */(x.split(' ', 2)));
        this.bpe_ranks = new Map(this.merges.map((x, i) => [JSON.stringify(x), i]));

        this.end_of_word_suffix = config.end_of_word_suffix;

        // NOTE: `continuing_subword_suffix` is custom (to support `BlenderbotSmallTokenizer`)
        this.continuing_subword_suffix = config.continuing_subword_suffix ?? null;

        this.byte_fallback = this.config.byte_fallback ?? false;

        if (this.byte_fallback) {
            this.text_encoder = new TextEncoder();
        }

        this.ignore_merges = this.config.ignore_merges ?? false;

        /**
         * The maximum length we should cache in a model.
         * Strings that are too long have minimal chances to cache hit anyway
         */
        this.max_length_to_cache = 256;

        /**
         * The default capacity for a `BPE`'s internal cache.
         */
        this.cache_capacity = 10000;
        this.cache = new LRUCache(this.cache_capacity);
    }
Create a BPE instance.
@param {Object} config The configuration object for BPE.
@param {Object} config.vocab A mapping of tokens to ids.
@param {string[]|[string, string][]} config.merges An array of BPE merges as strings.
@param {string} config.unk_token The unknown token used for out of vocabulary words.
@param {string} config.end_of_word_suffix The suffix to place at the end of each word.
@param {string} [config.continuing_subword_suffix] The suffix to insert between words.
@param {boolean} [config.byte_fallback=false] Whether to use spm byte-fallback trick (defaults to False)
@param {boolean} [config.ignore_merges=false] Whether or not to match tokens with the vocab before using merges.