encode(tokens) {
        const outputTokens = [];

        for (const token of tokens) {
            if (this.ignore_merges && this.tokens_to_ids.has(token)) {
                outputTokens.push(token);
                continue;
            }
            const bpe_token_list = this.bpe(token);

            for (const t of bpe_token_list) {
                if (this.tokens_to_ids.has(t)) {
                    outputTokens.push(t);
                } else if (this.byte_fallback) {
                    const byteTokens = Array.from(this.text_encoder.encode(t))
                        .map(x => `<0x${x.toString(16).toUpperCase().padStart(2, '0')}>`);
                    if (byteTokens.every(x => this.tokens_to_ids.has(x))) {
                        // Ensure the byte tokens are actually in the vocabulary, otherwise
                        // we fall back to the unknown token. For more information, see
                        // https://github.com/huggingface/transformers/issues/28096.
                        outputTokens.push(...byteTokens);
                    } else {
                        outputTokens.push(this.unk_token);
                    }
                } else {
                    outputTokens.push(this.unk_token);
                }
            }
        }

        return outputTokens;
    }
Encodes the input sequence of tokens using the BPE algorithm and returns the resulting subword tokens.
@param {string[]} tokens The input sequence of tokens to encode.
@returns {string[]} The resulting subword tokens after applying the BPE algorithm to the input sequence of tokens.