constructor(tokenizer, {
        skip_prompt = false,
        callback_function = null,
        token_callback_function = null,
        skip_special_tokens = true,
        decode_kwargs = {},
        ...kwargs
    } = {}) {
        super();
        this.tokenizer = tokenizer;
        this.skip_prompt = skip_prompt;
        this.callback_function = callback_function ?? stdout_write;
        this.token_callback_function = token_callback_function;
        this.decode_kwargs = { skip_special_tokens, ...decode_kwargs, ...kwargs };

        // variables used in the streaming process
        this.token_cache = [];
        this.print_len = 0;
        this.next_tokens_are_prompt = true;
    }
@param {import('../tokenizers.js').PreTrainedTokenizer} tokenizer
@param {Object} options
@param {boolean} [options.skip_prompt=false] Whether to skip the prompt tokens
@param {boolean} [options.skip_special_tokens=true] Whether to skip special tokens when decoding
@param {function(string): void} [options.callback_function=null] Function to call when a piece of text is ready to display
@param {function(bigint[]): void} [options.token_callback_function=null] Function to call when a new token is generated
@param {Object} [options.decode_kwargs={}] Additional keyword arguments to pass to the tokenizer's decode method