put(value) {
        if (value.length > 1) {
            throw Error('TextStreamer only supports batch size of 1');
        }

        const is_prompt = this.next_tokens_are_prompt;
        if (is_prompt) {
            this.next_tokens_are_prompt = false;
            if (this.skip_prompt) return;
        }

        const tokens = value[0];
        this.token_callback_function?.(tokens)

        // Add the new token to the cache and decodes the entire thing.
        this.token_cache = mergeArrays(this.token_cache, tokens);
        const text = this.tokenizer.decode(this.token_cache, this.decode_kwargs);

        let printable_text;
        if (is_prompt || text.endsWith('\n')) {
            // After the symbol for a new line, we flush the cache.
            printable_text = text.slice(this.print_len);
            this.token_cache = [];
            this.print_len = 0;
        } else if (text.length > 0 && is_chinese_char(text.charCodeAt(text.length - 1))) {
            // If the last token is a CJK character, we print the characters.
            printable_text = text.slice(this.print_len);
            this.print_len += printable_text.length;
        } else {
            // Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words,
            // which may change with the subsequent token -- there are probably smarter ways to do this!)
            printable_text = text.slice(this.print_len, text.lastIndexOf(' ') + 1);
            this.print_len += printable_text.length;
        }

        this.on_finalized_text(printable_text, false);
    }
Receives tokens, decodes them, and prints them to stdout as soon as they form entire words.
@param {bigint[][]} value