Update tokenization_chatglm.py (#3)

- Update tokenization_chatglm.py (3f7e063eb632a7f8f93a7eab49a8e76fc59e638c)


Co-authored-by: haoyuanhuang <[email protected]>

tokenization_chatglm.py

@@ -62,14 +62,16 @@ class ChatGLM4Tokenizer(PreTrainedTokenizer):
         vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}
         vocab.update(self.added_tokens_encoder)
         return vocab
-
-    def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
+        
+    def convert_tokens_to_string(self, tokens: List[Union[bytes, str, int]]) -> str:
         """
         Converts a sequence of tokens in a single string.
         """
         text = ""
         temp = b""
         for t in tokens:
+            if isinstance(t, int):
+                t = chr(t)
             if isinstance(t, str):
                 if temp:
                     text += temp.decode("utf-8", errors="replace")
@@ -78,7 +80,7 @@ class ChatGLM4Tokenizer(PreTrainedTokenizer):
             elif isinstance(t, bytes):
                 temp += t
             else:
-                raise TypeError("token should only be of type types or str")
+                raise TypeError("token should only be of type int, bytes or str")
         if temp:
             text += temp.decode("utf-8", errors="replace")
         return text