Update gptx_tokenizer.py

gptx_tokenizer.py

@@ -245,10 +245,9 @@ class HFGPTXTokenizer(PreTrainedTokenizer):
         """
         output = self.tok.decode(input=token_ids, num_threads=num_threads)
         if skip_special_tokens:
-            warnings.warn(
-                "skip_special_tokens currently not implemented"
-            )
-                    
+            for substring in self.additional_special_tokens:
+                output = output.replace(substring, "")
+        
         if clean_up_tokenization_spaces:
             warnings.warn(
                 "when cleaning up tokenization spaces, this will not behave "
@@ -259,6 +258,7 @@ class HFGPTXTokenizer(PreTrainedTokenizer):
         
         return output
 
+    
     def _convert_id_to_token(self, index: int) -> str:
         """
         Convert a token ID to its corresponding token string.