Merge remote-tracking branch 'upstream/main'

red-hat-data-services · Sep 20, 2024 · 20d8000 · 20d8000
2 parents 6871548 + 0346d0e
commit 20d8000
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/tuning/sft_trainer.py b/tuning/sft_trainer.py
@@ -262,6 +262,14 @@ def train(
         if tokenizer.unk_token is None:
             logger.warning("UNK token set to default, missing in tokenizer")
             special_tokens_dict["unk_token"] = configs.DEFAULT_UNK_TOKEN
+        if tokenizer.pad_token == tokenizer.eos_token:
+            logger.warning(
+                "PAD token set to default, to make it different from eos token"
+            )
+            if tokenizer.eos_token != configs.DEFAULT_PAD_TOKEN:
+                tokenizer.pad_token = configs.DEFAULT_PAD_TOKEN
+            else:
+                tokenizer.eos_token = configs.DEFAULT_EOS_TOKEN
 
     # TODO: lower priority but understand if resizing impacts inference quality and why its needed.
     # It makes sense if we manipulate tokenizer that we also save it and provide it to inference.