Fix Transformers 5 Trainer compatibility

FlagEmbedding/abc/finetune/embedder/AbsRunner.py

@@ -43,10 +43,11 @@ def __init__(
             os.path.exists(training_args.output_dir)
             and os.listdir(training_args.output_dir)
             and training_args.do_train
-            and not training_args.overwrite_output_dir
+            and training_args.resume_from_checkpoint is None
         ):
             raise ValueError(
-                f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
+                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
+                "Please use a new output directory or set --resume_from_checkpoint."
             )
 
         # Setup logging

FlagEmbedding/abc/finetune/embedder/AbsTrainer.py

@@ -1,4 +1,5 @@
 import logging
+import inspect
 from typing import Optional
 from abc import ABC, abstractmethod
 from transformers.trainer import Trainer
@@ -10,6 +11,39 @@ class AbsEmbedderTrainer(ABC, Trainer):
     """
     Abstract class for the trainer of embedder.
     """
+    def __init__(
+        self,
+        *args,
+        processing_class=None,
+        tokenizer=None,
+        **kwargs,
+    ):
+        """Initialize ``Trainer`` across Transformers API versions.
+
+        Transformers 5 renamed the ``tokenizer`` argument to
+        ``processing_class``.  FlagEmbedding accepts both names so callers
+        from older examples remain compatible with either Transformers API.
+        """
+        if processing_class is None:
+            processing_class = tokenizer
+
+        trainer_parameters = inspect.signature(Trainer.__init__).parameters
+        if "processing_class" in trainer_parameters:
+            kwargs["processing_class"] = processing_class
+        else:
+            kwargs["tokenizer"] = processing_class
+
+        super().__init__(*args, **kwargs)
+
+    def _save_processing_class(self, output_dir: str):
+        """Save the tokenizer/processor using the active Transformers API."""
+        processing_class = getattr(self, "processing_class", None)
+        if processing_class is None:
+            processing_class = getattr(self, "tokenizer", None)
+
+        if processing_class is not None and self.is_world_process_zero():
+            processing_class.save_pretrained(output_dir)
+
     @abstractmethod
     def _save(self, output_dir: Optional[str] = None, state_dict=None):
         pass

FlagEmbedding/abc/finetune/reranker/AbsRunner.py

@@ -43,10 +43,11 @@ def __init__(
             os.path.exists(training_args.output_dir)
             and os.listdir(training_args.output_dir)
             and training_args.do_train
-            and not training_args.overwrite_output_dir
+            and training_args.resume_from_checkpoint is None
         ):
             raise ValueError(
-                f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
+                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
+                "Please use a new output directory or set --resume_from_checkpoint."
             )
 
         # Setup logging

FlagEmbedding/abc/finetune/reranker/AbsTrainer.py

@@ -1,4 +1,5 @@
 import logging
+import inspect
 from typing import Optional
 from abc import ABC, abstractmethod
 from transformers.trainer import Trainer
@@ -10,6 +11,34 @@ class AbsRerankerTrainer(ABC, Trainer):
     """
     Abstract class for the trainer of reranker.
     """
+    def __init__(
+        self,
+        *args,
+        processing_class=None,
+        tokenizer=None,
+        **kwargs,
+    ):
+        """Initialize ``Trainer`` across Transformers API versions."""
+        if processing_class is None:
+            processing_class = tokenizer
+
+        trainer_parameters = inspect.signature(Trainer.__init__).parameters
+        if "processing_class" in trainer_parameters:
+            kwargs["processing_class"] = processing_class
+        else:
+            kwargs["tokenizer"] = processing_class
+
+        super().__init__(*args, **kwargs)
+
+    def _save_processing_class(self, output_dir: str):
+        """Save the tokenizer/processor using the active Transformers API."""
+        processing_class = getattr(self, "processing_class", None)
+        if processing_class is None:
+            processing_class = getattr(self, "tokenizer", None)
+
+        if processing_class is not None and self.is_world_process_zero():
+            processing_class.save_pretrained(output_dir)
+
     @abstractmethod
     def _save(self, output_dir: Optional[str] = None, state_dict=None):
         pass

FlagEmbedding/finetune/embedder/decoder_only/base/trainer.py

@@ -33,8 +33,7 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None):
         else:
             self.model.save(output_dir)
 
-        if self.tokenizer is not None and self.is_world_process_zero():
-            self.tokenizer.save_pretrained(output_dir)
+        self._save_processing_class(output_dir)
 
         torch.save(self.args, os.path.join(output_dir, "training_args.bin"))
 

FlagEmbedding/finetune/embedder/decoder_only/icl/trainer.py

@@ -33,8 +33,7 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None):
         else:
             self.model.save(output_dir)
 
-        if self.tokenizer is not None and self.is_world_process_zero():
-            self.tokenizer.save_pretrained(output_dir)
+        self._save_processing_class(output_dir)
 
         torch.save(self.args, os.path.join(output_dir, "training_args.bin"))
 

FlagEmbedding/finetune/embedder/encoder_only/base/trainer.py

@@ -32,8 +32,7 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None):
                 f'does not support save interface')
         else:
             self.model.save(output_dir)
-        if self.tokenizer is not None and self.is_world_process_zero():
-            self.tokenizer.save_pretrained(output_dir)
+        self._save_processing_class(output_dir)
 
         torch.save(self.args, os.path.join(output_dir, "training_args.bin"))
 

FlagEmbedding/finetune/embedder/encoder_only/m3/trainer.py

@@ -32,8 +32,7 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None):
                 f'does not support save interface')
         else:
             self.model.save(output_dir)
-        if self.tokenizer is not None and self.is_world_process_zero():
-            self.tokenizer.save_pretrained(output_dir)
+        self._save_processing_class(output_dir)
 
         torch.save(self.args, os.path.join(output_dir, "training_args.bin"))
 

FlagEmbedding/finetune/reranker/decoder_only/base/runner.py

@@ -89,7 +89,7 @@ def load_trainer(self) -> DecoderOnlyRerankerTrainer:
             args=self.training_args,
             train_dataset=self.train_dataset,
             data_collator=self.data_collator,
-            tokenizer=self.tokenizer
+            processing_class=self.tokenizer
         )
         return trainer
 

FlagEmbedding/finetune/reranker/decoder_only/base/trainer.py

@@ -35,8 +35,7 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None):
         else:
             self.model.save(output_dir)
 
-        if self.tokenizer is not None and self.is_world_process_zero():
-            self.tokenizer.save_pretrained(output_dir)
+        self._save_processing_class(output_dir)
 
         torch.save(self.args, os.path.join(output_dir, "training_args.bin"))
 

FlagEmbedding/finetune/reranker/decoder_only/layerwise/runner.py

@@ -90,7 +90,7 @@ def load_trainer(self) -> DecoderOnlyRerankerTrainer:
             args=self.training_args,
             train_dataset=self.train_dataset,
             data_collator=self.data_collator,
-            tokenizer=self.tokenizer
+            processing_class=self.tokenizer
         )
         return trainer
 

FlagEmbedding/finetune/reranker/decoder_only/layerwise/trainer.py

@@ -35,8 +35,7 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None):
         else:
             self.model.save(output_dir)
 
-        if self.tokenizer is not None and self.is_world_process_zero():
-            self.tokenizer.save_pretrained(output_dir)
+        self._save_processing_class(output_dir)
 
         torch.save(self.args, os.path.join(output_dir, "training_args.bin"))
 

FlagEmbedding/finetune/reranker/encoder_only/base/runner.py

@@ -71,6 +71,6 @@ def load_trainer(self) -> EncoderOnlyRerankerTrainer:
             args=self.training_args,
             train_dataset=self.train_dataset,
             data_collator=self.data_collator,
-            tokenizer=self.tokenizer
+            processing_class=self.tokenizer
         )
         return trainer

FlagEmbedding/finetune/reranker/encoder_only/base/trainer.py

@@ -30,8 +30,7 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None):
             raise NotImplementedError(f'MODEL {self.model.__class__.__name__} ' f'does not support save_pretrained interface')
         else:
             self.model.save_pretrained(output_dir)
-        if self.tokenizer is not None and self.is_world_process_zero():
-            self.tokenizer.save_pretrained(output_dir)
+        self._save_processing_class(output_dir)
 
         # Good practice: save your training arguments together with the trained model
         torch.save(self.args, os.path.join(output_dir, "training_args.bin"))

examples/README.md

@@ -104,13 +104,11 @@ torchrun --nproc_per_node 2 \
     --query_instruction_format '{}{}' \
     --knowledge_distillation False \
     --output_dir ./test_encoder_only_base_bge-large-en-v1.5 \
-    --overwrite_output_dir \
     --learning_rate 1e-5 \
     --fp16 \
     --num_train_epochs 1 \
     --per_device_train_batch_size 2 \
     --dataloader_drop_last True \
-    --warmup_ratio 0.1 \
     --gradient_checkpointing \
     --deepspeed ./finetune/ds_stage0.json \
     --logging_steps 1 \
@@ -120,6 +118,7 @@ torchrun --nproc_per_node 2 \
     --sentence_pooling_method cls \
     --normalize_embeddings True \
     --kd_loss_type kl_div
+# For transformers<=4.57.3, add --warmup_ratio 0.1; for transformers>=5.0.0, add --warmup_steps 0.1.
 ```
 
 ### 2. Reranker
@@ -137,19 +136,18 @@ torchrun --nproc_per_node 2 \
     --pad_to_multiple_of 8 \
     --knowledge_distillation False \
     --output_dir ./test_encoder_only_base_bge-reranker-large \
-    --overwrite_output_dir \
     --learning_rate 6e-5 \
     --fp16 \
     --num_train_epochs 1 \
     --per_device_train_batch_size 2 \
     --gradient_accumulation_steps 1 \
     --dataloader_drop_last True \
-    --warmup_ratio 0.1 \
     --gradient_checkpointing \
     --weight_decay 0.01 \
     --deepspeed ./finetune/ds_stage0.json \
     --logging_steps 1 \
     --save_steps 1000
+# For transformers<=4.57.3, add --warmup_ratio 0.1; for transformers>=5.0.0, add --warmup_steps 0.1.
 ```
 
 ## 5. Evaluation
@@ -187,4 +185,3 @@ python -m FlagEmbedding.evaluation.msmarco \
     --devices cuda:0 cuda:1 cuda:2 cuda:3 cuda:4 cuda:5 cuda:6 cuda:7 \
     --cache_dir ./cache/model
 ```
-

examples/finetune/embedder/README.md

@@ -184,13 +184,11 @@ torchrun --nproc_per_node 2 \
     --query_instruction_format '{}{}' \
     --knowledge_distillation False \
 	--output_dir ./test_encoder_only_base_bge-large-en-v1.5 \
-    --overwrite_output_dir \
     --learning_rate 1e-5 \
     --fp16 \
     --num_train_epochs 2 \
     --per_device_train_batch_size 2 \
     --dataloader_drop_last True \
-    --warmup_ratio 0.1 \
     --gradient_checkpointing \
     --deepspeed ../ds_stage0.json \
     --logging_steps 1 \
@@ -200,6 +198,7 @@ torchrun --nproc_per_node 2 \
     --sentence_pooling_method cls \
     --normalize_embeddings True \
     --kd_loss_type kl_div
+# For transformers<=4.57.3, add --warmup_ratio 0.1; for transformers>=5.0.0, add --warmup_steps 0.1.
 ```
 
 ### (2) bge-m3
@@ -223,13 +222,11 @@ torchrun --nproc_per_node 2 \
     --small_threshold 0 \
     --drop_threshold 0 \
     --output_dir ./test_encoder_only_m3_bge-m3_sd \
-    --overwrite_output_dir \
     --learning_rate 1e-5 \
     --fp16 \
     --num_train_epochs 2 \
     --per_device_train_batch_size 2 \
     --dataloader_drop_last True \
-    --warmup_ratio 0.1 \
     --gradient_checkpointing \
     --deepspeed ../ds_stage0.json \
     --logging_steps 1 \
@@ -243,6 +240,7 @@ torchrun --nproc_per_node 2 \
     --use_self_distill True \
     --fix_encoder False \
     --self_distill_start_step 0
+# For transformers<=4.57.3, add --warmup_ratio 0.1; for transformers>=5.0.0, add --warmup_steps 0.1.
 ```
 
 Here are some new arguments:
@@ -282,13 +280,11 @@ torchrun --nproc_per_node 2 \
     --small_threshold 0 \
     --drop_threshold 0 \
 	--output_dir ./test_decoder_only_base_bge-multilingual-gemma2_sd \
-    --overwrite_output_dir \
     --learning_rate 1e-4 \
     --fp16 \
     --num_train_epochs 1 \
     --per_device_train_batch_size 2 \
     --dataloader_drop_last True \
-    --warmup_ratio 0.1 \
     --gradient_checkpointing \
     --deepspeed ../ds_stage1.json \
     --logging_steps 1 \
@@ -298,6 +294,7 @@ torchrun --nproc_per_node 2 \
     --sentence_pooling_method last_token \
     --normalize_embeddings True \
     --kd_loss_type m3_kd_loss
+# For transformers<=4.57.3, add --warmup_ratio 0.1; for transformers>=5.0.0, add --warmup_steps 0.1.
 ```
 
 Here are some new arguments:
@@ -346,13 +343,11 @@ torchrun --nproc_per_node 2 \
     --retrieval_use_examples True \
     --icl_suffix_str '\n<response>' \
     --output_dir ./test_decoder_only_base_bge-en-icl_sd \
-    --overwrite_output_dir \
     --learning_rate 1e-4 \
     --fp16 \
     --num_train_epochs 1 \
     --per_device_train_batch_size 2 \
     --dataloader_drop_last True \
-    --warmup_ratio 0.1 \
     --gradient_checkpointing \
     --deepspeed ../ds_stage1.json \
     --logging_steps 1 \
@@ -362,6 +357,7 @@ torchrun --nproc_per_node 2 \
     --sentence_pooling_method last_token \
     --normalize_embeddings True \
     --kd_loss_type kl_div
+# For transformers<=4.57.3, add --warmup_ratio 0.1; for transformers>=5.0.0, add --warmup_steps 0.1.
 ```
 
 Here are some new arguments:
@@ -383,4 +379,3 @@ Here are some new arguments:
 - **`example_passage_max_len`**: The max length of example passage.
 - **`retrieval_use_examples`**: If passed, will use examples for retrieval.
 - **`icl_suffix_str`**: The suffix string for ICL dataset.
-

examples/finetune/embedder/decoder_only/base.sh

@@ -42,13 +42,11 @@ data_args="\
 
 training_args="\
     --output_dir ./test_decoder_only_base_bge-multilingual-gemma2 \
-    --overwrite_output_dir \
     --learning_rate 1e-4 \
     --fp16 \
     --num_train_epochs $num_train_epochs \
     --per_device_train_batch_size $per_device_train_batch_size \
     --dataloader_drop_last True \
-    --warmup_ratio 0.1 \
     --gradient_checkpointing \
     --deepspeed ../../ds_stage1.json \
     --logging_steps 1 \
@@ -60,6 +58,8 @@ training_args="\
     --kd_loss_type m3_kd_loss \
 "
 
+# For transformers<=4.57.3, add --warmup_ratio 0.1; for transformers>=5.0.0, add --warmup_steps 0.1.
+
 cmd="torchrun --nproc_per_node $num_gpus \
     -m FlagEmbedding.finetune.embedder.decoder_only.base \
     $model_args \

examples/finetune/embedder/decoder_only/base_same_dataset.sh

@@ -45,13 +45,11 @@ data_args="\
 
 training_args="\
     --output_dir ./test_decoder_only_base_bge-multilingual-gemma2_sd \
-    --overwrite_output_dir \
     --learning_rate 1e-4 \
     --fp16 \
     --num_train_epochs $num_train_epochs \
     --per_device_train_batch_size $per_device_train_batch_size \
     --dataloader_drop_last True \
-    --warmup_ratio 0.1 \
     --gradient_checkpointing \
     --deepspeed ../../ds_stage1.json \
     --logging_steps 1 \
@@ -63,6 +61,8 @@ training_args="\
     --kd_loss_type m3_kd_loss \
 "
 
+# For transformers<=4.57.3, add --warmup_ratio 0.1; for transformers>=5.0.0, add --warmup_steps 0.1.
+
 cmd="torchrun --nproc_per_node $num_gpus \
     -m FlagEmbedding.finetune.embedder.decoder_only.base \
     $model_args \