diff --git a/examples/README.md b/examples/README.md index 727b27c80b..f985d5529a 100644 --- a/examples/README.md +++ b/examples/README.md @@ -107,13 +107,13 @@ CUDA_VISIBLE_DEVICES=0 llamafactory-cli train examples/qlora_single_gpu/llama3_l ### LoRA Fine-Tuning on Multiple GPUs -#### Supervised Fine-Tuning with Accelerate on Single Node +#### Supervised Fine-Tuning on Single Node ```bash CUDA_VISIBLE_DEVICES=0,1,2,3 llamafactory-cli train examples/lora_multi_gpu/llama3_lora_sft.yaml ``` -#### Supervised Fine-Tuning with Accelerate on Multiple Nodes +#### Supervised Fine-Tuning on Multiple Nodes ```bash CUDA_VISIBLE_DEVICES=0,1,2,3 NNODES=2 RANK=0 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/lora_multi_gpu/llama3_lora_sft.yaml @@ -136,13 +136,13 @@ ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 llamafactory-cli train examples/lora_multi_npu ### Full-Parameter Fine-Tuning on Multiple GPUs -#### Supervised Fine-Tuning with Accelerate on Single Node +#### Supervised Fine-Tuning on Single Node ```bash CUDA_VISIBLE_DEVICES=0,1,2,3 llamafactory-cli train examples/full_multi_gpu/llama3_full_sft.yaml ``` -#### Supervised Fine-Tuning with Accelerate on Multiple Nodes +#### Supervised Fine-Tuning on Multiple Nodes ```bash CUDA_VISIBLE_DEVICES=0,1,2,3 NNODES=2 RANK=0 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/full_multi_gpu/llama3_full_sft.yaml diff --git a/examples/README_zh.md b/examples/README_zh.md index 6974faa905..cf5bbf49f3 100644 --- a/examples/README_zh.md +++ b/examples/README_zh.md @@ -107,13 +107,13 @@ CUDA_VISIBLE_DEVICES=0 llamafactory-cli train examples/qlora_single_gpu/llama3_l ### 多 GPU LoRA 微调 -#### 使用 Accelerate 进行单节点训练 +#### 在单机上进行指令监督微调 ```bash CUDA_VISIBLE_DEVICES=0,1,2,3 llamafactory-cli train examples/lora_multi_gpu/llama3_lora_sft.yaml ``` -#### 使用 Accelerate 进行多节点训练 +#### 在多机上进行指令监督微调 ```bash CUDA_VISIBLE_DEVICES=0,1,2,3 NNODES=2 RANK=0 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/lora_multi_gpu/llama3_lora_sft.yaml @@ -128,7 +128,7 @@ CUDA_VISIBLE_DEVICES=0,1,2,3 llamafactory-cli train examples/lora_multi_gpu/llam ### 多 NPU LoRA 微调 -#### 使用 DeepSpeed ZeRO-0 训练 +#### 使用 DeepSpeed ZeRO-0 进行指令监督微调 ```bash ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 llamafactory-cli train examples/lora_multi_npu/llama3_lora_sft_ds.yaml @@ -136,13 +136,13 @@ ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 llamafactory-cli train examples/lora_multi_npu ### 多 GPU 全参数微调 -#### 使用 DeepSpeed 进行单节点训练 +#### 在单机上进行指令监督微调 ```bash CUDA_VISIBLE_DEVICES=0,1,2,3 llamafactory-cli train examples/full_multi_gpu/llama3_full_sft.yaml ``` -#### 使用 DeepSpeed 进行多节点训练 +#### 在多机上进行指令监督微调 ```bash CUDA_VISIBLE_DEVICES=0,1,2,3 NNODES=2 RANK=0 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/full_multi_gpu/llama3_full_sft.yaml diff --git a/examples/extras/badam/llama3_lora_sft.yaml b/examples/extras/badam/llama3_lora_sft.yaml index 4a482749f0..242e63ab1e 100644 --- a/examples/extras/badam/llama3_lora_sft.yaml +++ b/examples/extras/badam/llama3_lora_sft.yaml @@ -28,10 +28,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 pure_bf16: true ### eval diff --git a/examples/extras/fsdp_qlora/llama3_lora_sft.yaml b/examples/extras/fsdp_qlora/llama3_lora_sft.yaml index e9c04fa927..920d8fdb19 100644 --- a/examples/extras/fsdp_qlora/llama3_lora_sft.yaml +++ b/examples/extras/fsdp_qlora/llama3_lora_sft.yaml @@ -29,10 +29,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/extras/galore/llama3_full_sft.yaml b/examples/extras/galore/llama3_full_sft.yaml index 87381fcc11..3db31fed05 100644 --- a/examples/extras/galore/llama3_full_sft.yaml +++ b/examples/extras/galore/llama3_full_sft.yaml @@ -29,10 +29,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 1 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 pure_bf16: true ### eval diff --git a/examples/extras/llama_pro/llama3_freeze_sft.yaml b/examples/extras/llama_pro/llama3_freeze_sft.yaml index 8ace8db8cc..214f411ae6 100644 --- a/examples/extras/llama_pro/llama3_freeze_sft.yaml +++ b/examples/extras/llama_pro/llama3_freeze_sft.yaml @@ -27,10 +27,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/extras/loraplus/llama3_lora_sft.yaml b/examples/extras/loraplus/llama3_lora_sft.yaml index 26c2b1d2d5..9936bcd308 100644 --- a/examples/extras/loraplus/llama3_lora_sft.yaml +++ b/examples/extras/loraplus/llama3_lora_sft.yaml @@ -26,10 +26,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/extras/mod/llama3_full_sft.yaml b/examples/extras/mod/llama3_full_sft.yaml index 6b724ed0ee..edfec44e08 100644 --- a/examples/extras/mod/llama3_full_sft.yaml +++ b/examples/extras/mod/llama3_full_sft.yaml @@ -26,10 +26,10 @@ overwrite_output_dir: true per_device_train_batch_size: 1 gradient_accumulation_steps: 8 optim: paged_adamw_8bit -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 pure_bf16: true ### eval diff --git a/examples/full_multi_gpu/llama3_full_sft.yaml b/examples/full_multi_gpu/llama3_full_sft.yaml index a96f1b8e5b..b8873e3ace 100644 --- a/examples/full_multi_gpu/llama3_full_sft.yaml +++ b/examples/full_multi_gpu/llama3_full_sft.yaml @@ -28,10 +28,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 2 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/lora_multi_gpu/llama3_lora_sft.yaml b/examples/lora_multi_gpu/llama3_lora_sft.yaml index 6389f21b31..5e5dd9e6e6 100644 --- a/examples/lora_multi_gpu/llama3_lora_sft.yaml +++ b/examples/lora_multi_gpu/llama3_lora_sft.yaml @@ -28,10 +28,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 2 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/lora_multi_gpu/llama3_lora_sft_ds.yaml b/examples/lora_multi_gpu/llama3_lora_sft_ds.yaml index 6011896af9..e8dee21637 100644 --- a/examples/lora_multi_gpu/llama3_lora_sft_ds.yaml +++ b/examples/lora_multi_gpu/llama3_lora_sft_ds.yaml @@ -29,10 +29,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 2 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/lora_multi_npu/llama3_lora_sft_ds.yaml b/examples/lora_multi_npu/llama3_lora_sft_ds.yaml index 65ab6347d1..825b845033 100644 --- a/examples/lora_multi_npu/llama3_lora_sft_ds.yaml +++ b/examples/lora_multi_npu/llama3_lora_sft_ds.yaml @@ -29,10 +29,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 2 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/lora_single_gpu/llama3_lora_dpo.yaml b/examples/lora_single_gpu/llama3_lora_dpo.yaml index f68244b7e1..62752e5739 100644 --- a/examples/lora_single_gpu/llama3_lora_dpo.yaml +++ b/examples/lora_single_gpu/llama3_lora_dpo.yaml @@ -27,10 +27,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.000005 +learning_rate: 5.0e-6 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/lora_single_gpu/llama3_lora_kto.yaml b/examples/lora_single_gpu/llama3_lora_kto.yaml index 4405aaecd6..6f68981850 100644 --- a/examples/lora_single_gpu/llama3_lora_kto.yaml +++ b/examples/lora_single_gpu/llama3_lora_kto.yaml @@ -25,10 +25,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.000005 +learning_rate: 5.0e-6 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/lora_single_gpu/llama3_lora_ppo.yaml b/examples/lora_single_gpu/llama3_lora_ppo.yaml index 88ce24f3e0..19e7ccb370 100644 --- a/examples/lora_single_gpu/llama3_lora_ppo.yaml +++ b/examples/lora_single_gpu/llama3_lora_ppo.yaml @@ -26,10 +26,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.00001 +learning_rate: 1.0e-5 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### generate diff --git a/examples/lora_single_gpu/llama3_lora_pretrain.yaml b/examples/lora_single_gpu/llama3_lora_pretrain.yaml index acb18ebf43..54c5d89aec 100644 --- a/examples/lora_single_gpu/llama3_lora_pretrain.yaml +++ b/examples/lora_single_gpu/llama3_lora_pretrain.yaml @@ -24,10 +24,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/lora_single_gpu/llama3_lora_reward.yaml b/examples/lora_single_gpu/llama3_lora_reward.yaml index 6bf2ca0248..c82f9414c7 100644 --- a/examples/lora_single_gpu/llama3_lora_reward.yaml +++ b/examples/lora_single_gpu/llama3_lora_reward.yaml @@ -25,10 +25,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.00001 +learning_rate: 1.0e-5 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/lora_single_gpu/llama3_lora_sft.yaml b/examples/lora_single_gpu/llama3_lora_sft.yaml index 5492bc3421..429cb6afda 100644 --- a/examples/lora_single_gpu/llama3_lora_sft.yaml +++ b/examples/lora_single_gpu/llama3_lora_sft.yaml @@ -25,10 +25,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/lora_single_gpu/llava1_5_lora_sft.yaml b/examples/lora_single_gpu/llava1_5_lora_sft.yaml index 8e4226da1d..acab488481 100644 --- a/examples/lora_single_gpu/llava1_5_lora_sft.yaml +++ b/examples/lora_single_gpu/llava1_5_lora_sft.yaml @@ -26,10 +26,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/qlora_single_gpu/llama3_lora_sft_aqlm.yaml b/examples/qlora_single_gpu/llama3_lora_sft_aqlm.yaml index d2658051e7..53cc12e285 100644 --- a/examples/qlora_single_gpu/llama3_lora_sft_aqlm.yaml +++ b/examples/qlora_single_gpu/llama3_lora_sft_aqlm.yaml @@ -25,10 +25,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/qlora_single_gpu/llama3_lora_sft_awq.yaml b/examples/qlora_single_gpu/llama3_lora_sft_awq.yaml index ba6d8ea595..1a92f8228b 100644 --- a/examples/qlora_single_gpu/llama3_lora_sft_awq.yaml +++ b/examples/qlora_single_gpu/llama3_lora_sft_awq.yaml @@ -25,10 +25,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/qlora_single_gpu/llama3_lora_sft_bitsandbytes.yaml b/examples/qlora_single_gpu/llama3_lora_sft_bitsandbytes.yaml index a3db35ff19..c7f72c66d7 100644 --- a/examples/qlora_single_gpu/llama3_lora_sft_bitsandbytes.yaml +++ b/examples/qlora_single_gpu/llama3_lora_sft_bitsandbytes.yaml @@ -26,10 +26,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/examples/qlora_single_gpu/llama3_lora_sft_gptq.yaml b/examples/qlora_single_gpu/llama3_lora_sft_gptq.yaml index cc9a454e82..45caf17cc1 100644 --- a/examples/qlora_single_gpu/llama3_lora_sft_gptq.yaml +++ b/examples/qlora_single_gpu/llama3_lora_sft_gptq.yaml @@ -25,10 +25,10 @@ overwrite_output_dir: true ### train per_device_train_batch_size: 1 gradient_accumulation_steps: 8 -learning_rate: 0.0001 +learning_rate: 1.0e-4 num_train_epochs: 3.0 lr_scheduler_type: cosine -warmup_steps: 0.1 +warmup_ratio: 0.1 fp16: true ### eval diff --git a/src/llamafactory/hparams/model_args.py b/src/llamafactory/hparams/model_args.py index 995d5f12b6..7003cbeead 100644 --- a/src/llamafactory/hparams/model_args.py +++ b/src/llamafactory/hparams/model_args.py @@ -107,7 +107,7 @@ class ModelArguments: ) vllm_maxlen: int = field( default=2048, - metadata={"help": "Maximum sequence length of the vLLM engine (including prompt and output)."}, + metadata={"help": "Maximum sequence (prompt + response) length of the vLLM engine."}, ) vllm_gpu_util: float = field( default=0.9,