Add train_clain_gpu_num

yuiseki · Mar 21, 2024 · 4d13953 · 4d13953
1 parent 86a26c9
commit 4d13953
Show file tree

Hide file tree

Showing 15 changed files with 54 additions and 4 deletions.
diff --git a/agent.sh b/agent.sh
@@ -53,5 +53,7 @@ for i in $(seq $GPU_NUM); do
 done
 echo "AVAILABLE_GPU_NUM: $AVAILABLE_GPU_NUM"
 echo "AVAILABLE_GPU_INDEX: ${AVAILABLE_GPU_INDEX[@]}"
-
+# AVAILABLE_GPU_INDEX: 5 6 7 8
+CUDA_VISIBLE_DEVICES=$(echo ${AVAILABLE_GPU_INDEX[@]} | sed 's/ /,/g')
+echo "CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES"
 
diff --git a/recipes/A5000_24GB_x8/color-coder.yaml b/recipes/A5000_24GB_x8/color-coder.yaml
@@ -10,6 +10,7 @@ dataset_train_split_test_size: 0.2
 lora_r: 8
 lora_alpha: 16
 lora_dropout: 0.05
+train_clain_gpu_num: 1
 train_per_device_train_batch_size: 32
 train_gradient_accumulation_steps: 16
 train_num_train_epochs: 4

diff --git a/recipes/A5000_24GB_x8/fake-news-detector.yaml b/recipes/A5000_24GB_x8/fake-news-detector.yaml
@@ -11,6 +11,7 @@ dataset_train_split_test_size: 0.2
 lora_r: 8
 lora_alpha: 16
 lora_dropout: 0.05
+train_clain_gpu_num: 1
 train_per_device_train_batch_size: 4
 train_gradient_accumulation_steps: 2
 train_num_train_epochs: 4

diff --git a/recipes/A5000_24GB_x8/hate-speech-detector.yaml b/recipes/A5000_24GB_x8/hate-speech-detector.yaml
@@ -11,6 +11,7 @@ dataset_train_split_test_size: 0.2
 lora_r: 8
 lora_alpha: 16
 lora_dropout: 0.05
+train_clain_gpu_num: 1
 train_per_device_train_batch_size: 8
 train_gradient_accumulation_steps: 4
 train_num_train_epochs: 4

diff --git a/recipes/A5000_24GB_x8/ja.yaml b/recipes/A5000_24GB_x8/ja.yaml
@@ -11,6 +11,7 @@ dataset_train_split_test_size: 0.2
 lora_r: 8
 lora_alpha: 16
 lora_dropout: 0.05
+train_clain_gpu_num: 4
 train_per_device_train_batch_size: 8
 train_gradient_accumulation_steps: 4
 train_num_train_epochs: 4
@@ -29,4 +30,4 @@ evaluations:
   -
     prompt: "格闘家ボブ・サップの出身国はどこでしょう？"
     context: ""
-    expected_output: "アメリカ"
+    expected_output: "アメリカ"
diff --git a/recipes/A5000_24GB_x8/sentiment-analyzer.yaml b/recipes/A5000_24GB_x8/sentiment-analyzer.yaml
@@ -11,6 +11,7 @@ dataset_train_split_test_size: 0.2
 lora_r: 8
 lora_alpha: 16
 lora_dropout: 0.05
+train_clain_gpu_num: 1
 train_per_device_train_batch_size: 6
 train_gradient_accumulation_steps: 2
 train_num_train_epochs: 4

diff --git a/recipes/A5000_24GB_x8/translator-en-ja-math.yaml b/recipes/A5000_24GB_x8/translator-en-ja-math.yaml
@@ -0,0 +1,33 @@
+target_task: tasks/nlp/translation.md
+base_model_id: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
+model_name: tinyllama-translator-en-ja-math-v1
+output_base_dir: output
+dataset_id: kunishou/OpenMathInstruct-1-1.8m-ja
+dataset_input_hint: Given the text, translate to Japanese.
+dataset_input_field_name: question
+dataset_output_field_name: question_ja
+dataset_train_split_seed: 42
+dataset_train_split_test_size: 0.2
+lora_r: 8
+lora_alpha: 16
+lora_dropout: 0.05
+train_clain_gpu_num: 4
+train_per_device_train_batch_size: 8
+train_gradient_accumulation_steps: 4
+train_num_train_epochs: 4
+train_max_steps: 1000
+train_fp16: True
+inference_max_new_tokens: 32
+evaluations:
+  -
+    prompt: "thank you"
+    expected_output: "ありがとう"
+  -
+    prompt: "Hello"
+    expected_output: "こんにちは"
+  -
+    prompt: "How are you?"
+    expected_output: "お元気ですか？"
+  -
+    prompt: "I am hungry"
+    expected_output: "お腹が空いています"
diff --git a/recipes/A5000_24GB_x8/translator-en-ja-oasst1.yaml b/recipes/A5000_24GB_x8/translator-en-ja-oasst1.yaml
@@ -1,5 +1,5 @@
-target_task: tasks/nlp/hate-speech-detection.md
-base_model_id: TinyLlama/TinyLlama-1.1B-Chat-v1.0
+target_task: tasks/nlp/translation.md
+base_model_id: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
 model_name: tinyllama-translator-en-ja-oasst1-v1
 output_base_dir: output
 dataset_id: kunishou/oasst1-89k-ja
@@ -13,6 +13,7 @@ dataset_train_split_test_size: 0.2
 lora_r: 8
 lora_alpha: 16
 lora_dropout: 0.05
+train_clain_gpu_num: 4
 train_per_device_train_batch_size: 8
 train_gradient_accumulation_steps: 4
 train_num_train_epochs: 4

diff --git a/recipes/RTX_3060_12GB/color-coder.yaml b/recipes/RTX_3060_12GB/color-coder.yaml
@@ -10,6 +10,7 @@ dataset_train_split_test_size: 0.2
 lora_r: 8
 lora_alpha: 16
 lora_dropout: 0.05
+train_clain_gpu_num: 1
 train_per_device_train_batch_size: 8
 train_gradient_accumulation_steps: 2
 train_num_train_epochs: 4

diff --git a/recipes/RTX_3060_12GB/fake-news-detector.yaml b/recipes/RTX_3060_12GB/fake-news-detector.yaml
@@ -11,6 +11,7 @@ dataset_train_split_test_size: 0.2
 lora_r: 8
 lora_alpha: 16
 lora_dropout: 0.05
+train_clain_gpu_num: 1
 train_per_device_train_batch_size: 2
 train_gradient_accumulation_steps: 2
 train_num_train_epochs: 4

diff --git a/recipes/RTX_3060_12GB/hate-speech-detector.yaml b/recipes/RTX_3060_12GB/hate-speech-detector.yaml
@@ -11,6 +11,7 @@ dataset_train_split_test_size: 0.2
 lora_r: 8
 lora_alpha: 16
 lora_dropout: 0.05
+train_clain_gpu_num: 1
 train_per_device_train_batch_size: 2
 train_gradient_accumulation_steps: 2
 train_num_train_epochs: 4

diff --git a/recipes/RTX_3060_12GB/onomatopoeia-ja.yaml b/recipes/RTX_3060_12GB/onomatopoeia-ja.yaml
@@ -10,6 +10,7 @@ dataset_train_split_test_size: 0.2
 lora_r: 8
 lora_alpha: 16
 lora_dropout: 0.05
+train_clain_gpu_num: 1
 train_per_device_train_batch_size: 8
 train_gradient_accumulation_steps: 2
 train_num_train_epochs: 4

diff --git a/recipes/RTX_3060_12GB/sentiment-analyzer.yaml b/recipes/RTX_3060_12GB/sentiment-analyzer.yaml
@@ -11,6 +11,7 @@ dataset_train_split_test_size: 0.2
 lora_r: 8
 lora_alpha: 16
 lora_dropout: 0.05
+train_clain_gpu_num: 1
 train_per_device_train_batch_size: 2
 train_gradient_accumulation_steps: 2
 train_num_train_epochs: 4

diff --git a/recipes/RTX_3060_12GB/sql-coder.yaml b/recipes/RTX_3060_12GB/sql-coder.yaml
@@ -12,6 +12,7 @@ dataset_train_split_test_size: 0.2
 lora_r: 8
 lora_alpha: 16
 lora_dropout: 0.05
+train_clain_gpu_num: 1
 train_per_device_train_batch_size: 8
 train_gradient_accumulation_steps: 2
 train_num_train_epochs: 4

diff --git a/src/loading.py b/src/loading.py
@@ -30,3 +30,6 @@ def load_yaml(file_path):
     except Exception as e:
         print(f"Error loading dataset: {e}")
         continue
+
+# wiki40b ja
+load_dataset('wiki40b', 'ja')