fix the hyperparameters for Electra-Large on squad (dmlc#1396)

* fix the hyperparameters for squad * typo * update
PawelGlomski-Intel · Oct 22, 2020 · 46c9d01 · 46c9d01
1 parent 149270a
commit 46c9d01
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/scripts/question_answering/commands/generate_commands.py b/scripts/question_answering/commands/generate_commands.py
@@ -61,7 +61,8 @@ def electra_large_cfg():
     cfg.model_name = 'google_electra_large'
     cfg.batch_size = 2
     cfg.num_accumulated = 4
-    cfg.lr = 1e-5
+    cfg.lr = 5e-5
+    cfg.max_grad_norm = 1
     cfg.layerwise_decay = 0.9
     return cfg
 

diff --git a/scripts/question_answering/commands/run_squad2_electra_large.sh b/scripts/question_answering/commands/run_squad2_electra_large.sh
@@ -6,11 +6,11 @@ MODEL_NAME=google_electra_large
 BATCH_SIZE=2
 NUM_ACCUMULATED=4
 EPOCHS=2
-LR=1e-05
+LR=5e-05
 WARMUP_RATIO=0.1
 WD=0
 MAX_SEQ_LENGTH=512
-MAX_GRAD_NORM=0.1
+MAX_GRAD_NORM=1
 LAYERWISE_DECAY=0.9
 
 # Prepare the Data