pg19_30bb_360m_2d_ss.yaml

io:
  name_model: 8k_30b_360m_2d_ss
  output_dir: /data/output < set this!
  dataset_dir: /data/datasets/pg19 < set this!
  dataset_id: pg19
  num_models_to_save: 5
  validate_amount: 100
  log_train_loss_amount: 1000
params:
  num_tokens: 256
  pad_token_id: 0
  input_seq_len: 8192
  seq_lens: [1024, 8]
  hidden_dims: [1024, 1024]
  num_layers: [28, 24]
  train_checkpoint_chunks: null
  block:
    d_state: 128
    d_conv: 4
    expand: 2
    headdim: 64
    pos_emb_type: null
train:
  target_elements: 30_000_000_000
  target_elements_strategy: sequence
  batch_size: 6
  shuffle_train: false
  learning_rate: 0.001
  gradient_clipping: 1
  gradient_accumulate_every: 8