Skip to content

Commit

Permalink
Support Multiple Content Features for VitsSVC (#95)
Browse files Browse the repository at this point in the history 
Fix bugs for training and resuming of VitsSVC; Support the usage of multiple content features
  • Loading branch information
@viewfinder-annn
viewfinder-annn authored Jan 22, 2024
1 parent 9da5a24 commit eafa50d
Show file tree
Hide file tree
Showing 8 changed files with 381 additions and 173 deletions.
73 changes: 47 additions & 26 deletions config/vitssvc.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,50 +3,69 @@
"model_type": "VITS",
"task_type": "svc",
"preprocess": {
"extract_phone": false,
// Config for features extraction
"extract_mel": true,
"extract_pitch": true,
"pitch_extractor": "parselmouth",
"extract_energy": true,
"extract_uv": true,
"extract_linear_spec": true,
"extract_audio": true,

"mel_min_max_norm": true,
// Config for features usage
"use_linear": true,
"use_mel": true,
"use_min_max_norm_mel": false,
"use_audio": true,
"use_frame_pitch": true,
"use_uv": true,
"use_spkid": true,
"use_contentvec": false,
"use_whisper": false,
"use_wenet": false,
"use_text": false,
"use_phone": true,
"use_phone": false,

"fmin": 0,
"fmax": null,
"fmax": 12000,
"f0_min": 50,
"f0_max": 1100,
// f0_bin in sovits
"pitch_bin": 256,
// filter_length in sovits
"n_fft": 2048,
"n_fft": 1024,
// hop_length in sovits
"hop_size": 512,
"hop_size": 256,
// win_length in sovits
"win_size": 2048,
"win_size": 1024,
"segment_size": 8192,
"n_mel": 100,
"sample_rate": 44100,
"sample_rate": 24000,

"mel_min_max_stats_dir": "mel_min_max_stats",
"whisper_dir": "whisper",
"contentvec_dir": "contentvec",
"wenet_dir": "wenet",
"mert_dir": "mert",

// Meta file
"train_file": "train.json",
"valid_file": "test.json",
"spk2id": "singers.json",
"utt2spk": "utt2singer"
},
"model": {
"condition_encoder": {
"merge_mode": "add",
"input_melody_dim": 1,
"use_log_f0": true,
"n_bins_melody": 256,
//# Quantization (0 for not quantization)
"output_melody_dim": 196,
"output_melody_dim": 384,
"input_loudness_dim": 1,
"use_log_loudness": false,
"use_log_loudness": true,
"n_bins_loudness": 256,
"output_loudness_dim": 196,
"output_loudness_dim": 384,
"use_whisper": false,
"use_contentvec": false,
"use_wenet": false,
Expand All @@ -55,25 +74,27 @@
"contentvec_dim": 256,
"mert_dim": 256,
"wenet_dim": 512,
"content_encoder_dim": 196,
"output_singer_dim": 196,
"content_encoder_dim": 384,
"singer_table_size": 512,
"output_content_dim": 196,
"use_spkid": true
"output_singer_dim": 384,
"output_content_dim": 384,
"use_spkid": true,

"pitch_max": 1100.0,
"pitch_min": 50.0,
},
"vits": {
"filter_channels": 256,
"gin_channels": 256,
"hidden_channels": 192,
"inter_channels": 192,
"hidden_channels": 384,
"inter_channels": 384,
"kernel_size": 3,
"n_flow_layer": 4,
"n_heads": 2,
"n_layers": 6,
"n_layers_q": 3,
"n_speakers": 512,
"p_dropout": 0.1,
"ssl_dim": 256,
"use_spectral_norm": false,
},
"generator": "hifigan",
Expand All @@ -86,10 +107,10 @@
11
],
"upsample_rates": [
8,8,2,2,2
8,8,2,2
],
"upsample_kernel_sizes": [
16,16,4,4,4
16,16,4,4
],
"upsample_initial_channel": 512,
"resblock_dilation_sizes": [
Expand All @@ -99,7 +120,7 @@
]
},
"melgan": {
"ratios": [8, 8, 2, 2, 2],
"ratios": [8, 8, 2, 2],
"ngf": 32,
"n_residual_layers": 3,
"num_D": 3,
Expand All @@ -112,10 +133,10 @@
"activation": "snakebeta",
"snake_logscale": true,
"upsample_rates": [
8,8,2,2,2,
8,8,2,2
],
"upsample_kernel_sizes": [
16,16,4,4,4,
16,16,4,4
],
"upsample_initial_channel": 512,
"resblock_kernel_sizes": [
Expand All @@ -133,10 +154,10 @@
"resblock": "1",
"harmonic_num": 8,
"upsample_rates": [
8,8,2,2,2,
8,8,2,2
],
"upsample_kernel_sizes": [
16,16,4,4,4,
16,16,4,4
],
"upsample_initial_channel": 768,
"resblock_kernel_sizes": [
Expand Down
86 changes: 15 additions & 71 deletions egs/svc/VitsSVC/exp_config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,34 +16,16 @@
"svcc": "[SVCC dataset path]",
"vctk": "[VCTK dataset path]"
},
"use_custom_dataset": [],
// TODO: Fill in the output log path. The default value is "Amphion/ckpts/svc"
"log_dir": "ckpts/svc",
"preprocess": {
// TODO: Fill in the output data path. The default value is "Amphion/data"
"processed_dir": "data",

"f0_min": 50,
"f0_max": 1100,
// f0_bin in sovits
"pitch_bin": 256,
// filter_length in sovits
"n_fft": 2048,
// hop_length in sovits
"hop_size": 512,
// win_length in sovits
"win_size": 2048,
"segment_size": 8192,
"n_mel": 100,
"sample_rate": 44100,
"sample_rate": 24000,

// Config for features extraction
"extract_mel": true,
"extract_pitch": true,
"pitch_extractor": "parselmouth",
"extract_energy": false,
"extract_uv": true,
"extract_linear_spec": true,
"extract_audio": true,
// contentvec
"extract_contentvec_feature": true,
"contentvec_sample_rate": 16000,
Expand All @@ -54,66 +36,48 @@
"whisper_sample_rate": 16000,
"whisper_frameshift": 0.01,
"whisper_downsample_rate": 2,
// wenet
"extract_wenet_feature": true,
"wenet_downsample_rate": 4,
"wenet_frameshift": 0.01,
"wenet_sample_rate": 16000,
// Fill in the content-based pretrained model's path
"contentvec_file": "pretrained/contentvec/checkpoint_best_legacy_500.pt",
"wenet_model_path": "pretrained/wenet/20220506_u2pp_conformer_exp/final.pt",
"wenet_config": "pretrained/wenet/20220506_u2pp_conformer_exp/train.yaml",
"whisper_model": "medium",
"whisper_model_path": "pretrained/whisper/medium.pt",
// Config for features usage
"use_mel": true,
"use_frame_pitch": true,
"use_uv": true,
"use_spkid": true,

"use_contentvec": true,
"use_whisper": true,
"use_text": false,
"use_phone": false,

"use_wenet": false,

// Extract content features using dataloader
"pin_memory": true,
"num_workers": 8,
"content_feature_batch_size": 16,
// Meta file
"train_file": "train.json",
"valid_file": "test.json",
"spk2id": "singers.json",
"utt2spk": "utt2singer"

},
"model": {
"condition_encoder": {
// Config for features usage
"merge_mode": "add",
"input_melody_dim": 1,
"use_log_f0": true,
"n_bins_melody": 256,
//# Quantization (0 for not quantization)
"output_melody_dim": 192,

"use_log_loudness": true,
"use_contentvec": true,
"use_whisper": true,
"use_mert": false,
"use_wenet": false,
"whisper_dim": 1024,
"contentvec_dim": 256,
"content_encoder_dim": 192,
"output_singer_dim": 192,
"singer_table_size": 512,
"output_content_dim": 192,
"use_spkid": true,

"pitch_max": 1100.0,
"pitch_min": 50.0,
"wenet_dim": 512,
},
"vits": {
"inter_channels": 192,
"hidden_channels": 192,
"inter_channels": 384,
"hidden_channels": 384,
"filter_channels": 256,
"n_heads": 2,
"n_layers": 6,
"kernel_size": 3,
"p_dropout": 0.1,
"ssl_dim": 256,
"n_flow_layer": 4,
"n_layers_q": 3,
"gin_channels": 256,
Expand All @@ -135,26 +99,6 @@
3,
2
],
"run_eval": [
true,
true
],
"adamw": {
"lr": 2.0e-4
},
"reducelronplateau": {
"factor": 0.8,
"patience": 30,
"min_lr": 1.0e-4
},
"dataloader": {
"num_worker": 8,
"pin_memory": true
},
"sampler": {
"holistic_shuffle": false,
"drop_last": true
}
},
"inference": {
"batch_size": 1,
Expand Down
Loading

0 comments on commit eafa50d

Please sign in to comment.