public-projects/V2Dial
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity Actions

initial commit

Browse source
This commit is contained in:
Andreas Bulling 2025-07-10 07:31:58 +02:00
commit 7be61f8c6d
137 changed files with 33491 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

43
config/ds_config.json Executable file
View file

@ -0,0 +1,43 @@
{
"train_micro_batch_size_per_gpu": 8,
"gradient_accumulation_steps": 1,
"optimizer":{
"type": "Adam",
"params": {
"lr": 1e-4,
"betas": [0.9, 0.999],
"weight_decay": 0.02,
"adam_w_mode": true
}
},
"scheduler":{
"type": "WarmupCosineLR",
"params":{}
},
"fp16": {
"enabled": false,
"auto_cast": false,
"loss_scale": 0,
"initial_scale_power": 16,
"loss_scale_window": 1000,
"hysteresis": 2,
"consecutive_hysteresis": false,
"min_loss_scale": 1
},
"zero_optimization": {
"stage": 0,
"overlap_comm": true,
"contiguous_gradients": true,
"sub_group_size": 1e9,
"reduce_bucket_size": "auto",
"stage3_prefetch_bucket_size": "auto",
"stage3_param_persistence_threshold": "auto",
"stage3_max_live_parameters": 1e9,
"stage3_max_reuse_distance": 1e9,
"stage3_gather_16bit_weights_on_model_save": true
},
"wandb":{
"enabled": true,
"project": "V2Dial"
}
}