public-projects/MST-MIXER
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity Actions

Make code public

Browse source
This commit is contained in:
Adnen Abdessaied 2024-07-08 11:41:28 +02:00
commit 8e03ef1c38
49 changed files with 545354 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

118
config/avsd_bart_base.json Normal file
View file

@ -0,0 +1,118 @@
{
"_name_or_path": "bart-base",
"activation_dropout": 0.1,
"activation_function": "gelu",
"add_bias_logits": false,
"add_final_layer_norm": false,
"architectures": [
"BartModel"
],
"attention_dropout": 0.1,
"bos_token_id": 0,
"classif_dropout": 0.1,
"classifier_dropout": 0.0,
"d_model": 768,
"decoder_attention_heads": 12,
"decoder_ffn_dim": 3072,
"decoder_layerdrop": 0.0,
"decoder_layers": 6,
"decoder_start_token_id": 2,
"dropout": 0.1,
"early_stopping": true,
"encoder_attention_heads": 12,
"encoder_ffn_dim": 3072,
"encoder_layerdrop": 0.0,
"encoder_layers": 6,
"eos_token_id": 2,
"forced_eos_token_id": 2,
"forced_bos_token_id": 0,
"gradient_checkpointing": false,
"id2label": {
"0": "LABEL_0",
"1": "LABEL_1",
"2": "LABEL_2"
},
"init_std": 0.02,
"is_encoder_decoder": true,
"label2id": {
"LABEL_0": 0,
"LABEL_1": 1,
"LABEL_2": 2
},
"max_position_embeddings": 1024,
"model_type": "bart",
"no_repeat_ngram_size": 3,
"normalize_before": false,
"normalize_embedding": true,
"num_beams": 4,
"num_hidden_layers": 6,
"pad_token_id": 1,
"scale_embedding": false,
"task_specific_params": {
"summarization": {
"length_penalty": 1.0,
"max_length": 128,
"min_length": 12,
"num_beams": 4
},
"summarization_cnn": {
"length_penalty": 2.0,
"max_length": 142,
"min_length": 56,
"num_beams": 4
},
"summarization_xsum": {
"length_penalty": 1.0,
"max_length": 62,
"min_length": 11,
"num_beams": 6
}
},
"torch_dtype": "float32",
"transformers_version": "4.12.0.dev0",
"use_cache": true,
"vocab_size": 50265,
"d_i3d_flow": 2048,
"d_i3d_rgb": 2048,
"d_sam": 512,
"d_audio": 128,
"top_k": 10,
"num_nn": 4,
"gnn_type": "appnp",
"use_random_graphs": false,
"integrate_all_gnn_features": true,
"use_elbo_local": true,
"use_elbo_global": true,
"use_non_linear": true,
"gnn_K": 2,
"gnn_alpha": 0.1,
"num_modalities": 6,
"local_gnn_d_hidden": 768,
"global_gnn_d_hidden": 768,
"num_local_gnn_heads": 2,
"num_global_gnn_heads": 4,
"local_gnn_dropout": 0.1,
"global_gnn_dropout": 0.1,
"local_fc_dropout": 0.1,
"global_fc_dropout": 0.1,
"num_local_gnn_layers": 2,
"num_global_gnn_layers": 2,
"num_local_fc_layers": 2,
"num_global_fc_layers": 2,
"use_local_gnn_bn": true,
"use_global_gnn_bn": true,
"use_local_fc_bn": true,
"use_global_fc_bn": true,
"local_gnn_concat": true,
"global_gnn_concat": true,
"num_local_gr_learner_heads": 8,
"num_global_gr_learner_heads":8,
"init_adj_ratio": 0.5,
"adj_ratio": 0.5,
"alpha": 0.9,
"gnns_every": 2,
"num_layers_state_fc_decoder": 2,
"dropout_state_fc_decoder": 0.3
}

116
config/avsd_bart_large.json Normal file
View file

@ -0,0 +1,116 @@
{
"activation_dropout": 0.1,
"activation_function": "gelu",
"add_bias_logits": false,
"add_final_layer_norm": false,
"architectures": [
"BartModel"
],
"attention_dropout": 0.1,
"bos_token_id": 0,
"classif_dropout": 0.1,
"classifier_dropout": 0.0,
"d_model": 1024,
"decoder_attention_heads": 16,
"decoder_ffn_dim": 4096,
"decoder_layerdrop": 0.0,
"decoder_layers": 12,
"decoder_start_token_id": 2,
"dropout": 0.1,
"early_stopping": true,
"encoder_attention_heads": 16,
"encoder_ffn_dim": 4096,
"encoder_layerdrop": 0.0,
"encoder_layers": 12,
"eos_token_id": 2,
"forced_eos_token_id": 2,
"forced_bos_token_id": 0,
"gradient_checkpointing": false,
"id2label": {
"0": "LABEL_0",
"1": "LABEL_1",
"2": "LABEL_2"
},
"init_std": 0.02,
"is_encoder_decoder": true,
"label2id": {
"LABEL_0": 0,
"LABEL_1": 1,
"LABEL_2": 2
},
"max_position_embeddings": 1024,
"model_type": "bart",
"no_repeat_ngram_size": 3,
"normalize_before": false,
"num_beams": 4,
"num_hidden_layers": 12,
"pad_token_id": 1,
"scale_embedding": false,
"task_specific_params": {
"summarization": {
"length_penalty": 1.0,
"max_length": 128,
"min_length": 12,
"num_beams": 4
},
"summarization_cnn": {
"length_penalty": 2.0,
"max_length": 142,
"min_length": 56,
"num_beams": 4
},
"summarization_xsum": {
"length_penalty": 1.0,
"max_length": 62,
"min_length": 11,
"num_beams": 6
}
},
"transformers_version": "4.7.0.dev0",
"use_cache": true,
"vocab_size": 50265,
"d_i3d_flow": 2048,
"d_i3d_rgb": 2048,
"d_sam": 512,
"d_audio": 128,
"top_k": 10,
"num_nn": 4,
"gnn_type": "appnp",
"use_random_graphs": false,
"integrate_all_gnn_features": true,
"use_elbo_local": true,
"use_elbo_global": true,
"use_non_linear": true,
"gnn_K": 2,
"gnn_alpha": 0.1,
"num_modalities": 6,
"local_gnn_d_hidden": 1024,
"global_gnn_d_hidden": 1024,
"num_local_gnn_heads": 2,
"num_global_gnn_heads": 4,
"local_gnn_dropout": 0.1,
"global_gnn_dropout": 0.1,
"local_fc_dropout": 0.1,
"global_fc_dropout": 0.1,
"num_local_gnn_layers": 1,
"num_global_gnn_layers": 1,
"num_local_fc_layers": 1,
"num_global_fc_layers": 1,
"use_local_gnn_bn": true,
"use_global_gnn_bn": true,
"use_local_fc_bn": true,
"use_global_fc_bn": true,
"local_gnn_concat": true,
"global_gnn_concat": true,
"num_local_gr_learner_heads": 8,
"num_global_gr_learner_heads": 8,
"init_adj_ratio": 0.5,
"adj_ratio": 0.5,
"alpha": 0.9,
"gnns_every": 4,
"num_layers_state_fc_decoder": 2,
"dropout_state_fc_decoder": 0.3
}

94
config/mst_mixer.conf Normal file
View file

@ -0,0 +1,94 @@
mixer {
task = avsd
#################################################################################
# datasets
# avsd
avsd_processed = features/
avsd_train = raw_data/train_set4DSTC7-AVSD.json
avsd_val = raw_data/valid_set4DSTC7-AVSD.json
avsd_test_dstc7 = raw_data/test_set4DSTC7-AVSD.json
avsd_test_dstc8 = raw_data/test_set4DSTC8-AVSD.json
avsd_test_dstc10 = raw_data/test_set4DSTC10-AVSD.json
avsd_feature_path = features/
avsd_i3d_rgb = features/i3d_rgb
avsd_i3d_rgb_test = features/i3d_rgb_testset
avsd_i3d_flow = features/i3d_flow_all
avsd_i3d_flow_test = features/i3d_flow_testset
avsd_audio = features/vggish_all
avsd_audio_test = features/vggish_testset
avsd_objects = features/sam
avsd_objects_test = features/sam_testset
dstc = 7
# NextQA
nextqa_root = processed/next_qa/annotations
nextqa_vid_feat = processed/next_qa/vid_feat
#################################################################################
# Model
bart_size = large # base, large
avsd_bart_base_config = config/avsd_bart_base.json
avsd_bart_large_config = config/avsd_bart_large.json
nextqa_bart_large_config = config/nextqa_bart_large.json
#################################################################################
# Logging & Checkpointing
log_dir = logs
output_dir_dstc7 = output/dstc7
output_dir_dstc8 = output/dstc8
output_dir_dstc10 = output/dstc10
output_dir_nextqa = output/nextqa
max_ckpt_to_keep = 5
start_ckpt_for_generating = none
loads_start_path = false
next_logging_pct = 0.1
save_ckpt=true
skip_saving_ckpt = false
stop_epochs = -1
resets_min_val_loss = false
restarts = false
uses_new_optimizer = true
sets_new_lr = false
################################################################################
# Data processing
expand_rnd = false
cap_sum = cap_sum
add_state_tokens = true
bart_max_input_len = 1024
num_workers = 0
n_history = 3
caption_drop_rate = 0.0
vis_feat_length = 36
#################################################################################
# Training
dp_type = ddp
batch_size = 16
num_epochs = 12
warmup_ratio = 0.1
batch_multiply = 1
skip_eval = false
stop_epoch = -1
random_seed = 54
learning_rate_bart = 1e-5
learning_rate_other = 1e-4
min_lr = 0
clip_grad_value = 1.0
print_output = false
eval_first = false
overfit_size = -1
elbo_global_coeff = 100
elbo_local_coeff = 100
gen_coeff = 1
#################################################################################
# Generation
gen_batch_size = 1
beam_depth = 5
max_generation_length = 20
min_generation_length = 1
length_penalty = 0.3
#################################################################################
# Misc.
master_port = 5101
use_cpu = false
}

116
config/nextqa_bart_large.json Normal file
View file

@ -0,0 +1,116 @@
{
"activation_dropout": 0.1,
"activation_function": "gelu",
"add_bias_logits": false,
"add_final_layer_norm": false,
"architectures": [
"BartModel"
],
"attention_dropout": 0.1,
"bos_token_id": 0,
"classif_dropout": 0.1,
"classifier_dropout": 0.0,
"d_model": 1024,
"decoder_attention_heads": 16,
"decoder_ffn_dim": 4096,
"decoder_layerdrop": 0.0,
"decoder_layers": 12,
"decoder_start_token_id": 2,
"dropout": 0.1,
"early_stopping": true,
"encoder_attention_heads": 16,
"encoder_ffn_dim": 4096,
"encoder_layerdrop": 0.0,
"encoder_layers": 12,
"eos_token_id": 2,
"forced_eos_token_id": 2,
"forced_bos_token_id": 0,
"gradient_checkpointing": false,
"id2label": {
"0": "LABEL_0",
"1": "LABEL_1",
"2": "LABEL_2"
},
"init_std": 0.02,
"is_encoder_decoder": true,
"label2id": {
"LABEL_0": 0,
"LABEL_1": 1,
"LABEL_2": 2
},
"max_position_embeddings": 1024,
"model_type": "bart",
"no_repeat_ngram_size": 3,
"normalize_before": false,
"num_beams": 4,
"num_hidden_layers": 12,
"pad_token_id": 1,
"scale_embedding": false,
"task_specific_params": {
"summarization": {
"length_penalty": 1.0,
"max_length": 128,
"min_length": 12,
"num_beams": 4
},
"summarization_cnn": {
"length_penalty": 2.0,
"max_length": 142,
"min_length": 56,
"num_beams": 4
},
"summarization_xsum": {
"length_penalty": 1.0,
"max_length": 62,
"min_length": 11,
"num_beams": 6
}
},
"transformers_version": "4.7.0.dev0",
"use_cache": true,
"vocab_size": 50265,
"d_i3d_flow": 2048,
"d_i3d_rgb": 2048,
"d_sam": 512,
"d_audio": 128,
"top_k": 10,
"num_nn": 4,
"gnn_type": "appnp",
"use_random_graphs": false,
"integrate_all_gnn_features": true,
"use_elbo_local": true,
"use_elbo_global": true,
"use_non_linear": true,
"gnn_K": 2,
"gnn_alpha": 0.1,
"num_modalities": 3,
"local_gnn_d_hidden": 1024,
"global_gnn_d_hidden": 1024,
"num_local_gnn_heads": 2,
"num_global_gnn_heads": 4,
"local_gnn_dropout": 0.1,
"global_gnn_dropout": 0.1,
"local_fc_dropout": 0.1,
"global_fc_dropout": 0.1,
"num_local_gnn_layers": 1,
"num_global_gnn_layers": 1,
"num_local_fc_layers": 1,
"num_global_fc_layers": 1,
"use_local_gnn_bn": true,
"use_global_gnn_bn": true,
"use_local_fc_bn": true,
"use_global_fc_bn": true,
"local_gnn_concat": true,
"global_gnn_concat": true,
"num_local_gr_learner_heads": 8,
"num_global_gr_learner_heads": 8,
"init_adj_ratio": 0.5,
"adj_ratio": 0.5,
"alpha": 0.9,
"gnns_every": 4,
"num_layers_state_fc_decoder": 2,
"dropout_state_fc_decoder": 0.3
}