release code base

config/config.py

@@ -0,0 +1,26 @@
+import json
+import os
+
+def read_default_config():
+    dirpath = os.path.dirname(__file__)
+    path = os.path.join(dirpath, "default.json")
+    with open(path) as config_file:
+        config = json.load(config_file)
+    return config
+
+def read_config(path):
+    with open(path) as config_file:
+        config = json.load(config_file)
+    return config
+
+def update_nested_dicts(old_dict, update_dict):
+    for key in update_dict:
+        if key in old_dict:
+            old_dict[key].update(update_dict[key])
+        else:
+            old_dict[key] = update_dict[key]
+    return old_dict
+
+
+
+ 

config/default.json

@@ -0,0 +1,43 @@
+{
+    "wandb": {
+        "entity": "TO_BE_DEFINED",
+        "name": "",
+        "group": "",
+        "tags": [],
+        "project": "olvit"
+
+    },
+    "model": {
+        "model_type": "base_model",
+        "feature_type": "none", 
+        "freeze_roberta": true,
+        "v_emb_dim": 16,
+        "dim_feedforward": 400,
+        "n_heads": 9,
+        "fc_dim": 128,
+        "dropout_p": 0.1,
+        "sample_rate_video": 10,
+        "n_encoder_layers": 6,
+        "add_choices_as_context": false,
+        "use_pretrained_lm": false,
+        "projection_as_in_aloe": false,
+        "pretrained_lm_name": ""
+    },
+    "training": {
+        "lr": 1e-4,
+        "total_steps": 200000,
+        "warmup_steps": 4000,
+        "accumulate_grad_batches": 1,
+        "batch_size": 128,
+        "epochs": 40, 
+        "seed": null
+    },
+    "datamodule": {
+        "fea_dir": "data/dvd/monet_feats/",
+        "data_dir": "data/dvd/dialogs/"
+    }, 
+    "checkpoint": {
+        "checkpoint_folder": "checkpoints/",
+        "checkpoint_file_name": "olvit"
+    }
+}

config/dvd.json

@@ -0,0 +1,49 @@
+{
+    "wandb": {
+        "name": "olvit",
+        "group": "dvd",
+        "tags": [],
+        "project": "olvit"
+
+    },
+    "model": {
+        "model_type": "discriminative",
+        "n_heads": 6,
+        "v_emb_dim": 36,
+        "dim_feedforward": 200,
+        "dropout_p": 0.1,
+        "fc_dim": 512,
+        "sample_rate_video": 20,
+        "n_transf_layers": 4,
+        "use_pretrained_lm": true,
+        "projection_as_in_aloe": true,
+        "pretrained_lm_name": "distilroberta-base",
+        "dataset": "dvd"
+    },
+    "extended_model": {
+        "hist_len_for_state_gen": 7,
+        "number_of_relevant_emb": 2,
+        "num_layers_v_state": 2,
+        "num_layers_d_state": 2,
+        "combiner_option": "OptionA",
+        "state_tracker_type": "Transformer",
+        "use_v_state": true,
+        "use_d_state": true,
+        "n_heads_combiner_transformer": 8,
+        "n_heads_state_tracker": 6,
+        "dim_feedforward_v_transformer": 140,
+        "dim_feedforward_d_transformer": 60
+    },
+    "training": {
+        "lr": 1e-4,
+        "warmup_steps": 4000,
+        "total_steps": 200000,
+        "batch_size": 128,
+        "seed": 12345,
+        "epochs": 1000 
+    },
+    "checkpoint": {
+        "checkpoint_folder": "checkpoints/dvd",
+        "checkpoint_file_name": "olvit"
+    }
+}

config/simmc.json

@@ -0,0 +1,61 @@
+{
+    "wandb": {
+        "name": "olvit",
+        "group": "simmc2",
+        "tags": [],
+        "project": "olvit"
+
+    },
+    "model": {
+        "model_type": "generative",
+        "dataset": "simmc2",
+        "feature_type": "object_text_features",
+        "object_feature_generator_dim": 50,
+        "n_object_feature_generator_layers": 2,        
+        "n_heads": 6,
+        "v_emb_dim": 516,
+        "emb_dim": 216,
+        "dim_feedforward": 200,
+        "dropout_p": 0.1,
+        "fc_dim": 512,
+        "sample_rate_video": 1,
+        "n_encoder_layers": 4,
+        "n_decoder_layers": 4,
+        "use_pretrained_lm": true,
+        "vocab_size": 50265,
+        "projection_as_in_aloe": false,
+        "pretrained_lm_name": "distilroberta-base"
+    },
+    "extended_model": {
+        "hist_len_for_state_gen": 3,
+        "number_of_relevant_emb": 2,
+        "num_layers_v_state": 2,
+        "num_layers_d_state": 2,
+        "combiner_option": "OptionA",
+        "state_tracker_type": "Transformer",
+        "use_v_state": true,
+        "use_d_state": true,
+        "n_heads_combiner_transformer": 8,
+        "n_heads_state_tracker": 6,
+        "dim_feedforward_v_transformer": 140,
+        "dim_feedforward_d_transformer": 60
+    },
+    "training": {
+        "lr": 1e-4,
+        "warmup_steps": 4000,
+        "total_steps": 200000,
+        "batch_size": 8,
+        "seed": 12345,
+        "epochs": 1000 
+    },
+    "datamodule": {
+        "fea_dir": "data/simmc/visual_features_resnet50_simmc2.1.pt",
+        "data_dir": "data/simmc/dialogs"
+    }, 
+    "checkpoint": {
+        "checkpoint_folder": "checkpoints/simmc/",
+        "checkpoint_file_name": "olvit",
+        "output_path": "output/simmc/",
+        "checkpoint_path": "TO_BE_DETERMINED"
+    }
+}