initial commit

models/common/vqa_tools/aokvqa/heuristics/README.md

@@ -0,0 +1,11 @@
+## Heuristics
+
+```bash
+# These scripts accept the same arguments.
+# heuristics/random_unweighted.py
+# heuristics/random_weighted.py
+# heuristics/most_common_answer.py
+
+python heuristics/random_unweighted.py --aokvqa-dir ${AOKVQA_DIR} --split val --mc --out ${PREDS_DIR}/random-unweighted_val-mc.json
+# Exclude --mc for the direct answer setting
+```

models/common/vqa_tools/aokvqa/heuristics/most_common_answer.py

@@ -0,0 +1,39 @@
+import os
+import json
+import argparse
+import pathlib
+from collections import Counter
+
+from load_aokvqa import load_aokvqa
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+parser.add_argument('--split', type=str, choices=['train', 'val', 'test'], required=True)
+parser.add_argument('--mc', action='store_true', dest='multiple_choice')
+parser.add_argument('--out', type=argparse.FileType('w'), required=True, dest='output_file')
+args = parser.parse_args()
+
+
+train_set = load_aokvqa(args.aokvqa_dir, 'train')
+train_freq = dict(Counter(
+    [d['choices'][d['correct_choice_idx']] for d in train_set]
+))
+most_common_answer = max(train_freq.keys(), key=train_freq.get)
+
+##
+
+eval_set = load_aokvqa(args.aokvqa_dir, args.split)
+
+predictions = {}
+
+for d in eval_set:
+    q = d['question_id']
+    predictions[q] = most_common_answer
+
+    if args.multiple_choice:
+        choices = [c for c in d['choices'] if c in train_freq.keys()]
+        if len(choices) > 0:
+            predictions[q] = max(choices, key=train_freq.get)
+
+json.dump(predictions, args.output_file)

models/common/vqa_tools/aokvqa/heuristics/random_unweighted.py

@@ -0,0 +1,38 @@
+import os
+import json
+from random import seed, sample
+import argparse
+import pathlib
+
+from load_aokvqa import load_aokvqa
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+parser.add_argument('--split', type=str, choices=['train', 'val', 'test'], required=True)
+parser.add_argument('--mc', action='store_true', dest='multiple_choice')
+parser.add_argument('--out', type=argparse.FileType('w'), required=True, dest='output_file')
+args = parser.parse_args()
+
+seed(0)
+
+train_set = load_aokvqa(args.aokvqa_dir, 'train')
+
+if args.multiple_choice is False:
+    choices = list(set(
+        [d['choices'][d['correct_choice_idx']] for d in train_set]
+    ))
+
+##
+
+predictions = {}
+
+eval_set = load_aokvqa(args.aokvqa_dir, args.split)
+
+for d in eval_set:
+    q = d['question_id']
+    if args.multiple_choice:
+        choices = d['choices']
+    predictions[q] = sample(choices, 1)[0]
+
+json.dump(predictions, args.output_file)

models/common/vqa_tools/aokvqa/heuristics/random_weighted.py

@@ -0,0 +1,46 @@
+import os
+import json
+import numpy as np
+import argparse
+import pathlib
+from collections import Counter
+
+from load_aokvqa import load_aokvqa
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+parser.add_argument('--split', type=str, choices=['train', 'val', 'test'], required=True)
+parser.add_argument('--mc', action='store_true', dest='multiple_choice')
+parser.add_argument('--out', type=argparse.FileType('w'), required=True, dest='output_file')
+args = parser.parse_args()
+
+np.random.seed(0)
+
+train_set = load_aokvqa(args.aokvqa_dir, 'train')
+train_freq = dict(Counter(
+    [d['choices'][d['correct_choice_idx']] for d in train_set]
+))
+
+if args.multiple_choice is False:
+    choices = list(train_freq.keys())
+    probs = [f / len(train_set) for f in train_freq.values()]
+
+##
+
+predictions = {}
+
+eval_set = load_aokvqa(args.aokvqa_dir, args.split)
+
+for d in eval_set:
+    if args.multiple_choice:
+        choices = d['choices']
+        probs = [train_freq.get(c, 0) for c in choices]
+        if probs == [0, 0, 0, 0]:
+            probs = [1, 1, 1, 1]
+        probs = [p / sum(probs) for p in probs]
+
+    q = d['question_id']
+    predictions[q] = np.random.choice(choices, size=1, p=probs)[0]
+
+json.dump(predictions, args.output_file)