initial commit

models/common/vqa_tools/aokvqa/evaluation/eval_predictions.py

@@ -0,0 +1,97 @@
+import argparse
+import pathlib
+import json
+import glob
+
+from load_aokvqa import load_aokvqa
+
+
+def eval_aokvqa(dataset, preds, multiple_choice=False, strict=True):
+
+    if isinstance(dataset, list):
+        dataset = { dataset[i]['question_id'] : dataset[i] for i in range(len(dataset)) }
+
+    if multiple_choice is False:
+        dataset = {k:v for k,v in dataset.items() if v['difficult_direct_answer'] is False}
+
+    if strict:
+        dataset_qids = set(dataset.keys())
+        preds_qids = set(preds.keys())
+        assert dataset_qids.issubset(preds_qids)
+
+    # dataset = q_id (str) : dataset element (dict)
+    # preds = q_id (str) : prediction (str)
+
+    acc = []
+
+    for q in dataset.keys():
+        if q not in preds.keys():
+            acc.append(0.0)
+            continue
+
+        pred = preds[q]
+        choices = dataset[q]['choices']
+        direct_answers = dataset[q]['direct_answers']
+
+        ## Multiple Choice setting
+        if multiple_choice:
+            if strict:
+                assert pred in choices, 'Prediction must be a valid choice'
+            correct_choice_idx = dataset[q]['correct_choice_idx']
+            acc.append( float(pred == choices[correct_choice_idx]) )
+        ## Direct Answer setting
+        else:
+            num_match = sum([pred.lower() == da.lower() for da in direct_answers])
+            vqa_acc = min(1.0, num_match / 3.0)
+            acc.append(vqa_acc)
+
+    acc = sum(acc) / len(acc) * 100
+
+    return acc
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+    parser.add_argument('--split', type=str, choices=['train', 'val', 'test'], required=True)
+    parser.add_argument('--preds', type=str, required=True, dest='prediction_files')
+    args = parser.parse_args()
+
+    dataset = load_aokvqa(args.aokvqa_dir, args.split)
+
+    for prediction_file in glob.glob(args.prediction_files):
+        predictions = json.load(open(prediction_file, 'r'))
+
+        # Multiple choice
+
+        mc_predictions = {}
+
+        for q in predictions.keys():
+            if 'multiple_choice' in predictions[q].keys():
+                mc_predictions[q] = predictions[q]['multiple_choice']
+
+        if mc_predictions != {}:
+            mc_acc = eval_aokvqa(
+                dataset,
+                mc_predictions,
+                multiple_choice=True,
+                strict=False
+            )
+            print(prediction_file, 'MC', mc_acc)
+
+        # Direct Answer
+
+        da_predictions = {}
+
+        for q in predictions.keys():
+            if 'direct_answer' in predictions[q].keys():
+                da_predictions[q] = predictions[q]['direct_answer']
+
+        if da_predictions != {}:
+            da_acc = eval_aokvqa(
+                dataset,
+                da_predictions,
+                multiple_choice=False,
+                strict=False
+            )
+            print(prediction_file, 'DA', da_acc)

models/common/vqa_tools/aokvqa/evaluation/load_aokvqa.py

@@ -0,0 +1,13 @@
+import os
+import json
+
+
+def load_aokvqa(aokvqa_dir, split, version='v1p0'):
+    assert split in ['train', 'val', 'test', 'test_w_ans']
+    dataset = json.load(open(
+        os.path.join(aokvqa_dir, f"aokvqa_{version}_{split}.json")
+    ))
+    return dataset
+
+def get_coco_path(split, image_id, coco_dir):
+    return os.path.join(coco_dir, f"{split}2017", f"{image_id:012}.jpg")

models/common/vqa_tools/aokvqa/evaluation/prepare_predictions.py

@@ -0,0 +1,31 @@
+import argparse
+import pathlib
+import json
+
+from load_aokvqa import load_aokvqa
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+    parser.add_argument('--split', type=str, choices=['train', 'val', 'test'], required=True)
+    parser.add_argument('--mc', type=argparse.FileType('r'), dest='mc_pred_file')
+    parser.add_argument('--da', type=argparse.FileType('r'), dest='da_pred_file')
+    parser.add_argument('--out', type=argparse.FileType('w'), dest='output_file')
+    args = parser.parse_args()
+    assert args.mc_pred_file or args.da_pred_file
+
+    dataset = load_aokvqa(args.aokvqa_dir, args.split)
+    mc_preds = json.load(args.mc_pred_file) if args.mc_pred_file else None
+    da_preds = json.load(args.da_pred_file) if args.da_pred_file else None
+    predictions = {}
+
+    for d in dataset:
+        q = d['question_id']
+        predictions[q] = {}
+        if mc_preds and q in mc_preds.keys():
+            predictions[q]['multiple_choice'] = mc_preds[q]
+        if da_preds and q in da_preds.keys():
+            predictions[q]['direct_answer'] = da_preds[q]
+
+    json.dump(predictions, args.output_file)

models/common/vqa_tools/aokvqa/evaluation/remap_predictions.py

@@ -0,0 +1,44 @@
+import argparse
+import pathlib
+import json
+from tqdm import tqdm
+
+from sentence_transformers import SentenceTransformer
+from sentence_transformers.util import cos_sim
+
+from load_aokvqa import load_aokvqa
+
+
+def map_to_choices(dataset, predictions, device='cpu'):
+    if isinstance(dataset, list):
+        dataset = { dataset[i]['question_id'] : dataset[i] for i in range(len(dataset)) }
+
+    if all([p in dataset[q]['choices'] for q, p in predictions.items()]):
+        return predictions
+
+    model = SentenceTransformer('sentence-transformers/average_word_embeddings_glove.6B.300d')
+    model.to(device)
+    for q in tqdm(predictions.keys()):
+        choices = dataset[q]['choices']
+        if predictions[q] not in choices:
+            choice_embeddings = model.encode([predictions[q]] + choices, convert_to_tensor=True)
+            a_idx = cos_sim(choice_embeddings[0], choice_embeddings[1:]).argmax().item()
+            predictions[q] = choices[a_idx]
+
+    return predictions
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--aokvqa-dir', type=pathlib.Path, required=True, dest='aokvqa_dir')
+    parser.add_argument('--split', type=str, choices=['train', 'val', 'test'], required=True)
+    parser.add_argument('--pred', type=argparse.FileType('r'), required=True, dest='prediction_file')
+    parser.add_argument('--out', type=argparse.FileType('w'), required=True, dest='output_file')
+    args = parser.parse_args()
+
+
+    dataset = load_aokvqa(args.aokvqa_dir, args.split)
+    predictions = json.load(args.prediction_file)
+    predictions = map_to_choices(dataset, predictions)
+
+    json.dump(predictions, args.output_file)