neuro-symbolic-visual-dialog/prog_generator/train_question_parser.py

913 lines
38 KiB
Python
Raw Normal View History

2022-08-10 16:49:55 +02:00
"""
author: Adnen Abdessaied
maintainer: "Adnen Abdessaied"
website: adnenabdessaied.de
version: 1.0.1
"""
import os
import sys
import json, torch, pickle, copy, time
import numpy as np
import torch.nn as nn
import torch.utils.data as Data
from tensorboardX import SummaryWriter
from copy import deepcopy
from clevrDialog_dataset import ClevrDialogQuestionDataset
import pickle
from tqdm import tqdm
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from executor.symbolic_executor import SymbolicExecutorClevr, SymbolicExecutorMinecraft
from models import SeqToSeqQ, QuestEncoder_1, QuestEncoder_2, Decoder, CaptionEncoder, SeqToSeqC
from optim import get_optim, adjust_lr
from options_caption_parser import Options as OptionsC
from options_question_parser import Options as OptionsQ
class Execution:
def __init__(self, optsQ, optsC):
self.opts = deepcopy(optsQ)
if self.opts.useCuda > 0 and torch.cuda.is_available():
self.device = torch.device("cuda:0")
print("[INFO] Using GPU {} ...".format(torch.cuda.get_device_name(0)))
else:
print("[INFO] Using CPU ...")
self.device = torch.device("cpu")
self.loss_fn = torch.nn.NLLLoss().to(self.device)
print("[INFO] Loading dataset ...")
self.datasetTr = ClevrDialogQuestionDataset(
self.opts.dataPathTr, self.opts.vocabPath, "train", "All tr data")
self.datasetVal = ClevrDialogQuestionDataset(
self.opts.dataPathVal, self.opts.vocabPath, "val", "All val data", train=False)
self.datasetTest = ClevrDialogQuestionDataset(
self.opts.dataPathTest, self.opts.vocabPath, "test", "All val data", train=False)
self.QuestionNet = constructQuestionNet(
self.opts,
self.datasetTr.lenVocabText,
self.datasetTr.lenVocabProg,
self.datasetTr.maxLenProg,
)
if os.path.isfile(self.opts.captionNetPath):
self.CaptionNet = constructCaptionNet(
optsC,
self.datasetTr.lenVocabText,
self.datasetTr.lenVocabProg,
self.datasetTr.maxLenProg
)
print('Loading CaptionNet from {}'.format(self.opts.captionNetPath))
state_dict = torch.load(self.opts.captionNetPath)['state_dict']
self.CaptionNet.load_state_dict(state_dict)
self.CaptionNet.to(self.device)
total_params_cap = sum(p.numel() for p in self.CaptionNet.parameters() if p.requires_grad)
print("The caption encoder has {} trainable parameters".format(total_params_cap))
self.QuestionNet.to(self.device)
# if os.path.isfile(self.opts.load_checkpoint_path):
# print('Loading QuestionNet from {}'.format(optsQ.load_checkpoint_path))
# state_dict = torch.load(self.opts.load_checkpoint_path)['state_dict']
# self.QuestionNet.load_state_dict(state_dict)
total_params_quest = sum(p.numel() for p in self.QuestionNet.parameters() if p.requires_grad)
print("The question encoder has {} trainable parameters".format(total_params_quest))
if "minecraft" in self.opts.scenesPath:
self.symbolicExecutor = SymbolicExecutorMinecraft(self.opts.scenesPath)
else:
self.symbolicExecutor = SymbolicExecutorClevr(self.opts.scenesPath)
tb_path = os.path.join(self.opts.run_dir, "tb_logdir")
if not os.path.isdir(tb_path):
os.makedirs(tb_path)
self.ckpt_path = os.path.join(self.opts.run_dir, "ckpt_dir")
if not os.path.isdir(self.ckpt_path):
os.makedirs(self.ckpt_path)
if not os.path.isdir(self.opts.text_log_dir):
os.makedirs(self.opts.text_log_dir)
self.writer = SummaryWriter(tb_path)
self.iter_val = 0
if os.path.isfile(self.opts.dependenciesPath):
with open(self.opts.dependenciesPath, "rb") as f:
self.dependencies = pickle.load(f)
def train(self):
self.QuestionNet.train()
# Define the multi-gpu training if needed
if len(self.opts.gpu_ids) > 1:
self.QuestionNet = nn.DataParallel(self.QuestionNet, device_ids=self.opts.gpu_ids)
# Load checkpoint if resume training
if os.path.isfile(self.opts.load_checkpoint_path):
print("[INFO] Resume trainig from ckpt {} ...".format(
self.opts.load_checkpoint_path
))
# Load the network parameters
ckpt = torch.load(self.opts.load_checkpoint_path)
print("[INFO] Checkpoint successfully loaded ...")
self.QuestionNet.load_state_dict(ckpt['state_dict'])
# Load the optimizer paramters
optim = get_optim(self.opts, self.QuestionNet, len(self.datasetTr)) # , ckpt['optim'], lr_base=ckpt['lr_base'])
# optim._step = int(data_size / self.__C.BATCH_SIZE * self.__C.CKPT_EPOCH)
optim.optimizer.load_state_dict(ckpt['optimizer'])
_iter = 0 # ckpt['last_iter']
epoch = 0 # ckpt['last_epoch']
else:
optim = get_optim(self.opts, self.QuestionNet, len(self.datasetTr))
_iter = 0
epoch = 0
trainTime = 0
bestValAcc = float("-inf")
bestCkp = 0
# Training loop
while _iter < self.opts.num_iters:
# Learning Rate Decay
if _iter in self.opts.lr_decay_marks:
adjust_lr(optim, self.opts.lr_decay_factor)
# Define multi-thread dataloader
dataloader = Data.DataLoader(
self.datasetTr,
batch_size=self.opts.batch_size,
shuffle=self.opts.shuffle_data,
num_workers=self.opts.num_workers,
)
# Iteration
time_start = 0
time_end = 0
for batch_iter, (quest, hist, prog, questionRound, _) in enumerate(dataloader):
time_start = time.time()
if _iter >= self.opts.num_iters:
break
quest = quest.to(self.device)
if self.opts.last_n_rounds < 10:
last_n_rounds_batch = []
for i, r in enumerate(questionRound.tolist()):
startIdx = max(r - self.opts.last_n_rounds, 0)
endIdx = max(r, self.opts.last_n_rounds)
if hist.dim() == 3:
assert endIdx - startIdx == self.opts.last_n_rounds
histBatch = hist[i, :, :]
last_n_rounds_batch.append(histBatch[startIdx:endIdx, :])
elif hist.dim() == 2:
startIdx *= 20
endIdx *= 20
histBatch = hist[i, :]
temp = histBatch[startIdx:endIdx].cpu()
if r > self.opts.last_n_rounds:
last_n_rounds_batch.append(torch.cat([torch.tensor([1]), temp, torch.tensor([2])], 0))
else:
last_n_rounds_batch.append(torch.cat([temp, torch.tensor([2, 0])], 0))
hist = torch.stack(last_n_rounds_batch, dim=0)
hist = hist.to(self.device)
prog = prog.to(self.device)
progTarget = prog.clone()
optim.zero_grad()
predSoftmax, _ = self.QuestionNet(quest, hist, prog[:, :-1])
loss = self.loss_fn(
# predSoftmax[:, :-1, :].contiguous().view(-1, predSoftmax.size(2)),
predSoftmax.contiguous().view(-1, predSoftmax.size(2)),
progTarget[:, 1:].contiguous().view(-1))
loss.backward()
if _iter % self.opts.validate_every == 0 and _iter > 0:
valAcc = self.val()
if valAcc > bestValAcc:
bestValAcc = valAcc
bestCkp = _iter
print("\n[INFO] Checkpointing model @ iter {} with val accuracy {}\n".format(_iter, valAcc))
state = {
'state_dict': self.QuestionNet.state_dict(),
'optimizer': optim.optimizer.state_dict(),
'lr_base': optim.lr_base,
'optim': optim.lr_base,
'last_iter': _iter,
'last_epoch': epoch,
}
# checkpointing
torch.save(
state,
os.path.join(self.ckpt_path, 'ckpt_iter' + str(_iter) + '.pkl')
)
# logging
self.writer.add_scalar(
'train/loss',
loss.cpu().data.numpy(),
global_step=_iter)
self.writer.add_scalar(
'train/lr',
optim._rate,
global_step=_iter)
if _iter % self.opts.display_every == 0:
time_end = time.time()
trainTime += time_end-time_start
print("\r[CLEVR-Dialog - %s (%d | %d)][epoch %2d][iter %4d/%4d][runtime %4f] loss: %.4f, lr: %.2e" % (
self.datasetTr.name,
batch_iter,
len(dataloader),
epoch,
_iter,
self.opts.num_iters,
trainTime,
loss.cpu().data.numpy(),
optim._rate,
), end=' ')
optim.step()
_iter += 1
epoch += 1
print("[INFO] Avg. epoch time: {} s".format(trainTime / epoch))
print("[INFO] Best model achieved val acc. {} @ iter {}".format(bestValAcc, bestCkp))
def val(self):
self.QuestionNet.eval()
total_correct = 0
total = 0
if len(self.opts.gpu_ids) > 1:
self.QuestionNet = nn.DataParallel(self.QuestionNet, device_ids=self.opts.gpu_ids)
self.QuestionNet = self.QuestionNet.eval()
dataloader = Data.DataLoader(
self.datasetVal,
batch_size=self.opts.batch_size,
shuffle=True,
num_workers=self.opts.num_workers,
pin_memory=False
)
_iterCur = 0
_totalCur = len(dataloader)
for step, (question, questionPrg, questionImgIdx, questionRounds, history, historiesProg, answer) in enumerate(dataloader):
# print("\rEvaluation: [step %4d/%4d]" % (
print("\rEvaluation: [step %4d/%4d]" % (
step,
int(len(dataloader)),
), end=' ')
question = question.to(self.device)
if history.dim() == 3:
caption = history.detach()
caption = caption[:, 0, :]
caption = caption[:, :16].to(self.device)
elif history.dim() == 2:
caption = history.detach()
caption = caption[:, :16].to(self.device)
if self.opts.last_n_rounds is not None:
last_n_rounds_batch = []
for i, r in enumerate(questionRounds.tolist()):
startIdx = max(r - self.opts.last_n_rounds, 0)
endIdx = max(r, self.opts.last_n_rounds)
if history.dim() == 3:
assert endIdx - startIdx == self.opts.last_n_rounds
histBatch = history[i, :, :]
last_n_rounds_batch.append(histBatch[startIdx:endIdx, :])
elif history.dim() == 2:
startIdx *= 20
endIdx *= 20
histBatch = history[i, :]
temp = histBatch[startIdx:endIdx]
if r > self.opts.last_n_rounds:
last_n_rounds_batch.append(torch.cat([torch.tensor([1]), temp, torch.tensor([2])], 0))
else:
last_n_rounds_batch.append(torch.cat([temp, torch.tensor([2, 0])], 0))
history = torch.stack(last_n_rounds_batch, dim=0)
history = history.to(self.device)
questionPrg = questionPrg.to(self.device)
questProgsToksPred = self.QuestionNet.sample(question, history)
questProgsPred = decodeProg(questProgsToksPred, self.datasetVal.vocab["idx_prog_to_token"])
targetProgs = decodeProg(questionPrg, self.datasetVal.vocab["idx_prog_to_token"], target=True)
correct = [1 if pred == gt else 0 for (pred, gt) in zip(questProgsPred, targetProgs)]
correct = sum(correct)
total_correct += correct
total += len(targetProgs)
self.QuestionNet.train()
return 100.0 * (total_correct / total)
# Evaluation
def eval_with_gt(self):
# Define the multi-gpu training if needed
all_pred_answers = []
all_gt_answers = []
all_question_types = []
all_penalties = []
all_pred_programs = []
all_gt_programs = []
first_failure_round = 0
total_correct = 0
total_acc_pen = 0
total = 0
total_quest_prog_correct = 0
if len(self.opts.gpu_ids) > 1:
self.QuestionNet = nn.DataParallel(self.QuestionNet, device_ids=self.opts.gpu_ids)
self.QuestionNet = self.QuestionNet.eval()
self.CaptionNet = self.CaptionNet.eval()
if self.opts.batch_size != self.opts.dialogLen:
print("[INFO] Changed batch size from {} to {}".format(self.opts.batch_size, self.opts.dialogLen))
self.opts.batch_size = self.opts.dialogLen
dataloader = Data.DataLoader(
self.datasetTest,
batch_size=self.opts.batch_size,
shuffle=False,
num_workers=self.opts.num_workers,
pin_memory=False
)
_iterCur = 0
_totalCur = len(dataloader)
for step, (question, questionPrg, questionImgIdx, questionRounds, history, historiesProg, answer) in enumerate(dataloader):
# print("\rEvaluation: [step %4d/%4d]" % (
# step + 1,
# int(data_size / self.opts.batch_size),
# ), end=' ')
# if step >= 5000:
# break
batchSize = question.size(0)
question = question.to(self.device)
# dependecy = self.dependencies[step*batchSize:(step+1)*batchSize]
if history.dim() == 3:
caption = history.detach()
caption = caption[:, 0, :]
caption = caption[:, :16].to(self.device)
elif history.dim() == 2:
caption = history.detach()
caption = caption[:, :16].to(self.device)
if self.opts.last_n_rounds < 10:
last_n_rounds_batch = []
for i, r in enumerate(questionRounds.tolist()):
startIdx = max(r - self.opts.last_n_rounds, 0)
endIdx = max(r, self.opts.last_n_rounds)
if history.dim() == 3:
assert endIdx - startIdx == self.opts.last_n_rounds
histBatch = history[i, :, :]
last_n_rounds_batch.append(histBatch[startIdx:endIdx, :])
elif history.dim() == 2:
startIdx *= 20
endIdx *= 20
histBatch = history[i, :]
temp = histBatch[startIdx:endIdx]
if r > self.opts.last_n_rounds:
last_n_rounds_batch.append(torch.cat([torch.tensor([1]), temp, torch.tensor([2])], 0))
else:
last_n_rounds_batch.append(torch.cat([temp, torch.tensor([2, 0])], 0))
history = torch.stack(last_n_rounds_batch, dim=0)
history = history.to(self.device)
questionPrg = questionPrg.to(self.device)
historiesProg = historiesProg.tolist()
questionRounds = questionRounds.tolist()
answer = answer.tolist()
answers = list(map(lambda a: self.datasetTest.vocab["idx_text_to_token"][a], answer))
questionImgIdx = questionImgIdx.tolist()
# if "minecraft" in self.opts.scenesPath:
# questionImgIdx = [idx - 1 for idx in questionImgIdx]
questProgsToksPred = self.QuestionNet.sample(question, history)
capProgsToksPred = self.CaptionNet.sample(caption)
questProgsPred = decodeProg(questProgsToksPred, self.datasetTest.vocab["idx_prog_to_token"])
capProgsPred = decodeProg(capProgsToksPred, self.datasetTest.vocab["idx_prog_to_token"])
targetProgs = decodeProg(questionPrg, self.datasetTest.vocab["idx_prog_to_token"], target=True)
questionTypes = [targetProg[0] for targetProg in targetProgs]
# progHistories = getProgHistories(historiesProg[0], dataset.vocab["idx_prog_to_token"])
progHistories = [getProgHistories(progHistToks, self.datasetTest.vocab["idx_prog_to_token"]) for progHistToks in historiesProg]
pred_answers = []
all_pred_programs.append([capProgsPred[0]] + questProgsPred)
all_gt_programs.append([progHistories[0]] + (targetProgs))
for i in range(batchSize):
# if capProgsPred[i][0] == "extreme-center":
# print("bla")
# print("idx = {}".format(questionImgIdx[i]))
ans = self.getPrediction(
questProgsPred[i],
capProgsPred[i],
progHistories[i],
questionImgIdx[i]
)
# if ans == "Error":
# print(capProgsPred[i])
pred_answers.append(ans)
# print(pred_answers)
correct = [1 if pred == ans else 0 for (pred, ans) in zip(pred_answers, answers)]
correct_prog = [1 if pred == ans else 0 for (pred, ans) in zip(questProgsPred, targetProgs)]
idx_false = np.argwhere(np.array(correct) == 0).squeeze(-1)
if idx_false.shape[-1] > 0:
first_failure_round += idx_false[0] + 1
else:
first_failure_round += self.opts.dialogLen + 1
correct = sum(correct)
correct_prog = sum(correct_prog)
total_correct += correct
total_quest_prog_correct += correct_prog
total += len(answers)
all_pred_answers.append(pred_answers)
all_gt_answers.append(answers)
all_question_types.append(questionTypes)
penalty = np.zeros_like(penalty)
all_penalties.append(penalty)
_iterCur += 1
if _iterCur % self.opts.display_every == 0:
print("[Evaluation] step {0} / {1} | acc. = {2:.2f}".format(
_iterCur, _totalCur, 100.0 * (total_correct / total)))
ffr = 1.0 * (first_failure_round/_totalCur)/(self.opts.dialogLen + 1)
textOut = "\n --------------- Average First Failure Round --------------- \n"
textOut += "{} / {}".format(ffr, self.opts.dialogLen)
# print(total_correct, total)
accuracy = total_correct / total
vd_acc = total_acc_pen / total
quest_prog_acc = total_quest_prog_correct / total
textOut += "\n --------------- Overall acc. --------------- \n"
textOut += "{}".format(100.0 * accuracy)
textOut += "\n --------------- Overall VD acc. --------------- \n"
textOut += "{}".format(100.0 * vd_acc)
textOut += "\n --------------- Question Prog. Acc --------------- \n"
textOut += "{}".format(100.0 * quest_prog_acc)
textOut += get_per_round_acc(
all_pred_answers, all_gt_answers, all_penalties)
textOut += get_per_question_type_acc(
all_pred_answers, all_gt_answers, all_question_types, all_penalties)
# textOut += get_per_dependency_type_acc(
# all_pred_answers, all_gt_answers, all_penalties)
textOut += "\n --------------- Done --------------- \n"
print(textOut)
fname = self.opts.questionNetPath.split("/")[-3] + "results_{}_{}.txt".format(self.opts.last_n_rounds, self.opts.dialogLen)
pred_answers_fname = self.opts.questionNetPath.split("/")[-3] + "_pred_answers_{}_{}.pkl".format(self.opts.last_n_rounds, self.opts.dialogLen)
pred_answers_fname = os.path.join("/projects/abdessaied/clevr-dialog/output/pred_answers", pred_answers_fname)
model_name = "NSVD_stack" if "stack" in self.opts.questionNetPath else "NSVD_concat"
experiment_name = "minecraft"
# experiment_name += "_{}".format(self.opts.dialogLen)
prog_output_fname = os.path.join("/projects/abdessaied/clevr-dialog/output/prog_output/{}_{}.pkl".format(model_name, experiment_name))
fpath = os.path.join(self.opts.text_log_dir, fname)
with open(fpath, "w") as f:
f.writelines(textOut)
with open(pred_answers_fname, "wb") as f:
pickle.dump(all_pred_answers, f, protocol=pickle.HIGHEST_PROTOCOL)
with open(prog_output_fname, "wb") as f:
pickle.dump((all_gt_programs, all_pred_programs, all_pred_answers), f, protocol=pickle.HIGHEST_PROTOCOL)
# Evaluation
def eval_with_pred(self):
# Define the multi-gpu training if needed
all_pred_answers = []
all_gt_answers = []
all_question_types = []
all_penalties = []
first_failure_round = 0
total_correct = 0
total_acc_pen = 0
total = 0
samples = {}
if len(self.opts.gpu_ids) > 1:
self.QuestionNet = nn.DataParallel(self.QuestionNet, device_ids=self.opts.gpu_ids)
self.QuestionNet = self.QuestionNet.eval()
self.CaptionNet = self.CaptionNet.eval()
if self.opts.batch_size != self.opts.dialogLen:
print("[INFO] Changed batch size from {} to {}".format(self.opts.batch_size, self.opts.dialogLen))
self.opts.batch_size = self.opts.dialogLen
dataloader = Data.DataLoader(
self.datasetTest,
batch_size=self.opts.batch_size,
shuffle=False,
num_workers=self.opts.num_workers,
pin_memory=False
)
_iterCur = 0
_totalCur = len(dataloader)
step = 0
for step, (question, questionPrg, questionImgIdx, questionRounds, history, historiesProg, answer) in enumerate(dataloader):
question = question.tolist()
questions = decode(question, self.datasetTest.vocab["idx_text_to_token"], target=True)
questions = list(map(lambda q: " ".join(q), questions))
targetProgs = decode(questionPrg, self.datasetTest.vocab["idx_prog_to_token"], target=True)
questionTypes = [targetProg[0] for targetProg in targetProgs]
targetProgs = list(map(lambda q: " ".join(q), targetProgs))
historiesProg = historiesProg.tolist()
progHistories = [getProgHistories(progHistToks, self.datasetTest.vocab["idx_prog_to_token"]) for progHistToks in historiesProg]
answer = answer.tolist()
answers = list(map(lambda a: self.datasetTest.vocab["idx_text_to_token"][a], answer))
questionImgIdx = questionImgIdx.tolist()
if self.opts.encoderType == 2:
histories_eval = [history[0, 0, :].tolist()]
caption = history.detach()
caption = caption[0, 0, :].unsqueeze(0)
caption = caption[:, :16].to(self.device)
elif self.opts.encoderType == 1:
caption = history.detach()
histories_eval = [history[0, :20].tolist()]
caption = caption[0, :16].unsqueeze(0).to(self.device)
cap = decode(caption, self.datasetTest.vocab["idx_text_to_token"], target=False)
capProgToksPred = self.CaptionNet.sample(caption)
capProgPred = decode(capProgToksPred, self.datasetTest.vocab["idx_prog_to_token"])[0]
pred_answers = []
pred_quest_prog = []
for i, (q, prog_hist, img_idx) in enumerate(zip(question, progHistories, questionImgIdx)):
_round = i + 1
if _round <= self.opts.last_n_rounds:
start = 0
else:
start = _round - self.opts.last_n_rounds
end = len(histories_eval)
quest = torch.tensor(q).unsqueeze(0).to(self.device)
if self.opts.encoderType == 3:
hist = torch.stack([torch.tensor(h) for h in histories_eval[start:end]], dim=0).unsqueeze(0).to(self.device)
elif self.opts.encoderType == 1:
histories_eval_copy = deepcopy(histories_eval)
histories_eval_copy[-1].append(self.datasetTest.vocab["text_token_to_idx"]["<END>"])
hist = torch.cat([torch.tensor(h) for h in histories_eval_copy[start:end]], dim=-1).unsqueeze(0).to(self.device)
questProgsToksPred = self.QuestionNet.sample(quest, hist)
questProgsPred = decode(questProgsToksPred, self.datasetTest.vocab["idx_prog_to_token"])[0]
pred_quest_prog.append(" ".join(questProgsPred))
ans = self.getPrediction(
questProgsPred,
capProgPred,
prog_hist,
img_idx
)
ans_idx = self.datasetTest.vocab["text_token_to_idx"].get(
ans, self.datasetTest.vocab["text_token_to_idx"]["<UNK>"])
q[q.index(self.datasetTest.vocab["text_token_to_idx"]["<END>"])] = self.datasetTest.vocab["text_token_to_idx"]["<NULL>"]
q[-1] = self.datasetTest.vocab["text_token_to_idx"]["<END>"]
q.insert(-1, ans_idx)
if self.opts.encoderType == 3:
histories_eval.append(copy.deepcopy(q))
elif self.opts.encoderType == 0:
del q[0]
del q[-1]
histories_eval.append(copy.deepcopy(q))
pred_answers.append(ans)
correct = [1 if pred == ans else 0 for (pred, ans) in zip(pred_answers, answers)]
idx_false = np.argwhere(np.array(correct) == 0).squeeze(-1)
if idx_false.shape[-1] > 0:
first_failure_round += idx_false[0] + 1
else:
first_failure_round += self.opts.dialogLen + 1
correct = sum(correct)
total_correct += correct
total += len(answers)
all_pred_answers.append(pred_answers)
all_gt_answers.append(answers)
all_question_types.append(questionTypes)
_iterCur += 1
if _iterCur % self.opts.display_every == 0:
print("[Evaluation] step {0} / {1} | acc. = {2:.2f}".format(
_iterCur, _totalCur, 100.0 * (total_correct / total)
))
samples["{}_{}".format(questionImgIdx[0], (step % 5) + 1)] = {
"caption": " ".join(cap[0]),
"cap_prog_gt": " ".join(progHistories[0][0]),
"cap_prog_pred": " ".join(capProgPred),
"questions": questions,
"quest_progs_gt": targetProgs,
"quest_progs_pred": pred_quest_prog,
"answers": answers,
"preds": pred_answers,
"acc": correct,
}
ffr = 1.0 * self.opts.dialogLen * (first_failure_round/total)
textOut = "\n --------------- Average First Failure Round --------------- \n"
textOut += "{} / {}".format(ffr, self.opts.dialogLen)
# print(total_correct, total)
accuracy = total_correct / total
vd_acc = total_acc_pen / total
textOut += "\n --------------- Overall acc. --------------- \n"
textOut += "{}".format(100.0 * accuracy)
textOut += "\n --------------- Overall VD acc. --------------- \n"
textOut += "{}".format(100.0 * vd_acc)
textOut += get_per_round_acc(
all_pred_answers, all_gt_answers, all_penalties)
textOut += get_per_question_type_acc(
all_pred_answers, all_gt_answers, all_question_types, all_penalties)
textOut += "\n --------------- Done --------------- \n"
print(textOut)
if step >= len(dataloader):
fname = self.opts.questionNetPath.split("/")[-3] + "_results_{}_{}_{}.txt".format(self.opts.last_n_rounds, self.opts.dialogLen, self.acc_type)
pred_answers_fname = self.opts.questionNetPath.split("/")[-3] + "_pred_answers_{}_{}.pkl".format(self.opts.last_n_rounds, self.opts.dialogLen)
pred_answers_fname = os.path.join("/projects/abdessaied/clevr-dialog/output/pred_answers", pred_answers_fname)
fpath = os.path.join(self.opts.text_log_dir, fname)
with open(fpath, "w") as f:
f.writelines(textOut)
with open(pred_answers_fname, "wb") as f:
pickle.dump(all_pred_answers, f, protocol=pickle.HIGHEST_PROTOCOL)
def getPrediction(self, questProgPred, capProgPred, historyProg, imgIndex):
self.symbolicExecutor.reset(imgIndex)
# if round one, execute the predicted caption program first then answer the question
if len(historyProg) == 1:
captionFuncLabel = capProgPred[0]
captionFuncArgs = capProgPred[1:]
questionFuncLabel = questProgPred[0]
questionFuncArgs = questProgPred[1:]
try:
_ = self.symbolicExecutor.execute(captionFuncLabel, captionFuncArgs)
except:
return "Error"
try:
predAnswer = self.symbolicExecutor.execute(questionFuncLabel, questionFuncArgs)
except:
return "Error"
# If it is not the first round, we have to execute the program history and
# then answer the question.
else:
questionFuncLabel = questProgPred[0]
questionFuncArgs = questProgPred[1:]
for prg in historyProg:
# prg = prg.split(" ")
FuncLabel = prg[0]
FuncArgs = prg[1:]
try:
_ = self.symbolicExecutor.execute(FuncLabel, FuncArgs)
except:
return "Error"
try:
predAnswer = self.symbolicExecutor.execute(questionFuncLabel, questionFuncArgs)
except:
return "Error"
return str(predAnswer)
def run(self, run_mode, epoch=None):
self.set_seed(self.opts.seed)
if run_mode == 'train':
self.train()
elif run_mode == 'test_with_gt':
print('Testing with gt answers in history')
print('Loading ckpt {}'.format(self.opts.questionNetPath))
state_dict = torch.load(self.opts.questionNetPath)['state_dict']
self.QuestionNet.load_state_dict(state_dict)
self.eval_with_gt()
elif run_mode == 'test_with_pred':
print('Testing with predicted answers in history')
print('Loading ckpt {}'.format(self.opts.questionNetPath))
state_dict = torch.load(self.opts.questionNetPath)['state_dict']
self.QuestionNet.load_state_dict(state_dict)
self.eval_with_pred()
else:
exit(-1)
def set_seed(self, seed):
"""Sets the seed for reproducibility.
Args:
seed (int): The seed used
"""
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(seed)
print('[INFO] Seed set to {}...'.format(seed))
def constructQuestionNet(opts, lenVocabText, lenVocabProg, maxLenProg):
decoder = Decoder(opts, lenVocabProg, maxLenProg)
if opts.encoderType == 1:
encoder = QuestEncoder_1(opts, lenVocabText)
elif opts.encoderType == 2:
encoder = QuestEncoder_2(opts, lenVocabText)
net = SeqToSeqQ(encoder, decoder)
return net
def constructCaptionNet(opts, lenVocabText, lenVocabProg, maxLenProg):
decoder = Decoder(opts, lenVocabProg, maxLenProg)
encoder = CaptionEncoder(opts, lenVocabText)
net = SeqToSeqC(encoder, decoder)
return net
def getProgHistories(progHistToks, prgIdxToToken):
progHist = []
temp = []
for tok in progHistToks:
if tok not in [0, 1, 2]:
temp.append(prgIdxToToken[tok])
# del progHistToks[i]
if tok == 2:
# del progHistToks[i]
# progHist.append(" ".join(temp))
progHist.append(temp)
temp = []
return progHist
def getHistoriesFromStack(histToks, textIdxToToken):
histories = "\n"
temp = []
for i, roundToks in enumerate(histToks):
for tok in roundToks:
if tok not in [0, 1, 2]:
temp.append(textIdxToToken[tok])
# del progHistToks[i]
if tok == 2:
# del progHistToks[i]
if i == 0:
histories += " ".join(temp) + ".\n"
else:
histories += " ".join(temp[:-1]) + "? | {}.\n".format(temp[-1])
# histories.append(temp)
temp = []
break
return histories
def getHistoriesFromConcat(histToks, textIdxToToken):
histories = []
temp = []
for tok in histToks:
if tok not in [0, 1, 2]:
temp.append(textIdxToToken[tok])
# del progHistToks[i]
if tok == 2:
# del progHistToks[i]
histories.append(" ".join(temp[:-1]) + "? | {}".format(temp[-1]))
# histories.append(temp)
temp = []
return histories
def decodeProg(tokens, prgIdxToToken, target=False):
tokensBatch = tokens.tolist()
progsBatch = []
for tokens in tokensBatch:
prog = []
for tok in tokens:
if tok == 2: # <END> has index 2
break
prog.append(prgIdxToToken.get(tok))
if target:
prog = prog[1:]
# progsBatch.append(" ".join(prog))
progsBatch.append(prog)
return progsBatch
def printPred(predSoftmax, gts, prgIdxToToken):
assert predSoftmax.size(0) == gts.size(0)
tokens = predSoftmax.topk(1)[1].squeeze(-1)
tokens = tokens.tolist()
gts = gts.tolist()
message = "\n ------------------------ \n"
for token, gt in zip(tokens, gts):
message += "Prediction: "
for tok in token:
message += prgIdxToToken.get(tok) + " "
message += "\n Target : "
for tok in gt:
message += prgIdxToToken.get(tok) + " "
message += "\n ------------------------ \n"
return message
def get_per_round_acc(preds, gts, penalties):
res = {}
for img_preds, img_gt, img_pen in zip(preds, gts, penalties):
img_preds = list(img_preds)
img_gt = list(img_gt)
img_pen = list(img_pen)
for i, (pred, gt, pen) in enumerate(zip(img_preds, img_gt, img_pen)):
_round = str(i + 1)
if _round not in res:
res[_round] = {
"correct": 0,
"all": 0
}
res[_round]["all"] += 1
if pred == gt:
res[_round]["correct"] += 0.5**pen
textOut = "\n --------------- Per round Acc --------------- \n"
for k in res:
textOut += "{}: {} %\n".format(k, 100.0 * (res[k]["correct"]/res[k]["all"]))
return textOut
def get_per_question_type_acc(preds, gts, qtypes, penalties):
res1 = {}
res2 = {}
for img_preds, img_gt, img_qtypes, img_pen in zip(preds, gts, qtypes, penalties):
# img_preds = list(img_preds)
# img_gt = list(img_gt)
img_pen = list(img_pen)
for pred, gt, temp, pen in zip(img_preds, img_gt, img_qtypes, img_pen):
if temp not in res1:
res1[temp] = {
"correct": 0,
"all": 0
}
temp_cat = temp.split("-")[0]
if temp_cat not in res2:
res2[temp_cat] = {
"correct": 0,
"all": 0
}
res1[temp]["all"] += 1
res2[temp_cat]["all"] += 1
if pred == gt:
res1[temp]["correct"] += 0.5**pen
res2[temp_cat]["correct"] += 0.5**pen
textOut = "\n --------------- Per question Type Acc --------------- \n"
for k in res1:
textOut += "{}: {} %\n".format(k, 100.0 * (res1[k]["correct"]/res1[k]["all"]))
textOut += "\n --------------- Per question Category Acc --------------- \n"
for k in res2:
textOut += "{}: {} %\n".format(k, 100.0 * (res2[k]["correct"]/res2[k]["all"]))
return textOut
def decode(tokens, prgIdxToToken, target=False):
if type(tokens) != list:
tokens = tokens.tolist()
progsBatch = []
for token in tokens:
prog = []
for tok in token:
if tok == 2: # <END> has index 2
break
prog.append(prgIdxToToken.get(tok))
if target:
prog = prog[1:]
# progsBatch.append(" ".join(prog))
progsBatch.append(prog)
return progsBatch
if __name__ == "__main__":
optsC = OptionsC().parse()
optsQ = OptionsQ().parse()
exe = Execution(optsQ, optsC)
exe.run("test")
print("[INFO] Done ...")