import json import numpy as np import matplotlib import matplotlib.pyplot as plt import matplotlib.patches as patches from pattern.text.en import singularize import nengo.spa as spa import scipy.integrate as integrate RGB_COLORS = [] hex_colors = ['#8a3ffc', '#ff7eb6', '#6fdc8c', '#d2a106', '#ba4e00', '#33b1ff', '#570408', '#fa4d56', '#4589ff', '#08bdba', '#d4bbff', '#007d79', '#d12771', '#bae6ff'] for h in hex_colors: RGB_COLORS.append(matplotlib.colors.to_rgb(h)) for i, (name, h) in enumerate(matplotlib.colors.cnames.items()): if i > 10: RGB_COLORS.append(matplotlib.colors.to_rgb(h)) f = open('gqa_all_relations_map.json') RELATION_DICT = json.load(f) f.close() f = open('gqa_all_vocab_classes.json') CLASS_DICT = json.load(f) f.close() f = open('gqa_all_attributes.json') ATTRIBUTE_DICT = json.load(f) f.close() def bbox_to_mask(x, y, w, h, img_size=(500, 500), name=None, visualize=False): img = np.zeros(img_size) mask_w = np.ones(w) for j in range(y, y+h): img[j][x:x+w] = mask_w if visualize: fig = plt.figure(figsize=(img_size[0] // 80, img_size[1] // 80)) plt.imshow(img, cmap='gray') if name: plt.title(name) plt.axis('off') plt.show() return img def make_good_unitary(D, eps=1e-3, rng=np.random): """from https://github.com/ctn-waterloo/cogsci2019-ssp/tree/master""" a = rng.rand((D - 1) // 2) sign = rng.choice((-1, +1), len(a)) phi = sign * np.pi * (eps + a * (1 - 2 * eps)) assert np.all(np.abs(phi) >= np.pi * eps) assert np.all(np.abs(phi) <= np.pi * (1 - eps)) fv = np.zeros(D, dtype='complex64') fv[0] = 1 fv[1:(D + 1) // 2] = np.cos(phi) + 1j * np.sin(phi) fv[-1:D // 2:-1] = np.conj(fv[1:(D + 1) // 2]) if D % 2 == 0: fv[D // 2] = 1 assert np.allclose(np.abs(fv), 1) v = np.fft.ifft(fv) # assert np.allclose(v.imag, 0, atol=1e-5) v = v.real assert np.allclose(np.fft.fft(v), fv) assert np.allclose(np.linalg.norm(v), 1) return spa.SemanticPointer(v) def get_heatmap_vectors(xs, ys, x_axis_sp, y_axis_sp): """from https://github.com/ctn-waterloo/cogsci2019-ssp/tree/master: Precompute spatial semantic pointers for every location in the linspace Used to quickly compute heat maps by a simple vectorized dot product (matrix multiplication) """ if x_axis_sp.__class__.__name__ == 'SemanticPointer': dim = len(x_axis_sp.v) else: dim = len(x_axis_sp) x_axis_sp = spa.SemanticPointer(data=x_axis_sp) y_axis_sp = spa.SemanticPointer(data=y_axis_sp) vectors = np.zeros((len(xs), len(ys), dim)) for i, x in enumerate(xs): for j, y in enumerate(ys): p = encode_point( x=x, y=y, x_axis=x_axis_sp, y_axis=y_axis_sp, ) vectors[i, j, :] = p.v return vectors def power(s, e): """from https://github.com/ctn-waterloo/cogsci2019-ssp/tree/master""" x = np.fft.ifft(np.fft.fft(s.v) ** e).real return spa.SemanticPointer(data=x) def encode_point(x, y, x_axis, y_axis): """from https://github.com/ctn-waterloo/cogsci2019-ssp/tree/master""" return power(x_axis, x) * power(y_axis, y) def encode_region(x, y, x_axis, y_axis): """from https://github.com/ctn-waterloo/cogsci2019-ssp/tree/master""" print(integrate.quad(power(x_axis, x) * power(y_axis, y), x, x+28)) return integrate.quad(power(x_axis, x) * power(y_axis, y), x, x+28) def plot_heatmap(img, img_area, encoded_pos, xs, ys, heatmap_vectors, name='', vmin=-1, vmax=1, invert=False): """from https://github.com/ctn-waterloo/cogsci2019-ssp/tree/master""" assert encoded_pos.__class__.__name__ == 'SemanticPointer' # sp has shape (dim,) and heatmap_vectors have shape (xs, ys, dim) so the result will be (xs, ys) vec_sim = np.tensordot(encoded_pos.v, heatmap_vectors, axes=([0], [2])) num_plots = 3 if img_area is not None else 2 fig, axs = plt.subplots(1, num_plots, figsize=(4 * num_plots + 3, 3)) fig.suptitle(name) axs[0].imshow(img) axs[0].axis('off') if img_area is not None: axs[1].imshow(img_area, cmap='gray') axs[1].set_xticks(np.arange(0, len(xs), 20), np.arange(0, img.shape[1], img.shape[1] / len(xs)).astype(int)[::20]) axs[1].set_yticks(np.arange(0, len(ys), 10), np.arange(0, img.shape[0], img.shape[0] / len(ys)).astype(int)[::10]) axs[1].axis('off') im = axs[2].imshow(np.transpose(vec_sim), origin='upper', interpolation='none', extent=(xs[-1], xs[0], ys[-1], ys[0]), vmin=vmin, vmax=vmax, cmap='plasma') axs[2].axis('off') else: im = axs[1].imshow(np.transpose(vec_sim), origin='upper', interpolation='none', extent=(xs[-1], xs[0], ys[-1], ys[0]), vmin=vmin, vmax=vmax, cmap='plasma') axs[1].axis('off') fig.colorbar(im, ax=axs.ravel().tolist()) plt.show() def generate_region_vector(desired, xs, ys, x_axis_sp, y_axis_sp): """from https://github.com/ctn-waterloo/cogsci2019-ssp/tree/master""" vector = np.zeros_like((x_axis_sp.v)) for i, x in enumerate(xs): for j, y in enumerate(ys): if desired[j, i] == 1: vector += encode_point(x, y, x_axis_sp, y_axis_sp).v sp = spa.SemanticPointer(data=vector) sp.normalize() return sp def bb_intersection_over_union(boxA, boxB): """from https://pyimagesearch.com/2016/11/07/intersection-over-union-iou-for-object-detection/""" # determine the (x, y)-coordinates of the intersection rectangle xA = max(boxA[0], boxB[0]) yA = max(boxA[1], boxB[1]) xB = min(boxA[2], boxB[2]) yB = min(boxA[3], boxB[3]) # compute the area of intersection rectangle interArea = abs(max((xB - xA, 0)) * max((yB - yA), 0)) if interArea == 0: return 0 # compute the area of both the prediction and ground-truth # rectangles boxAArea = abs((boxA[2] - boxA[0]) * (boxA[3] - boxA[1])) boxBArea = abs((boxB[2] - boxB[0]) * (boxB[3] - boxB[1])) # compute the intersection over union by taking the intersection # area and dividing it by the sum of prediction + ground-truth # areas - the interesection area iou = interArea / float(boxAArea + boxBArea - interArea) # return the intersection over union value return iou def encode_point_multidim(values, axes): """ power(x_axis, x) * power(y_axis, y) for variable dimensions """ assert len(values) == len(axes), f'number of values {len(values)} does not match number of axes {len(axes)}' res = 1 for v, a in zip(values, axes): res *= power(a, v) return res def get_heatmap_vectors_multidim(xs, ys, ws, hs, x_axis, y_axis, w_axis, h_axis): """ adaptation of get_heatmap_vectors for 4 dimensions """ assert x_axis.__class__.__name__ == 'SemanticPointer', f'Axes need to be of type SemanticPointer but are {x_axis.__class__.__name__}' dim = len(x_axis.v) vectors = np.zeros((len(xs), len(ys), len(ws), len(hs), dim)) for i, x in enumerate(xs): for j, y in enumerate(ys): for n, w in enumerate(ws): for k, h in enumerate(hs): p = encode_point_multidim(values=[x, y, w, h], axes=[x_axis, y_axis, w_axis, h_axis]) vectors[i, j, n, k, :] = p.v return vectors def ssp_to_loc_multidim(sp, heatmap_vectors, linspace): """ adaptation of loc_match from https://github.com/ctn-waterloo/cogsci2019-ssp/tree/master Convert an SSP to the approximate 4-dim location that it represents. Uses the heatmap vectors as a lookup table """ xs, ys, ws, hs = linspace assert sp.__class__.__name__ == 'SemanticPointer', \ f'Queried object needs to be of type SemanticPointer but is {sp.__class__.__name__}' # axes: a list of axes to be summed over, first sequence applying to first tensor, second to second tensor vs = np.tensordot(sp.v, heatmap_vectors, axes=([0], [4])) res = np.unravel_index(np.argmax(vs, axis=None), vs.shape) x = xs[res[0]] y = ys[res[1]] w = ws[res[2]] h = hs[res[3]] return np.array([x, y, w, h]) def encode_image_ssp(img, sg_data, axes, new_size, dim, visualize=True): """encode all objects in an image to an SSP memory""" img_size = img.shape[:2] if img_size[1] / 2 < img_size[0]: scale = img_size[0] / new_size[0] else: scale = img_size[1] / new_size[1] # scale width and height to fixed size of 10 w_scale = img_size[1] / 10 h_scale = img_size[0] / 10 if visualize: print(f'Original image {img_size[1]}x{img_size[0]} --> {np.array(img_size) / scale}') fig, ax = plt.subplots(1,1) ax.imshow(img, interpolation='none', origin='upper', extent=[0, img_size[1] / scale, img_size[0] / scale, 0]) plt.axis('off') encoded_items = {} encoded_ssps = {} memory = spa.SemanticPointer(data=np.zeros(dim)) name_lst = [] for i, obj in enumerate(sg_data.items()): id_num, obj_dict = obj name = obj_dict.get('name') name = singularize(name) name_lst.append(name) name += '_' + str(name_lst.count(name)) # extract ground truth data and scale to fit to SSPs x, y, width, height = obj_dict.get('x'), obj_dict.get('y'), obj_dict.get('w'), obj_dict.get('h') x, y, width, height = x / scale, y / scale, width / w_scale, height / h_scale width = width if width >= 1 else 1 height = height if height >= 1 else 1 # Round values to next int (otherwise decoding gets buggy) item = np.round([x, y, width, height], decimals=0).astype(int) encoded_items[name] = item #print(name, item) pos = encode_point_multidim(list(item), axes) ssp = spa.SemanticPointer(dim) encoded_ssps[name] = ssp memory += ssp * pos if visualize: x, y, width, height = item width, height = (width * w_scale) / scale, (height * h_scale) / scale rect = patches.Rectangle((x, y), width, height, linewidth = 2, label = name, edgecolor = 'c', facecolor = 'none') ax.add_patch(rect) if visualize: plt.show() return encoded_items, encoded_ssps, memory