162 lines
No EOL
7.6 KiB
Python
162 lines
No EOL
7.6 KiB
Python
# visualise data in the ADT dataset
|
|
import numpy as np
|
|
import matplotlib.pyplot as plt
|
|
from mpl_toolkits.mplot3d import Axes3D
|
|
|
|
|
|
# play human pose using a skeleton
|
|
class Player_Skeleton:
|
|
def __init__(self, fps=30.0, object_num=10):
|
|
|
|
self._fps = fps
|
|
self.object_num = object_num
|
|
# names of all the joints: head + left_hand + right_hand + left_hand_joint + right_hand_joint + gaze_direction + head_direction
|
|
self._joint_names = ['Head', 'LHand', 'RHand', 'LThumb1', 'LThumb2', 'LThumb3', 'LIndex1', 'LIndex2', 'LIndex3', 'LMiddle1', 'LMiddle2', 'LMiddle3', 'LRing1', 'LRing2', 'LRing3', 'LPinky1', 'LPinky2', 'LPinky3', 'RThumb1', 'RThumb2', 'RThumb3', 'RIndex1', 'RIndex2', 'RIndex3', 'RMiddle1', 'RMiddle2', 'RMiddle3', 'RRing1', 'RRing2', 'RRing3', 'RPinky1', 'RPinky2', 'RPinky3', 'Gaze_direction', 'Head_direction']
|
|
|
|
self._joint_ids = {name: idx for idx, name in enumerate(self._joint_names)}
|
|
|
|
# parent of every joint
|
|
self._joint_parent_names = {
|
|
# root
|
|
'Head': 'Head',
|
|
'LHand': 'LHand',
|
|
'RHand': 'RHand',
|
|
'LThumb1': 'LHand',
|
|
'LThumb2': 'LThumb1',
|
|
'LThumb3': 'LThumb2',
|
|
'LIndex1': 'LHand',
|
|
'LIndex2': 'LIndex1',
|
|
'LIndex3': 'LIndex2',
|
|
'LMiddle1': 'LHand',
|
|
'LMiddle2': 'LMiddle1',
|
|
'LMiddle3': 'LMiddle2',
|
|
'LRing1': 'LHand',
|
|
'LRing2': 'LRing1',
|
|
'LRing3': 'LRing2',
|
|
'LPinky1': 'LHand',
|
|
'LPinky2': 'LPinky1',
|
|
'LPinky3': 'LPinky2',
|
|
'RThumb1': 'RHand',
|
|
'RThumb2': 'RThumb1',
|
|
'RThumb3': 'RThumb2',
|
|
'RIndex1': 'RHand',
|
|
'RIndex2': 'RIndex1',
|
|
'RIndex3': 'RIndex2',
|
|
'RMiddle1': 'RHand',
|
|
'RMiddle2': 'RMiddle1',
|
|
'RMiddle3': 'RMiddle2',
|
|
'RRing1': 'RHand',
|
|
'RRing2': 'RRing1',
|
|
'RRing3': 'RRing2',
|
|
'RPinky1': 'RHand',
|
|
'RPinky2': 'RPinky1',
|
|
'RPinky3': 'RPinky2',
|
|
'Gaze_direction': 'Head',
|
|
'Head_direction': 'Head',}
|
|
|
|
# id of joint parent
|
|
self._joint_parent_ids = [self._joint_ids[self._joint_parent_names[child_name]] for child_name in self._joint_names]
|
|
self._joint_links = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
|
|
# colors: 0 for head, 1 for left, 2 for right
|
|
self._link_colors = [1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 4]
|
|
|
|
self._fig = plt.figure()
|
|
self._ax = plt.gca(projection='3d')
|
|
|
|
self._plots = []
|
|
for i in range(len(self._joint_links)):
|
|
if self._link_colors[i] == 0:
|
|
color = "#3498db"
|
|
if self._link_colors[i] == 1:
|
|
color = "#3498db"
|
|
if self._link_colors[i] == 2:
|
|
color = "#3498db"
|
|
if self._link_colors[i] == 3:
|
|
color = "#6aa84f"
|
|
if self._link_colors[i] == 4:
|
|
color = "#a64d79"
|
|
self._plots.append(self._ax.plot([0, 0], [0, 0], [0, 0], lw=2.0, c=color))
|
|
|
|
for i in range(self.object_num):
|
|
self._plots.append(self._ax.plot([0, 0], [0, 0], [0, 0], lw=1.0, c='#ff0000'))
|
|
|
|
self._ax.set_xlabel("x")
|
|
self._ax.set_ylabel("y")
|
|
self._ax.set_zlabel("z")
|
|
|
|
# play the sequence of human pose in xyz representations
|
|
def play_xyz(self, pose_xyz, gaze, head, objects):
|
|
gaze_direction = pose_xyz[:, :3] + gaze[:, :3]*0.5
|
|
head_direction = pose_xyz[:, :3] + head[:, :3]*0.5
|
|
pose_xyz = np.concatenate((pose_xyz, gaze_direction), axis = 1)
|
|
pose_xyz = np.concatenate((pose_xyz, head_direction), axis = 1)
|
|
|
|
for i in range(pose_xyz.shape[0]):
|
|
joint_number = len(self._joint_names)
|
|
pose_xyz_tmp = pose_xyz[i].reshape(joint_number, 3)
|
|
objects_xyz = objects[i, :, :, :]
|
|
for j in range(len(self._joint_links)):
|
|
idx = self._joint_links[j]
|
|
start_point = pose_xyz_tmp[idx]
|
|
end_point = pose_xyz_tmp[self._joint_parent_ids[idx]]
|
|
x = np.array([start_point[0], end_point[0]])
|
|
y = np.array([start_point[2], end_point[2]])
|
|
z = np.array([start_point[1], end_point[1]])
|
|
self._plots[j][0].set_xdata(x)
|
|
self._plots[j][0].set_ydata(y)
|
|
self._plots[j][0].set_3d_properties(z)
|
|
|
|
for j in range(len(self._joint_links), len(self._joint_links) + objects_xyz.shape[0]):
|
|
object_xyz = objects_xyz[j - len(self._joint_links), :, :]
|
|
self._plots[j][0].set_xdata(object_xyz[:, 0])
|
|
self._plots[j][0].set_ydata(object_xyz[:, 2])
|
|
self._plots[j][0].set_3d_properties(object_xyz[:, 1])
|
|
|
|
r = 1.0
|
|
x_root, y_root, z_root = pose_xyz_tmp[0, 0], pose_xyz_tmp[0, 2], pose_xyz_tmp[0, 1]
|
|
self._ax.set_xlim3d([-r + x_root, r + x_root])
|
|
self._ax.set_ylim3d([-r + y_root, r + y_root])
|
|
self._ax.set_zlim3d([-r + z_root, r + z_root])
|
|
#self._ax.view_init(elev=30, azim=-110)
|
|
|
|
self._ax.grid(False)
|
|
#self._ax.axis('off')
|
|
|
|
self._ax.set_aspect('auto')
|
|
plt.show(block=False)
|
|
self._fig.canvas.draw()
|
|
past_time = f"{i / self._fps:.1f}"
|
|
plt.title(f"Time: {past_time} s", fontsize=15)
|
|
plt.pause(0.000000001)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
data_path = '/scratch/hu/pose_forecast/adt_hoigaze/test/Apartment_release_meal_skeleton_seq132_'
|
|
gaze_path = data_path + 'gaze.npy'
|
|
head_path = data_path + 'head.npy'
|
|
hand_path = data_path + 'hand.npy'
|
|
hand_joint_path = data_path + 'handjoints.npy'
|
|
object_left_hand_path = data_path + 'object_left.npy'
|
|
object_right_hand_path = data_path + 'object_right.npy'
|
|
|
|
gaze = np.load(gaze_path) # gaze_direction (3) + gaze_2d (2) + frame_id (1)
|
|
print("Gaze shape: {}".format(gaze.shape))
|
|
gaze_direction = gaze[:, :3]
|
|
head = np.load(head_path) # head_direction (3) + head_translation (3)
|
|
print("Head shape: {}".format(head.shape))
|
|
head_direction = head[:, :3]
|
|
head_translation = head[:, 3:]
|
|
hand_translation = np.load(hand_path) # left_hand_translation (3) + right_hand_translation (3)
|
|
print("Hand shape: {}".format(hand_translation.shape))
|
|
hand_joint = np.load(hand_joint_path) # left_hand (15*3) + right_hand (15*3) + hand_dominance + closest_hand
|
|
print("Hand joint shape: {}".format(hand_joint.shape))
|
|
hand_joint = hand_joint[:, :90]
|
|
pose = np.concatenate((head_translation, hand_translation), axis=1)
|
|
pose = np.concatenate((pose, hand_joint), axis=1)
|
|
object_left = np.load(object_left_hand_path)[:, :, :, :]
|
|
object_right = np.load(object_right_hand_path)[:, :, :, :]
|
|
object_all = np.concatenate((object_left, object_right), axis=1)
|
|
print("Object shape: {}".format(object_all.shape))
|
|
|
|
player = Player_Skeleton(object_num = object_all.shape[1])
|
|
player.play_xyz(pose, gaze_direction, head_direction, object_all) |