Added GUI

This commit is contained in:
apenzko 2021-10-17 12:49:49 +02:00
parent ae9774cf0d
commit 74df5cb3f0
23 changed files with 3174 additions and 2 deletions

View File

@ -1,3 +1,31 @@
# conan
# ConAn
This is the official repository for [ConAn: A Usable Tool for Multimodal <u>Con</u>versation <u>An</u>alysis](https://www.perceptualui.org/publications/penzkofer21_icmi.pdf) <br>
ConAn our graphical tool for multimodal conversation analysis takes 360 degree videos recorded during multiperson group interactions as input. ConAn integrates state-of-the-art models for gaze estimation, active speaker detection,
facial action unit detection, and body movement detection and can output quantitative reports both at individual and group
level, as well as different visualizations that provide qualitative insights into group interaction.
ConAn: A Usable Tool for Multimodal <u>Con</u>versation <u>An</u>alysis
## Installation
For the graphical user interface (GUI) you need python>3.6 to install the [requirements](requirements.txt) via pip:
```
pip install requirements.txt
```
## Get Started
To test the GUI you can download our example use case videos from googledrive: <br>
As well as the respective processed ``.dat`` files which include all the analyses.
Run [main.py](main.py) and import the video file you would like to analyze.
## Processing
## Citation
Please cite this paper if you use ConAn or parts of this publication in your research:
```
@inproceedings{penzkofer21_icmi,
author = {Penzkofer, Anna and Müller, Philipp and Bühler, Felix and Mayer, Sven and Bulling, Andreas},
title = {ConAn: A Usable Tool for Multimodal Conversation Analysis},
booktitle = {Proc. ACM International Conference on Multimodal Interaction (ICMI)},
year = {2021},
doi = {10.1145/3462244.3479886},
video = {https://www.youtube.com/watch?v=H2KfZNgx6CQ}
}
```

151
container.py Normal file
View File

@ -0,0 +1,151 @@
import numpy as np
import pandas as pd
import pickle as pkl
import os.path
import cv2
from PyQt5 import QtCore, QtGui
class DataContainer():
def __init__(self, movie_fileName, data_fileName):
self.movie_fileName = movie_fileName
self.data_fileName = data_fileName
self.frameCount = 0
self.fps = 0
self.frameSize = [0, 0]
self.dataGaze = None
self.dataMovement = None
self.number_ids = None
self.dataRTGene = None
self.image = None
def readData(self, verbose=False, rtgene=False):
if (verbose):
print("## Start Reading Data")
# Read Video Data
f = self.movie_fileName
print(f)
if os.path.isfile(f):
cap = cv2.VideoCapture(f)
ret, frame = cap.read()
self.image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
h, w, ch = self.image.shape
bytesPerLine = ch * w
convertToQtFormat = QtGui.QImage(self.image.data, w, h, bytesPerLine, QtGui.QImage.Format_RGB888)
self.fps = cap.get(cv2.CAP_PROP_FPS)
self.frameCount = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
width = cap.get(cv2.CAP_PROP_FRAME_WIDTH) # float
height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT) # float
self.frameSize = [width, height]
print(self.fps, self.frameCount)
if (verbose):
print("Video frameCount %i" % self.frameCount)
duration = self.frameCount / self.fps
minutes = int(duration / 60)
seconds = duration % 60
print('Video duration (M:S) = ' + str(minutes) + ':' + str(seconds))
else:
print("WARNING: no video avaibale.")
# read data file
with open(self.data_fileName, 'rb') as f:
data = pkl.load(f)
# Read RT-Gene Gaze
if rtgene:
f = "%s_Gaze.pkl" % self.dataPath[0]
df_gt = pd.read_pickle('exampledata/G2_VID1_GroundTruth.pkl')
df_gt = df_gt[['Frame', 'ID0_target_spher', 'ID1_target_spher']]
if os.path.isfile(f):
df = pd.read_pickle(f)
self.number_ids = len(df.PId.unique())
self.dataRTGene = df.pivot(index='Frame', columns="PId", values=["GazeTheta", "GazePhi", "HeadCenter",
"HeadPoseYaw", "HeadPoseTheta",
"Phi", "Theta"])
lst = []
for label in ["GazeTheta", "GazePhi", "Head", "HeadPoseYaw", "HeadPoseTheta", "Phi", "Theta"]:
for head_id in range(self.number_ids):
lst.append("ID%i_%s" % (head_id, label))
self.dataRTGene.columns = lst
self.dataRTGene = self.dataRTGene.reset_index()
self.dataRTGene = pd.merge(self.dataRTGene, df_gt, on=['Frame'])
self.dataRTGene = self.dataRTGene.rename(
columns={'ID0_target_spher': 'ID1_target', 'ID1_target_spher': 'ID0_target'},
errors='raise')
print('Detected %i IDs in video' % self.number_ids)
if (verbose):
print("Gaze sample count %i" % len(self.dataRTGene))
else:
print("WARNING: no RT-Gene data avaibale.")
# Read Gaze Data
if "HeadPose" in data:
self.dataGaze = data["HeadPose"]
self.number_ids = len([col for col in self.dataGaze.columns if 'head' in col])
print('Detected %i IDs in video' % self.number_ids)
if (verbose):
print("Gaze sample count %i" % len(self.dataGaze))
else:
print("WARNING: no gaze data avaibale.")
"""
# Read OpenPose Data
f = self.__get_filename_with_substring("OpenPose")
if os.path.isfile(f):
self.dataPose = pd.read_pickle(f)
if (verbose):
print("Pose sample count %i" % len(self.dataPose))
else:
print("WARNING: no pose data avaibale.")
"""
# Read Movement Data
if "BodyMovement" in data:
self.dataMovement = data["BodyMovement"]
if verbose:
print('Body movement sample count %i' % len(self.dataMovement))
else:
print('WARNING: no body movement data available.')
def getFrameCount(self):
return self.frameCount
def getFrameSize(self):
return self.frameSize
def getFPS(self):
return self.fps
def getVideo(self):
return self.movie_fileName
def getGazeData(self):
return self.dataGaze
def getFrame(self, frameIdx):
return frameIdx
def getFrameCurrent(self):
return 1
def getNumberIDs(self):
return self.number_ids
def getMovementData(self):
return self.dataMovement
def getRTGeneData(self):
return self.dataRTGene
def getImage(self):
return self.image

183
gui.ui Normal file
View File

@ -0,0 +1,183 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>ConAn</class>
<widget class="QMainWindow" name="ConAn">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>800</width>
<height>450</height>
</rect>
</property>
<property name="windowTitle">
<string>MainWindow</string>
</property>
<widget class="QWidget" name="centralwidget">
<layout class="QGridLayout" name="gridLayout">
<item row="4" column="0" colspan="2">
<widget class="QTabWidget" name="tab">
<property name="sizePolicy">
<sizepolicy hsizetype="Preferred" vsizetype="Preferred">
<horstretch>4</horstretch>
<verstretch>1</verstretch>
</sizepolicy>
</property>
<property name="minimumSize">
<size>
<width>0</width>
<height>300</height>
</size>
</property>
<property name="currentIndex">
<number>4</number>
</property>
<widget class="WidgetGaze" name="gaze">
<attribute name="title">
<string>Eye Gaze</string>
</attribute>
</widget>
<widget class="WidgetSpeaking" name="speak">
<attribute name="title">
<string>Speaking Activity</string>
</attribute>
</widget>
<widget class="WidgetPose" name="pose">
<property name="sizePolicy">
<sizepolicy hsizetype="Preferred" vsizetype="Ignored">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<attribute name="title">
<string>Body Pose</string>
</attribute>
</widget>
<widget class="WidgetFacialExpression" name="face">
<attribute name="title">
<string>Facial Expression</string>
</attribute>
</widget>
<widget class="WidgetObject" name="object">
<attribute name="title">
<string>Object Detection</string>
</attribute>
</widget>
</widget>
</item>
<item row="2" column="0" colspan="2">
<widget class="WidgetTimeLine" name="time" native="true">
<property name="sizePolicy">
<sizepolicy hsizetype="Preferred" vsizetype="Preferred">
<horstretch>4</horstretch>
<verstretch>1</verstretch>
</sizepolicy>
</property>
<property name="minimumSize">
<size>
<width>0</width>
<height>60</height>
</size>
</property>
</widget>
</item>
<item row="0" column="0">
<widget class="WidgetPlayer" name="video" native="true">
<property name="sizePolicy">
<sizepolicy hsizetype="Preferred" vsizetype="Preferred">
<horstretch>3</horstretch>
<verstretch>1</verstretch>
</sizepolicy>
</property>
</widget>
</item>
<item row="0" column="1">
<widget class="WidgetVideoSettings" name="vsettings" native="true">
<property name="sizePolicy">
<sizepolicy hsizetype="Preferred" vsizetype="Preferred">
<horstretch>1</horstretch>
<verstretch>1</verstretch>
</sizepolicy>
</property>
</widget>
</item>
</layout>
</widget>
<widget class="QMenuBar" name="menubar">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>800</width>
<height>21</height>
</rect>
</property>
</widget>
<widget class="QStatusBar" name="statusbar"/>
<action name="actionOpen">
<property name="text">
<string>Open</string>
</property>
</action>
<action name="actionExit">
<property name="text">
<string>Exit</string>
</property>
</action>
</widget>
<customwidgets>
<customwidget>
<class>WidgetTimeLine</class>
<extends>QWidget</extends>
<header>uiwidget/widgettimeline</header>
<container>1</container>
</customwidget>
<customwidget>
<class>WidgetPlayer</class>
<extends>QWidget</extends>
<header>uiwidget/widgetplayer</header>
<container>1</container>
</customwidget>
<customwidget>
<class>WidgetPose</class>
<extends>QWidget</extends>
<header>uiwidget/widgetpose</header>
<container>1</container>
</customwidget>
<customwidget>
<class>WidgetSpeaking</class>
<extends>QWidget</extends>
<header>uiwidget/widgetspeaking</header>
<container>1</container>
</customwidget>
<customwidget>
<class>WidgetGaze</class>
<extends>QWidget</extends>
<header>uiwidget/widgetgaze</header>
<container>1</container>
</customwidget>
<customwidget>
<class>WidgetFacialExpression</class>
<extends>QWidget</extends>
<header>uiwidget/widgetfacialexpression</header>
<container>1</container>
</customwidget>
<customwidget>
<class>WidgetVideoSettings</class>
<extends>QWidget</extends>
<header>uiwidget/widgetvideosettings</header>
<container>1</container>
</customwidget>
<customwidget>
<class>WidgetObject</class>
<extends>QWidget</extends>
<header location="global">uiwidget/widgetobject</header>
<container>1</container>
</customwidget>
</customwidgets>
<tabstops>
<tabstop>tab</tabstop>
</tabstops>
<resources/>
<connections/>
</ui>

BIN
icons/logo.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.3 KiB

BIN
icons/pause.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.5 KiB

BIN
icons/play.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.8 KiB

BIN
icons/stop.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 KiB

287
main.py Normal file
View File

@ -0,0 +1,287 @@
#!/usr/bin/env python3
import sys
import json
import numpy as np
import pandas as pd
import os
import PyQt5
from PyQt5 import QtGui
from PyQt5 import QtCore
from PyQt5.QtCore import QSize, QUrl, pyqtSignal, QFile, QTextStream
from PyQt5.QtGui import QIcon
from PyQt5.QtMultimedia import QMediaContent
from PyQt5.QtWidgets import QAction, QFileDialog, QStatusBar
from PyQt5.QtGui import QPalette, QColor
print("Qt version {0}".format(QtCore.QT_VERSION_STR))
from PyQt5 import QtWidgets
import pyqtgraph as pg
import cv2
print("OpenCV version {0} ".format(cv2.__version__))
from uiwidget import widgetplayer, widgettimeline, widgetgaze, widgetspeaking, widgetpose, widgetfacialexpression, widgetobject, widgetvideosettings
from processing import Processor
class MainWindow(QtWidgets.QMainWindow):
sendPath = pyqtSignal(str)
sendSegments = pyqtSignal(np.ndarray)
def __init__(self, verbose=False, parent=None):
super(MainWindow, self).__init__(parent)
self.setWindowTitle('Conversation Analysis')
PyQt5.uic.loadUi('gui.ui', self) # Load the .ui file
self.show() # Show the GUI
self.resize(1600, 900)
self.dataContainer = None
self.verbose = verbose
self.movieDir = ''
self.segments = []
self.frames_bitmask = None
self.frame = None
self.frameCount = None
self.fps = None
self._setupMenuBar()
self.status = QStatusBar()
self.setStatusBar(self.status)
self.status.showMessage('Welcome', 1000)
self.wPlayer = self.findChild(widgetplayer.WidgetPlayer, "video")
self.wTimeLine = self.findChild(widgettimeline.WidgetTimeLine, "time")
self.wTab = self.findChild(QtWidgets.QTabWidget, "tab")
self.wGaze = self.findChild(widgetgaze.WidgetGaze, "gaze")
self.wSpeaking = self.findChild(widgetspeaking.WidgetSpeaking, "speak")
self.wPose = self.findChild(widgetpose.WidgetPose, "pose")
self.wFace = self.findChild(widgetfacialexpression.WidgetFacialExpression, "face")
self.wObject = self.findChild(widgetobject.WidgetObject, "object")
self.wVideoSettings = self.findChild(widgetvideosettings.WidgetVideoSettings, "vsettings")
self.wPlayer.mediaPlayer.positionChanged.connect(self.wTimeLine.updateSlider)
self.wPlayer.sendState.connect(self.wTimeLine.mediaStateChanged)
self.wTimeLine.signalSetPosition.connect(self.wPlayer.mediaPlayer.setPosition)
self.wTimeLine.signalPlay.connect(self.wPlayer.play)
self.processor = Processor()
self.processor.signalInit.connect(self.wPose.initMovementGraph)
self.processor.signalInit.connect(self.wSpeaking.initSpeakingGraph)
self.processor.signalInit.connect(self.wGaze.initGazeGraph)
self.processor.signalInitTags.connect(self.wObject.setInit)
self.processor.signalInit.connect(self.wFace.setInit)
self.processor.signalInit.connect(self.wVideoSettings.setInit)
self.wPlayer.frameAvailable.connect(self.processor.saveCurrentFrameData)
self.processor.signalPoseSetInit.connect(self.wPose.setInit)
self.processor.signalGazeSetInit.connect(self.wGaze.setInit)
self.processor.signalSpeakerSetInit.connect(self.wSpeaking.setInit)
self.processor.signalUpdateHandVelocity.connect(self.wPose.updateHandVelocity)
self.processor.signalUpdateMovementGraph.connect(self.wPose.updateMovementGraph)
self.processor.signalUpdateSpeakGraph.connect(self.wSpeaking.updateSpeakingGraph)
self.processor.signalUpdateFaceAus.connect(self.wFace.updateFrame)
self.processor.signalUpdateFaceImgs.connect(self.wFace.updateImages)
self.processor.signalUpdateTagGraph.connect(self.wObject.updateTagGraph)
self.processor.signalVideoLabel.connect(self.wPlayer.draw_labels)
self.processor.signalPosePoints.connect(self.wPlayer.draw_pose)
self.processor.signalPoseChangedLabels.connect(self.wPose.updateLables)
self.processor.signalSpeakChangedLabels.connect(self.wSpeaking.updateLables)
self.processor.signalUpdateGazeGraph.connect(self.wGaze.updateGazeGraph)
self.processor.signalUpdateGazeMap.connect(self.wPlayer.draw_gaze)
self.processor.signalUpdateTags.connect(self.wPlayer.draw_tags)
self.wVideoSettings.signalVisualize.connect(self.processor.onVisualize)
self.wVideoSettings.signalSelectID.connect(self.wPose.onSelectedID)
self.wVideoSettings.signalSelectID.connect(self.wGaze.onSelectedID)
self.wVideoSettings.signalSelectID.connect(self.wPlayer.onSelectedID)
self.wVideoSettings.signalSelectID.connect(self.processor.onSelectedID)
self.processor.signalClearLabels.connect(self.wPlayer.clear_labels)
self.processor.signalClearPose.connect(self.wPlayer.clear_pose)
self.processor.signalClearGaze.connect(self.wPlayer.clear_gaze)
self.processor.signalClearTags.connect(self.wPlayer.clear_tags)
self.processor.signalDeactivatePoseTab.connect(self.togglePoseTab)
self.processor.signalDeactivateFaceTab.connect(self.toggleFaceTab)
self.processor.signalDeactivateGazeTab.connect(self.toggleGazeTab)
self.processor.signalDeactivateSpeakingTab.connect(self.toggleSpeakingTab)
self.processor.signalDeactivateObjectTab.connect(self.toggleObjectTab)
self.wTab.setCurrentIndex(0)
self.wTab.currentChanged.connect(self.processor.tabChanged)
def openProject(self, movie_fileName, data_fileName):
self.status.showMessage('Reading data...')
self.processor.readData(movie_fileName, data_fileName)
self.processor.calculateMovementMeasures()
self.processor.calculateSpeakingMeasures()
self.wPlayer.mediaPlayer.positionChanged.connect(self.processor.updateFrame)
self.wPlayer.mediaPlayer.positionChanged.connect(self.updateFrame)
self.fps = self.processor.getFPS()
# Variables for segment updates --> bool array for all frames
self.frame = 0
self.frameCount = self.processor.getFrameCount()
self.frames_bitmask = np.ones(self.frameCount)
self.segments = [(0, self.frameCount)]
self.wTimeLine.setInit(self.processor.getFrameCount(), self.processor.getNumberIDs(), self.processor.getFPS())
self.wPlayer.setInit(self.processor.getVideo(), self.processor.getFPS(),
self.processor.getOriginalVideoResolution(), self.processor.getNumberIDs(),
self.processor.getColors(), self.processor.getTags(), self.processor.getTagColors())
self.wTimeLine.rangeslider.segmentsChanged.connect(self.segmentsChanged)
self.sendSegments.connect(self.processor._updateSegments)
self.processor.setReady(True)
self.status.showMessage('Ready')
def segmentsChanged(self, segments, last_segment_changed):
self.segments = segments
self.frames_bitmask = np.zeros(self.frameCount)
for segment in segments:
start, end = segment
self.frames_bitmask[start:end] = np.ones((end - start))
self.sendSegments.emit(self.frames_bitmask)
start, end = segments[last_segment_changed]
if start > self.frame:
self.wPlayer.setFrame(start)
if end < self.frame:
self.wPlayer.pause()
self.wPlayer.setFrame(end)
"""
def closeEvent(self, event):
self.wPlayer.stop()
#quit_msg = "Are you sure you want to exit the program?"
# reply = QtGui.QMessageBox.question(self, 'Message',
# quit_msg, QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
#
# if reply == QtGui.QMessageBox.Yes:
# event.accept()
# else:
# event.ignore()
"""
def open(self):
movie_fileName, _ = QFileDialog.getOpenFileName(self, "Open Movie File", "", "Files (*.mp4)", self.movieDir)
data_fileName, _ = QFileDialog.getOpenFileName(
self, "Open Preprocessed File", "", "Files (*.dat)", self.movieDir)
if movie_fileName and data_fileName and os.path.isfile(movie_fileName) and os.path.isfile(data_fileName):
self.status.showMessage('Loading data... ')
self.openProject(movie_fileName, data_fileName)
else:
print("ERROR: select two files")
def export(self):
self.status.showMessage('Exporting data... This might take some time.')
# has to be defined in the main window, and then call the processing function
self.processor.export()
self.status.showMessage('Exported data successfully!')
def _setupMenuBar(self):
self.mainMenu = self.menuBar()
fileMenu = self.mainMenu.addMenu('&File')
openAct = QAction('&Open', self)
openAct.setStatusTip('Open files')
openAct.setShortcut('Ctrl+O')
openAct.triggered.connect(self.open)
fileMenu.addAction(openAct)
exportAct = QAction('&Export', self)
exportAct.setStatusTip('Export calculations')
exportAct.setShortcut('Ctrl+E')
exportAct.triggered.connect(self.export)
fileMenu.addAction(exportAct)
exitAct = QAction('&Exit', self)
exitAct.setStatusTip('Exit application')
exitAct.setShortcut('Ctrl+Q')
exitAct.triggered.connect(self.close)
fileMenu.addAction(exitAct)
#@QtCore.pyqtSlot(str)
# def statusUpdate(self, message):
# self.status.showMessage(message)
def updateFrame(self, position):
self.frame = int((position / 1000.0) * self.fps)
if self.frames_bitmask[self.frame] == 0:
for segment in self.segments:
if segment[0] > self.frame:
self.wPlayer.setFrame(segment[0] + 1)
break
@QtCore.pyqtSlot(bool)
def togglePoseTab(self, deactivate):
self.wTab.setTabEnabled(2, not deactivate)
@QtCore.pyqtSlot(bool)
def toggleGazeTab(self, deactivate):
self.wTab.setTabEnabled(0, not deactivate)
@QtCore.pyqtSlot(bool)
def toggleFaceTab(self, deactivate):
self.wTab.setTabEnabled(3, not deactivate)
@QtCore.pyqtSlot(bool)
def toggleSpeakingTab(self, deactivate):
self.wTab.setTabEnabled(1, not deactivate)
@QtCore.pyqtSlot(bool)
def toggleObjectTab(self, deactivate):
self.wTab.setTabEnabled(4, not deactivate)
if __name__ == "__main__":
app = QtWidgets.QApplication(sys.argv)
# print(sys.argv)
app.setWindowIcon(QtGui.QIcon('icons/logo.png'))
app.setStyle("Fusion")
# Now use a palette to switch to dark colors:
palette = QPalette()
palette.setColor(QPalette.Window, QColor(53, 53, 53))
palette.setColor(QPalette.WindowText, QtCore.Qt.white)
palette.setColor(QPalette.Base, QColor(25, 25, 25))
palette.setColor(QPalette.AlternateBase, QColor(53, 53, 53))
palette.setColor(QPalette.ToolTipBase, QtCore.Qt.black)
palette.setColor(QPalette.ToolTipText, QtCore.Qt.white)
palette.setColor(QPalette.Text, QtCore.Qt.white)
palette.setColor(QPalette.Button, QColor(53, 53, 53))
palette.setColor(QPalette.ButtonText, QtCore.Qt.white)
palette.setColor(QPalette.BrightText, QtCore.Qt.red)
palette.setColor(QPalette.Link, QColor(42, 130, 218))
palette.setColor(QPalette.Highlight, QColor(42, 130, 218))
palette.setColor(QPalette.HighlightedText, QtCore.Qt.black)
app.setPalette(palette)
if ("--verbose" in sys.argv):
verbose = True
print("### Verbose ENABLED")
else:
verbose = False
main = MainWindow(verbose=verbose)
main.setWindowTitle('ConAn: A Usable Tool for Multimodal Conversation Analysis')
main.show()
sys.exit(app.exec_())

1015
processing.py Normal file

File diff suppressed because it is too large Load Diff

5
requirements.txt Normal file
View File

@ -0,0 +1,5 @@
PyQt5==5.14.2
opencv-python==4.2.0.34
pandas
pyqtgraph
matplotlib

0
uiwidget/__init__.py Normal file
View File

View File

@ -0,0 +1,89 @@
import numpy as np
import pyqtgraph as pg
from PyQt5 import QtWidgets, QtCore, QtGui
from PyQt5.QtGui import QColor
lst_aus = ['AU1: Inner Brow Raiser', 'AU2: Outer Brow Raiser', 'AU4: Brow Lowerer', 'AU5: Upper Lid Raiser',
'AU6: Cheek Raiser', 'AU9: Nose Wrinkler', 'AU12: Lip Corner Puller', 'AU15: Lip Corner Depressor',
'AU17: Chin Raiser', 'AU20: Lip Stretcher', 'AU25: Lips Part', 'AU26: Jaw Drop']
class WidgetFacialExpression(QtWidgets.QWidget):
def __init__(self, parent=None):
super(WidgetFacialExpression, self).__init__(parent)
self.faceLayout = QtWidgets.QHBoxLayout()
self.setLayout(self.faceLayout)
self.numberIDs = None
self.valueLabels = dict()
self.imgPlots = dict()
@QtCore.pyqtSlot(list, int)
def setInit(self, colors, numberIDs):
self.numberIDs = numberIDs
for id_no in range(numberIDs):
idLayout = QtWidgets.QHBoxLayout()
labelNameLayout = QtWidgets.QVBoxLayout()
labelValueLayout = QtWidgets.QVBoxLayout()
imageLayout = QtWidgets.QVBoxLayout()
imageWidget = pg.PlotWidget(background=QColor(53, 53, 53))
imageWidget.invertY()
imageWidget.hideAxis('bottom'), imageWidget.hideAxis('left')
imageWidget.setMaximumHeight(150), imageWidget.setMaximumWidth(150)
imageWidget.setAspectLocked(True)
self.imgPlots[id_no] = imageWidget
color = tuple([int(a * 255) for a in colors[id_no]])
labelID = QtWidgets.QLabel('ID%i' % id_no)
labelID.setStyleSheet('font: bold 12px; color: black; background-color: rgb(%i,%i,%i)' % color)
#labelID.setFixedWidth(60)
labelNameLayout.addWidget(labelID)
labelID = QtWidgets.QLabel(' ')
#labelID.setStyleSheet('background-color: rgb(%i,%i,%i)' % color)
labelValueLayout.addWidget(labelID)
lst = []
for au in lst_aus:
nLabel = QtWidgets.QLabel(au)
labelNameLayout.addWidget(nLabel)
vLabel = QtWidgets.QLabel(' ')
labelValueLayout.addWidget(vLabel)
lst.append(vLabel)
self.valueLabels[id_no] = lst
idLayout.addWidget(imageWidget)
idLayout.addLayout(labelNameLayout)
idLayout.addLayout(labelValueLayout)
self.faceLayout.addLayout(idLayout)
@QtCore.pyqtSlot(dict, int)
def updateImages(self, imgs, id_no):
if imgs[id_no] is not None:
img = np.moveaxis(imgs[id_no], 0, 1)
img = pg.ImageItem(img)
self.imgPlots[id_no].addItem(img)
@QtCore.pyqtSlot(dict)
def updateFrame(self, aus):
if self.numberIDs is None:
return
for id_no in range(self.numberIDs):
if len(aus[id_no]) > 0:
for i, label in enumerate(self.valueLabels[id_no]):
if not np.any(np.isnan(np.array(aus[id_no].flatten()[0], dtype=np.float64))):
label.setText('%.2f' % aus[id_no].flatten()[0][i])

147
uiwidget/widgetgaze.py Normal file
View File

@ -0,0 +1,147 @@
import numpy as np
import pyqtgraph
import matplotlib
matplotlib.use("Qt5Agg")
import matplotlib.animation as animation
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.figure import Figure
from PyQt5 import QtWidgets, QtGui
from PyQt5 import QtCore
import pyqtgraph as pg
import pyqtgraph.exporters
from utils.colors import random_colors
from utils.util import sperical2equirec
class WidgetGaze(QtWidgets.QWidget):
def __init__(self, parent=None):
super(WidgetGaze, self).__init__(parent)
layout = QtWidgets.QGridLayout()
# Setup gaze graph
self.gazeGraph = pg.PlotWidget()
self.gazeGraph.setBackground('w')
self.gazeGraph.setYRange(-1.25, 1.25, padding=0)
self.gazeGraph.setXRange(-1.25, 1.25, padding=0)
self.gazeGraph.hideAxis('left')
self.gazeGraph.hideAxis('bottom')
self.gazeGraph.setAspectLocked()
self.gazeGraph.getPlotItem().setTitle(title='Top-down View of Gaze')
self.gazeGraphPlots = []
self.measures = QtWidgets.QWidget()
self.measuresLayout = QtWidgets.QHBoxLayout()
# self.gazeMap = QtWidgets.QWidget()
# self.heatmapSlider = HeatmapSlider()
# self.heatmapSlider.signalSetThreshold.connect(self.setThreshold)
# self.heatmapSlider.signalSaveImage.connect(self.gazeMap.saveImage)
# row, column, row span, column span
layout.addWidget(self.measures, 0, 1, 2, 1)
layout.addWidget(self.gazeGraph, 0, 0, 2, 1)
layout.setColumnStretch(0, 1)
layout.setColumnStretch(1, 1)
self.setLayout(layout)
# layout.addWidget(self.gazeMap, 0, 0, 3, 1)
# layout.addWidget(self.heatmapSlider, 3, 0, 1, 1)
self.gazeLabels = []
self.colors = None
@QtCore.pyqtSlot(dict, list, int)
def setInit(self, measures, colors, numberIDs):
"""Initialize measure widget with labels for all IDs"""
self.colors = colors # Necessary for ID updates
idLayout = QtWidgets.QVBoxLayout()
labelID = QtWidgets.QLabel(' ')
labelID.setFixedWidth(60)
labelID.setFixedHeight(20)
labelA = QtWidgets.QLabel('LookSomeone: ')
labelNoLook = QtWidgets.QLabel('TotalNoLook: ')
labelG = QtWidgets.QLabel('TotalWatched: ')
labelRatio = QtWidgets.QLabel('RatioWatcherLookSOne: ')
label = QtWidgets.QLabel('Tracked: ')
# labelVel = QtWidgets.QLabel('totNoLook: ')
idLayout.addWidget(labelID)
idLayout.addWidget(labelA)
idLayout.addWidget(labelNoLook)
idLayout.addWidget(labelG)
idLayout.addWidget(labelRatio)
idLayout.addWidget(label)
# idLayout.addWidget(labelVel)
self.measuresLayout.insertLayout(-1, idLayout)
for id_no in range(numberIDs):
idLayout = QtWidgets.QVBoxLayout()
color = tuple([int(a * 255) for a in colors[id_no]])
labelID = QtWidgets.QLabel('ID%i' % id_no)
labelID.setStyleSheet('font: bold 12px; color: black; background-color: rgb(%i,%i,%i)' % color)
labelID.setFixedWidth(60)
labelID.setFixedHeight(20)
# Look Someone
labelA = QtWidgets.QLabel('{:.2%}'.format(measures[id_no][1] / measures[id_no][2]))
labelNoLook = QtWidgets.QLabel('{:.2%}'.format((measures[id_no][2] - measures[id_no][1]) / measures[id_no][2]))
# Total Watched
labelG = QtWidgets.QLabel('{:.2%}'.format(measures[id_no][0] / measures[id_no][2]))
# ratio totWatcher / lookSomeone
labelRatio = QtWidgets.QLabel('{:.2}'.format(measures[id_no][0] / measures[id_no][1]))
label = QtWidgets.QLabel('%i frames' % measures[id_no][2])
# labelVel = QtWidgets.QLabel('%.2f' % np.random.uniform(0, 1))
idLayout.addWidget(labelID)
idLayout.addWidget(labelA)
idLayout.addWidget(labelNoLook)
idLayout.addWidget(labelG)
idLayout.addWidget(labelRatio)
idLayout.addWidget(label)
# idLayout.addWidget(labelVel)
# self.gazeLabels.append(labelVel)
self.measuresLayout.insertLayout(-1, idLayout)
self.measures.setLayout(self.measuresLayout)
@QtCore.pyqtSlot(list, int)
def initGazeGraph(self, colors, numberIDs):
""" initialize gaze graph """
# Big circle
x1, y1 = self.get_circle(radius=1)
self.gazeGraph.addItem(self.gazeGraph.plot(x1, y1, pen=pg.mkPen(0.5)))
# Camera
x2, y2 = self.get_circle(radius=0.02)
self.gazeGraph.addItem(self.gazeGraph.plot(x2, y2, pen=pg.mkPen(color=(0, 0, 0), width=3)))
for id_no in range(numberIDs):
color = tuple([int(a * 255) for a in colors[id_no]])
plt = self.gazeGraph.plot(x=[], y=[], pen=pg.mkPen(color=color, width=2))
self.gazeGraphPlots.append(plt)
@QtCore.pyqtSlot(dict, int)
def updateGazeGraph(self, data, numberIDs):
""" frame updates for gaze graph """
for id_no in range(numberIDs):
if data and id_no in data:
self.gazeGraphPlots[id_no].setData(data[id_no][0], data[id_no][1])
def get_circle(self, radius):
""" helper function returns circle to x, y coordinates"""
theta = np.linspace(0, 2 * np.pi, 100)
x = radius * np.cos(theta)
y = radius * np.sin(theta)
return np.array(x), np.array(y)
@QtCore.pyqtSlot(list)
def onSelectedID(self, lst):
"""Change color to None of gaze graph plot if ID should not be visible"""
for i, button in enumerate(lst):
if not button.isChecked():
self.gazeGraphPlots[i].setPen(None)
else:
color = tuple([int(a * 255) for a in self.colors[i]])
pen = pg.mkPen(color=color)
self.gazeGraphPlots[i].setPen(pen)

81
uiwidget/widgetobject.py Normal file
View File

@ -0,0 +1,81 @@
import numpy as np
import pyqtgraph
from PyQt5 import QtWidgets, QtCore
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QTextEdit
import pyqtgraph as pg
class WidgetObject(QtWidgets.QWidget):
def __init__(self, parent=None):
super(WidgetObject, self).__init__(parent)
self.frame = 0
self.tagFields = QtWidgets.QWidget()
self.tagLayout = QtWidgets.QVBoxLayout()
# Setup Graph Plot Widget
self.tagGraph = pg.PlotWidget()
self.tagGraph.setBackground('w')
self.tagGraph.setYRange(0, 400, padding=0)
self.tagGraph.getPlotItem().getAxis('bottom').setTickSpacing(minor=50, major=100)
self.tagGraph.getPlotItem().setTitle(title='Movement of Object Tags')
self.tagPlots = dict()
self.tagTextFields = dict()
self.plotText = dict()
layout = QtWidgets.QGridLayout()
layout.addWidget(self.tagGraph, 0, 0)
layout.addWidget(self.tagFields, 0, 1)
self.setLayout(layout)
@QtCore.pyqtSlot(list, tuple, dict, list)
def setInit(self, tags, frameSize, tracked, colors):
for i, tag in enumerate(tags):
label = QtWidgets.QLabel('Object tag #%i:' % tag)
label.setStyleSheet('color: black; background-color: rgb(%i,%i,%i)' % colors[i])
label.setFixedHeight(20)
field = QtWidgets.QTextEdit()
field.setFixedHeight(20)
field.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
field.textChanged.connect(self.tagTextChanged)
self.tagTextFields[tag] = field
trackedLabel = QtWidgets.QLabel('Tracked: {:.0%}'.format(tracked[tag]))
oneTagLayout = QtWidgets.QHBoxLayout()
oneTagLayout.addWidget(label)
oneTagLayout.addWidget(field)
oneTagLayout.addWidget(trackedLabel)
self.tagLayout.insertLayout(-1, oneTagLayout)
x = list(range(-200, 0)) # 200 time points
y = [0 for _ in range(200)] # 200 data points
dataLine = self.tagGraph.plot(x, y, pen=pg.mkPen(color=colors[i]))
self.tagPlots[tag] = dataLine
text = pg.TextItem(text='', color=colors[i])
text.setAnchor((1, i + 1))
self.plotText[tag] = text
self.tagGraph.addItem(text)
self.tagFields.setLayout(self.tagLayout)
@QtCore.pyqtSlot(dict)
def updateTagGraph(self, tagData):
for tag, values in tagData.items():
self.tagPlots[tag].setData(x=values[1], y=values[0]) # Update the data.
if tagData:
self.tagGraph.setXRange(np.min(values[1]), np.max(values[1]))
def tagTextChanged(self):
for tag, field in self.tagTextFields.items():
self.plotText[tag].setText(field.toPlainText())
x, y = self.tagPlots[tag].getData()
if len(x) > 0 and len(y) > 0:
#print(tag, x[-1], y[-1])
self.plotText[tag].setPos(x[-1], y[-1])

309
uiwidget/widgetplayer.py Normal file
View File

@ -0,0 +1,309 @@
from PyQt5 import QtCore, QtGui, QtWidgets, QtMultimedia, QtMultimediaWidgets, Qt
import os
import numpy as np
class WidgetPlayer(QtWidgets.QWidget):
updateFrame = QtCore.pyqtSignal(int)
# sendFileName = QtCore.pyqtSignal(str)
sendState = QtCore.pyqtSignal(QtMultimedia.QMediaPlayer.State)
frameAvailable = QtCore.pyqtSignal(QtGui.QImage)
labels = list()
colors = list()
pose_data = list()
gaze_data = list()
tag_data = dict()
tags = list()
tag_colors = dict()
def __init__(self, parent=None):
super(WidgetPlayer, self).__init__(parent)
self.root = QtCore.QFileInfo(__file__).absolutePath()
# mediaplayer for decoding the video
self.mediaPlayer = QtMultimedia.QMediaPlayer(self, QtMultimedia.QMediaPlayer.VideoSurface)
#self.mediaPlayer.setMuted(True)
# top = graphicsscene, middle = graphiscview, bottom = graphicsvideoitem, lowest = graphisctextitems, ...
self._scene = QtWidgets.QGraphicsScene(self)
self._scene.setBackgroundBrush(QtGui.QBrush(QtGui.QColor('black')))
self._gv = QtWidgets.QGraphicsView(self._scene)
self._videoitem = QtMultimediaWidgets.QGraphicsVideoItem()
self._videoitem.setPos(0, 0)
self._videoitem.setZValue(-1000)
self._scene.addItem(self._videoitem)
if os.name != 'nt':
# grab frames to forward them to facial emotion tab
probe = QtMultimedia.QVideoProbe(self)
probe.videoFrameProbed.connect(self.on_videoFrameProbed)
probe.setSource(self.mediaPlayer)
# disable scrollbars
self._gv.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
self._gv.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
# just a holder for the graphics view to expand to maximum to use full size
self.lay = QtWidgets.QVBoxLayout(self)
self.lay.setContentsMargins(0, 0, 0, 0)
self.lay.addWidget(self._gv)
self.errorLabel = QtWidgets.QLabel()
self.errorLabel.setSizePolicy(QtWidgets.QSizePolicy.Preferred,
QtWidgets.QSizePolicy.Maximum)
self.mediaPlayer.setVideoOutput(self._videoitem)
self.mediaPlayer.stateChanged.connect(self.on_stateChanged)
self.mediaPlayer.positionChanged.connect(self.mediaChangedPosition)
# self.mediaPlayer.durationChanged.connect(self.durationChanged)
self.mediaPlayer.error.connect(self.handleError)
self.movieDir = ''
self.duration = 0
def setInit(self, video, fps, originalVideoResolution, number_ids, colors, tags, tag_colors):
self.fps = fps
self.originalVideoResolution = originalVideoResolution
f = os.path.abspath(video)
self.mediaPlayer.setMedia(QtMultimedia.QMediaContent(QtCore.QUrl.fromLocalFile(f)))
self.mediaPlayer.setNotifyInterval(1000 // self.fps)
# init pose data
for i in range(number_ids):
self.pose_data.append(self._scene.addPath(QtGui.QPainterPath()))
# init gaze data
for i in range(number_ids):
self.gaze_data.append(self._scene.addPath(QtGui.QPainterPath()))
# init label data
for i in range(number_ids):
self.labels.append(self._scene.addPath(QtGui.QPainterPath()))
# init tag data
if tags:
for i, tag in enumerate(tags):
self.tag_data[tag] = self._scene.addPath(QtGui.QPainterPath())
self.tag_colors[tag] = tag_colors[i]
self.number_ids = number_ids
self.colors = colors
self.tags = tags
def play(self):
if self.mediaPlayer.state() == QtMultimedia.QMediaPlayer.PlayingState:
self.mediaPlayer.pause()
else:
self.mediaPlayer.play()
self.sendState.emit(self.mediaPlayer.state())
def pause(self):
if self.mediaPlayer.state() == QtMultimedia.QMediaPlayer.PlayingState:
self.mediaPlayer.pause()
self.sendState.emit(self.mediaPlayer.state())
def setFrame(self, frame):
# RESPECT FPS! position is time in millisconds
position = int(frame * 1000 / self.fps)
# print("Received", position)
self.mediaPlayer.setPosition(position)
def stop(self):
self.mediaPlayer.stop()
@QtCore.pyqtSlot(QtMultimedia.QMediaPlayer.State)
def on_stateChanged(self, state):
self.focus_on_video()
def mediaChangedPosition(self, position):
frame = int((position / 1000.0) * self.fps)
# print("Video Running %i" % frame)
self.updateFrame.emit(frame)
self._gv.fitInView(self._videoitem, QtCore.Qt.KeepAspectRatio)
def handleError(self):
# self.playButton.setEnabled(False)
print("Error: " + self.mediaPlayer.errorString())
def createButtons(self):
iconSize = QtCore.QSize(28, 28)
openButton = QtWidgets.QToolButton()
openButton.setStyleSheet('border: none;')
openButton.setIcon(QtGui.QIcon(self.root + '/icons/open.png'))
openButton.setIconSize(iconSize)
openButton.setToolTip("Open File")
# openButton.clicked.connect(self.open)
self.playButton = QtWidgets.QToolButton()
self.playButton.setStyleSheet('border: none;')
self.playButton.setIcon(QtGui.QIcon(self.root + '/icons/play.png'))
self.playButton.setIconSize(iconSize)
self.playButton.setToolTip("Play movie")
self.playButton.clicked.connect(self.play)
self.playButton.setEnabled(False)
self.stopButton = QtWidgets.QToolButton()
self.stopButton.setStyleSheet('border: none;')
self.stopButton.setIcon(QtGui.QIcon(self.root + '/icons/stop.png'))
self.stopButton.setIconSize(iconSize)
self.stopButton.setToolTip("Stop movie")
self.stopButton.clicked.connect(self.stop)
self.stopButton.setEnabled(False)
@QtCore.pyqtSlot(QtMultimedia.QVideoFrame)
def on_videoFrameProbed(self, frame):
cloneFrame = QtMultimedia.QVideoFrame(frame)
cloneFrame.map(QtMultimedia.QAbstractVideoBuffer.ReadOnly)
image = QtGui.QImage(cloneFrame.bits(), cloneFrame.width(), cloneFrame.height(), cloneFrame.bytesPerLine(),
QtMultimedia.QVideoFrame.imageFormatFromPixelFormat(cloneFrame.pixelFormat()))
self.frameAvailable.emit(image)
cloneFrame.unmap()
def focus_on_video(self):
native_video_resolution = self.mediaPlayer.metaData("Resolution")
# we also update the sceneview to zoom to the video
if native_video_resolution is not None:
self._videoitem.setSize(QtCore.QSizeF(native_video_resolution.width(), native_video_resolution.height()))
self._gv.fitInView(self._videoitem, QtCore.Qt.KeepAspectRatio)
# set scale of video to bigger size
if self.originalVideoResolution is not None:
width_ratio = self.originalVideoResolution[0] / native_video_resolution.width()
self._videoitem.setScale(width_ratio)
@QtCore.pyqtSlot()
def clear_tags(self):
self.focus_on_video()
# clear all tags
for tag in self.tags:
self._scene.removeItem(self.tag_data[tag])
self.tag_data[tag] = self._scene.addPath(QtGui.QPainterPath())
@QtCore.pyqtSlot(int, list, list)
def draw_tags(self, tag, lstX, lstY):
# this is removing the old tag data
self._scene.removeItem(self.tag_data[tag])
path = QtGui.QPainterPath()
path.setFillRule(Qt.Qt.WindingFill)
# set starting points
for (x, y) in zip(lstX, lstY):
path.addRect(x-50, y-50, 100, 100)
# by adding it gets converted into an QGraphicsPathItem
# save it for later removal
self.tag_data[tag] = self._scene.addPath(path)
# set colors
color = self.tag_colors[tag]
pen = QtGui.QPen(QtGui.QColor(color[0], color[1], color[2], 255), 2, QtCore.Qt.SolidLine)
self.tag_data[tag].setPen(pen)
# fill ellipses - alpha value is set to 50%
# self.tag_data[tag].setBrush(QtGui.QColor(color[0], color[1], color[2], int(0.5 * 255)))
@QtCore.pyqtSlot()
def clear_labels(self):
self.focus_on_video()
# clear all labels
for id_no in range(self.number_ids):
self._scene.removeItem(self.labels[id_no])
self.labels[id_no] = self._scene.addPath(QtGui.QPainterPath())
@QtCore.pyqtSlot(int, int, int)
def draw_labels(self, id_no, x, y):
# this is removing the old pose data
self._scene.removeItem(self.labels[id_no])
path = QtGui.QPainterPath()
# then draw text
font = QtGui.QFont("Arial", 70)
font.setStyleStrategy(QtGui.QFont.ForceOutline)
# sadly there is no easy way to claculate the width of the text so minus 100 is fine, but not ideal
# also moving the text up by 500, so that is does not cover the face
path.addText(x - 100, y - 300, font, "ID " + str(id_no))
# by adding it gets converted into an QGraphicsPathItem
# save it for later removal
self.labels[id_no] = self._scene.addPath(path)
# set colors
color = tuple([int(a * 255) for a in self.colors[id_no]])
# alpha value is set to 70%
pen = QtGui.QPen(QtGui.QColor(color[0], color[1], color[2], int(0.9 * 255)), 10, QtCore.Qt.SolidLine)
self.labels[id_no].setPen(pen)
@QtCore.pyqtSlot()
def clear_pose(self):
# empty pose data
for id_no in range(self.number_ids):
self._scene.removeItem(self.pose_data[id_no])
self.pose_data[id_no] = self._scene.addPath(QtGui.QPainterPath())
@QtCore.pyqtSlot(int, list, list)
def draw_pose(self, id_no, lstX, lstY):
# this is removing the old pose data
self._scene.removeItem(self.pose_data[id_no])
if len(lstX) > 0 and len(lstY) > 0:
path = QtGui.QPainterPath()
# set starting points
path.moveTo(lstX[0], lstY[0])
# then draw remaing lines
for (x, y) in zip(lstX[1:], lstY[1:]):
path.lineTo(x, y)
# by adding it gets converted into an QGraphicsPathItem
# save it for later removal
self.pose_data[id_no] = self._scene.addPath(path)
# set colors
color = tuple([int(a * 255) for a in self.colors[id_no]])
# alpha value is set to 70%
pen = QtGui.QPen(QtGui.QColor(color[0], color[1], color[2], int(0.7 * 255)), 10, QtCore.Qt.SolidLine)
self.pose_data[id_no].setPen(pen)
else:
self.pose_data[id_no] = self._scene.addPath(QtGui.QPainterPath())
@QtCore.pyqtSlot()
def clear_gaze(self):
# empty pose data
for id_no in range(self.number_ids):
self._scene.removeItem(self.gaze_data[id_no])
self.gaze_data[id_no] = self._scene.addPath(QtGui.QPainterPath())
@QtCore.pyqtSlot(int, list, list)
def draw_gaze(self, id_no, lstX, lstY):
# this is removing the old pose data
self._scene.removeItem(self.gaze_data[id_no])
path = QtGui.QPainterPath()
path.setFillRule(Qt.Qt.WindingFill)
# set starting points
for (x, y) in zip(lstX, lstY):
path.addEllipse(x, y, 100, 100)
# by adding it gets converted into an QGraphicsPathItem
# save it for later removal
self.gaze_data[id_no] = self._scene.addPath(path)
# set colors
color = tuple([int(a * 255) for a in self.colors[id_no]])
# alpha value is set to 50%
pen = QtGui.QPen(QtGui.QColor(color[0], color[1], color[2], int(0.5 * 255)), 1, QtCore.Qt.SolidLine)
self.gaze_data[id_no].setPen(pen)
# fill ellipses
self.gaze_data[id_no].setBrush(QtGui.QColor(color[0], color[1], color[2], int(0.5 * 255)))
@QtCore.pyqtSlot(list)
def onSelectedID(self, lst):
self.clear_labels()
self.clear_gaze()
self.clear_pose()

147
uiwidget/widgetpose.py Normal file
View File

@ -0,0 +1,147 @@
import numpy as np
import pyqtgraph as pg
from PyQt5 import QtWidgets, QtGui, QtCore
class WidgetPose(QtWidgets.QWidget):
video_label_signal = QtCore.pyqtSignal(list)
def __init__(self, parent=None):
super(WidgetPose, self).__init__(parent)
layout = QtWidgets.QGridLayout()
self.poseGraph = QtWidgets.QWidget()
self.measures = QtWidgets.QWidget()
self.measuresLayout = QtWidgets.QHBoxLayout()
# Setup Movement Graph Plot Widget
self.movementGraph = pg.PlotWidget()
self.movementGraph.setBackground('w')
self.movementGraph.setYRange(0, 400, padding=0)
self.movementGraph.getPlotItem().getAxis('bottom').setTickSpacing(minor=50, major=100)
self.movementGraph.getPlotItem().setTitle(title='Body Movement over Time')
self.movementPlots = []
layout.addWidget(self.movementGraph, 1, 0, 1, 1)
layout.addWidget(self.measures, 1, 1, 1, 1)
layout.setColumnStretch(0, 1)
layout.setColumnStretch(1, 1)
self.setLayout(layout)
self.colors = None
self.labels = dict()
@QtCore.pyqtSlot(list)
def onSelectedID(self, lst):
"""Change color of movement graph plot if ID should not be visible"""
for i, button in enumerate(lst):
if not button.isChecked():
self.movementPlots[i].setPen(None)
elif self.colors is not None:
color = tuple([int(a * 255) for a in self.colors[i]])
pen = pg.mkPen(color=color)
self.movementPlots[i].setPen(pen)
@QtCore.pyqtSlot(dict, list, int)
def updateMovementGraph(self, data, colors, numberIDs):
"""Plot ID specific movement data from processing class
data[id: (movements, frames)]
"""
# handle NaN https://github.com/pyqtgraph/pyqtgraph/issues/1057
# downgrade to 5.13 fixes the issue
for id_no in range(numberIDs):
if data.get(id_no):
if not np.all(np.isnan(data.get(id_no)[0])):
self.movementPlots[id_no].setData(data.get(id_no)[1], data.get(id_no)[0]) # Update the data.
self.movementGraph.setXRange(np.min(data.get(id_no)[1]), np.max(data.get(id_no)[1]))
@QtCore.pyqtSlot(dict, int)
def updateHandVelocity(self, data, numberIDs):
"""Update Velocity Label
data[id: velocity for frame]"""
for id_no in range(numberIDs):
if data.get(id_no) is not None:
self.labels['Velocity'][id_no].setText('%.2f' % data[id_no])
else:
self.labels['Velocity'][id_no].setText(' ')
@QtCore.pyqtSlot(list, int)
def initMovementGraph(self, colors, numberIDs):
"""Initialize plot lines with 0
colors: plot color for each ID
"""
for i in range(numberIDs):
x = list(range(-200, 0)) # 100 time points
y = [0 for _ in range(200)] # 100 data points
color = tuple([int(a * 255) for a in colors[i]])
pen = pg.mkPen(color=color)
dataLine = self.movementGraph.plot(x, y, pen=pen)
self.movementPlots.append(dataLine)
@QtCore.pyqtSlot(dict, dict, list, int)
def setInit(self, mostActivity, hand, colors, numberIDs):
self.colors = colors
idLayout = QtWidgets.QVBoxLayout()
labelID = QtWidgets.QLabel(' ')
labelID.setFixedWidth(60)
labelID.setFixedHeight(20)
label = QtWidgets.QLabel('Most body activity in frame: ')
labelA = QtWidgets.QLabel('Hands above table (relative): ')
labelG = QtWidgets.QLabel('Gestures (relative): ')
labelVel = QtWidgets.QLabel('Hand velocity: ')
idLayout.addWidget(labelID)
idLayout.addWidget(label)
idLayout.addWidget(labelA)
idLayout.addWidget(labelG)
idLayout.addWidget(labelVel)
self.measuresLayout.insertLayout(-1, idLayout)