Added GUI
This commit is contained in:
parent
ae9774cf0d
commit
74df5cb3f0
32
README.md
32
README.md
|
@ -1,3 +1,31 @@
|
|||
# conan
|
||||
# ConAn
|
||||
This is the official repository for [ConAn: A Usable Tool for Multimodal <u>Con</u>versation <u>An</u>alysis](https://www.perceptualui.org/publications/penzkofer21_icmi.pdf) <br>
|
||||
ConAn – our graphical tool for multimodal conversation analysis – takes 360 degree videos recorded during multiperson group interactions as input. ConAn integrates state-of-the-art models for gaze estimation, active speaker detection,
|
||||
facial action unit detection, and body movement detection and can output quantitative reports both at individual and group
|
||||
level, as well as different visualizations that provide qualitative insights into group interaction.
|
||||
|
||||
ConAn: A Usable Tool for Multimodal <u>Con</u>versation <u>An</u>alysis
|
||||
## Installation
|
||||
For the graphical user interface (GUI) you need python>3.6 to install the [requirements](requirements.txt) via pip:
|
||||
```
|
||||
pip install requirements.txt
|
||||
```
|
||||
## Get Started
|
||||
To test the GUI you can download our example use case videos from googledrive: <br>
|
||||
As well as the respective processed ``.dat`` files which include all the analyses.
|
||||
Run [main.py](main.py) and import the video file you would like to analyze.
|
||||
## Processing
|
||||
|
||||
|
||||
|
||||
## Citation
|
||||
Please cite this paper if you use ConAn or parts of this publication in your research:
|
||||
```
|
||||
@inproceedings{penzkofer21_icmi,
|
||||
author = {Penzkofer, Anna and Müller, Philipp and Bühler, Felix and Mayer, Sven and Bulling, Andreas},
|
||||
title = {ConAn: A Usable Tool for Multimodal Conversation Analysis},
|
||||
booktitle = {Proc. ACM International Conference on Multimodal Interaction (ICMI)},
|
||||
year = {2021},
|
||||
doi = {10.1145/3462244.3479886},
|
||||
video = {https://www.youtube.com/watch?v=H2KfZNgx6CQ}
|
||||
}
|
||||
```
|
151
container.py
Normal file
151
container.py
Normal file
|
@ -0,0 +1,151 @@
|
|||
import numpy as np
|
||||
import pandas as pd
|
||||
import pickle as pkl
|
||||
import os.path
|
||||
import cv2
|
||||
from PyQt5 import QtCore, QtGui
|
||||
|
||||
|
||||
class DataContainer():
|
||||
|
||||
def __init__(self, movie_fileName, data_fileName):
|
||||
self.movie_fileName = movie_fileName
|
||||
self.data_fileName = data_fileName
|
||||
|
||||
self.frameCount = 0
|
||||
self.fps = 0
|
||||
self.frameSize = [0, 0]
|
||||
|
||||
self.dataGaze = None
|
||||
self.dataMovement = None
|
||||
self.number_ids = None
|
||||
self.dataRTGene = None
|
||||
self.image = None
|
||||
|
||||
def readData(self, verbose=False, rtgene=False):
|
||||
if (verbose):
|
||||
print("## Start Reading Data")
|
||||
|
||||
# Read Video Data
|
||||
f = self.movie_fileName
|
||||
print(f)
|
||||
if os.path.isfile(f):
|
||||
cap = cv2.VideoCapture(f)
|
||||
ret, frame = cap.read()
|
||||
self.image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
|
||||
h, w, ch = self.image.shape
|
||||
bytesPerLine = ch * w
|
||||
convertToQtFormat = QtGui.QImage(self.image.data, w, h, bytesPerLine, QtGui.QImage.Format_RGB888)
|
||||
|
||||
self.fps = cap.get(cv2.CAP_PROP_FPS)
|
||||
self.frameCount = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
width = cap.get(cv2.CAP_PROP_FRAME_WIDTH) # float
|
||||
height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT) # float
|
||||
self.frameSize = [width, height]
|
||||
print(self.fps, self.frameCount)
|
||||
|
||||
if (verbose):
|
||||
print("Video frameCount %i" % self.frameCount)
|
||||
duration = self.frameCount / self.fps
|
||||
minutes = int(duration / 60)
|
||||
seconds = duration % 60
|
||||
print('Video duration (M:S) = ' + str(minutes) + ':' + str(seconds))
|
||||
else:
|
||||
print("WARNING: no video avaibale.")
|
||||
|
||||
# read data file
|
||||
with open(self.data_fileName, 'rb') as f:
|
||||
data = pkl.load(f)
|
||||
|
||||
# Read RT-Gene Gaze
|
||||
if rtgene:
|
||||
f = "%s_Gaze.pkl" % self.dataPath[0]
|
||||
df_gt = pd.read_pickle('exampledata/G2_VID1_GroundTruth.pkl')
|
||||
df_gt = df_gt[['Frame', 'ID0_target_spher', 'ID1_target_spher']]
|
||||
if os.path.isfile(f):
|
||||
df = pd.read_pickle(f)
|
||||
self.number_ids = len(df.PId.unique())
|
||||
|
||||
self.dataRTGene = df.pivot(index='Frame', columns="PId", values=["GazeTheta", "GazePhi", "HeadCenter",
|
||||
"HeadPoseYaw", "HeadPoseTheta",
|
||||
"Phi", "Theta"])
|
||||
lst = []
|
||||
for label in ["GazeTheta", "GazePhi", "Head", "HeadPoseYaw", "HeadPoseTheta", "Phi", "Theta"]:
|
||||
for head_id in range(self.number_ids):
|
||||
lst.append("ID%i_%s" % (head_id, label))
|
||||
self.dataRTGene.columns = lst
|
||||
self.dataRTGene = self.dataRTGene.reset_index()
|
||||
self.dataRTGene = pd.merge(self.dataRTGene, df_gt, on=['Frame'])
|
||||
self.dataRTGene = self.dataRTGene.rename(
|
||||
columns={'ID0_target_spher': 'ID1_target', 'ID1_target_spher': 'ID0_target'},
|
||||
errors='raise')
|
||||
|
||||
print('Detected %i IDs in video' % self.number_ids)
|
||||
if (verbose):
|
||||
print("Gaze sample count %i" % len(self.dataRTGene))
|
||||
else:
|
||||
print("WARNING: no RT-Gene data avaibale.")
|
||||
|
||||
# Read Gaze Data
|
||||
if "HeadPose" in data:
|
||||
self.dataGaze = data["HeadPose"]
|
||||
self.number_ids = len([col for col in self.dataGaze.columns if 'head' in col])
|
||||
|
||||
print('Detected %i IDs in video' % self.number_ids)
|
||||
if (verbose):
|
||||
print("Gaze sample count %i" % len(self.dataGaze))
|
||||
else:
|
||||
print("WARNING: no gaze data avaibale.")
|
||||
|
||||
"""
|
||||
# Read OpenPose Data
|
||||
f = self.__get_filename_with_substring("OpenPose")
|
||||
if os.path.isfile(f):
|
||||
self.dataPose = pd.read_pickle(f)
|
||||
if (verbose):
|
||||
print("Pose sample count %i" % len(self.dataPose))
|
||||
else:
|
||||
print("WARNING: no pose data avaibale.")
|
||||
"""
|
||||
|
||||
# Read Movement Data
|
||||
if "BodyMovement" in data:
|
||||
self.dataMovement = data["BodyMovement"]
|
||||
if verbose:
|
||||
print('Body movement sample count %i' % len(self.dataMovement))
|
||||
else:
|
||||
print('WARNING: no body movement data available.')
|
||||
|
||||
def getFrameCount(self):
|
||||
return self.frameCount
|
||||
|
||||
def getFrameSize(self):
|
||||
return self.frameSize
|
||||
|
||||
def getFPS(self):
|
||||
return self.fps
|
||||
|
||||
def getVideo(self):
|
||||
return self.movie_fileName
|
||||
|
||||
def getGazeData(self):
|
||||
return self.dataGaze
|
||||
|
||||
def getFrame(self, frameIdx):
|
||||
return frameIdx
|
||||
|
||||
def getFrameCurrent(self):
|
||||
return 1
|
||||
|
||||
def getNumberIDs(self):
|
||||
return self.number_ids
|
||||
|
||||
def getMovementData(self):
|
||||
return self.dataMovement
|
||||
|
||||
def getRTGeneData(self):
|
||||
return self.dataRTGene
|
||||
|
||||
def getImage(self):
|
||||
return self.image
|
183
gui.ui
Normal file
183
gui.ui
Normal file
|
@ -0,0 +1,183 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ui version="4.0">
|
||||
<class>ConAn</class>
|
||||
<widget class="QMainWindow" name="ConAn">
|
||||
<property name="geometry">
|
||||
<rect>
|
||||
<x>0</x>
|
||||
<y>0</y>
|
||||
<width>800</width>
|
||||
<height>450</height>
|
||||
</rect>
|
||||
</property>
|
||||
<property name="windowTitle">
|
||||
<string>MainWindow</string>
|
||||
</property>
|
||||
<widget class="QWidget" name="centralwidget">
|
||||
<layout class="QGridLayout" name="gridLayout">
|
||||
<item row="4" column="0" colspan="2">
|
||||
<widget class="QTabWidget" name="tab">
|
||||
<property name="sizePolicy">
|
||||
<sizepolicy hsizetype="Preferred" vsizetype="Preferred">
|
||||
<horstretch>4</horstretch>
|
||||
<verstretch>1</verstretch>
|
||||
</sizepolicy>
|
||||
</property>
|
||||
<property name="minimumSize">
|
||||
<size>
|
||||
<width>0</width>
|
||||
<height>300</height>
|
||||
</size>
|
||||
</property>
|
||||
<property name="currentIndex">
|
||||
<number>4</number>
|
||||
</property>
|
||||
<widget class="WidgetGaze" name="gaze">
|
||||
<attribute name="title">
|
||||
<string>Eye Gaze</string>
|
||||
</attribute>
|
||||
</widget>
|
||||
<widget class="WidgetSpeaking" name="speak">
|
||||
<attribute name="title">
|
||||
<string>Speaking Activity</string>
|
||||
</attribute>
|
||||
</widget>
|
||||
<widget class="WidgetPose" name="pose">
|
||||
<property name="sizePolicy">
|
||||
<sizepolicy hsizetype="Preferred" vsizetype="Ignored">
|
||||
<horstretch>0</horstretch>
|
||||
<verstretch>0</verstretch>
|
||||
</sizepolicy>
|
||||
</property>
|
||||
<attribute name="title">
|
||||
<string>Body Pose</string>
|
||||
</attribute>
|
||||
</widget>
|
||||
<widget class="WidgetFacialExpression" name="face">
|
||||
<attribute name="title">
|
||||
<string>Facial Expression</string>
|
||||
</attribute>
|
||||
</widget>
|
||||
<widget class="WidgetObject" name="object">
|
||||
<attribute name="title">
|
||||
<string>Object Detection</string>
|
||||
</attribute>
|
||||
</widget>
|
||||
</widget>
|
||||
</item>
|
||||
<item row="2" column="0" colspan="2">
|
||||
<widget class="WidgetTimeLine" name="time" native="true">
|
||||
<property name="sizePolicy">
|
||||
<sizepolicy hsizetype="Preferred" vsizetype="Preferred">
|
||||
<horstretch>4</horstretch>
|
||||
<verstretch>1</verstretch>
|
||||
</sizepolicy>
|
||||
</property>
|
||||
<property name="minimumSize">
|
||||
<size>
|
||||
<width>0</width>
|
||||
<height>60</height>
|
||||
</size>
|
||||
</property>
|
||||
</widget>
|
||||
</item>
|
||||
<item row="0" column="0">
|
||||
<widget class="WidgetPlayer" name="video" native="true">
|
||||
<property name="sizePolicy">
|
||||
<sizepolicy hsizetype="Preferred" vsizetype="Preferred">
|
||||
<horstretch>3</horstretch>
|
||||
<verstretch>1</verstretch>
|
||||
</sizepolicy>
|
||||
</property>
|
||||
</widget>
|
||||
</item>
|
||||
<item row="0" column="1">
|
||||
<widget class="WidgetVideoSettings" name="vsettings" native="true">
|
||||
<property name="sizePolicy">
|
||||
<sizepolicy hsizetype="Preferred" vsizetype="Preferred">
|
||||
<horstretch>1</horstretch>
|
||||
<verstretch>1</verstretch>
|
||||
</sizepolicy>
|
||||
</property>
|
||||
</widget>
|
||||
</item>
|
||||
</layout>
|
||||
</widget>
|
||||
<widget class="QMenuBar" name="menubar">
|
||||
<property name="geometry">
|
||||
<rect>
|
||||
<x>0</x>
|
||||
<y>0</y>
|
||||
<width>800</width>
|
||||
<height>21</height>
|
||||
</rect>
|
||||
</property>
|
||||
</widget>
|
||||
<widget class="QStatusBar" name="statusbar"/>
|
||||
<action name="actionOpen">
|
||||
<property name="text">
|
||||
<string>Open</string>
|
||||
</property>
|
||||
</action>
|
||||
<action name="actionExit">
|
||||
<property name="text">
|
||||
<string>Exit</string>
|
||||
</property>
|
||||
</action>
|
||||
</widget>
|
||||
<customwidgets>
|
||||
<customwidget>
|
||||
<class>WidgetTimeLine</class>
|
||||
<extends>QWidget</extends>
|
||||
<header>uiwidget/widgettimeline</header>
|
||||
<container>1</container>
|
||||
</customwidget>
|
||||
<customwidget>
|
||||
<class>WidgetPlayer</class>
|
||||
<extends>QWidget</extends>
|
||||
<header>uiwidget/widgetplayer</header>
|
||||
<container>1</container>
|
||||
</customwidget>
|
||||
<customwidget>
|
||||
<class>WidgetPose</class>
|
||||
<extends>QWidget</extends>
|
||||
<header>uiwidget/widgetpose</header>
|
||||
<container>1</container>
|
||||
</customwidget>
|
||||
<customwidget>
|
||||
<class>WidgetSpeaking</class>
|
||||
<extends>QWidget</extends>
|
||||
<header>uiwidget/widgetspeaking</header>
|
||||
<container>1</container>
|
||||
</customwidget>
|
||||
<customwidget>
|
||||
<class>WidgetGaze</class>
|
||||
<extends>QWidget</extends>
|
||||
<header>uiwidget/widgetgaze</header>
|
||||
<container>1</container>
|
||||
</customwidget>
|
||||
<customwidget>
|
||||
<class>WidgetFacialExpression</class>
|
||||
<extends>QWidget</extends>
|
||||
<header>uiwidget/widgetfacialexpression</header>
|
||||
<container>1</container>
|
||||
</customwidget>
|
||||
<customwidget>
|
||||
<class>WidgetVideoSettings</class>
|
||||
<extends>QWidget</extends>
|
||||
<header>uiwidget/widgetvideosettings</header>
|
||||
<container>1</container>
|
||||
</customwidget>
|
||||
<customwidget>
|
||||
<class>WidgetObject</class>
|
||||
<extends>QWidget</extends>
|
||||
<header location="global">uiwidget/widgetobject</header>
|
||||
<container>1</container>
|
||||
</customwidget>
|
||||
</customwidgets>
|
||||
<tabstops>
|
||||
<tabstop>tab</tabstop>
|
||||
</tabstops>
|
||||
<resources/>
|
||||
<connections/>
|
||||
</ui>
|
BIN
icons/logo.png
Normal file
BIN
icons/logo.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 4.3 KiB |
BIN
icons/pause.png
Normal file
BIN
icons/pause.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 2.5 KiB |
BIN
icons/play.png
Normal file
BIN
icons/play.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 2.8 KiB |
BIN
icons/stop.png
Normal file
BIN
icons/stop.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 1.6 KiB |
287
main.py
Normal file
287
main.py
Normal file
|
@ -0,0 +1,287 @@
|
|||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import json
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import os
|
||||
|
||||
import PyQt5
|
||||
from PyQt5 import QtGui
|
||||
from PyQt5 import QtCore
|
||||
from PyQt5.QtCore import QSize, QUrl, pyqtSignal, QFile, QTextStream
|
||||
from PyQt5.QtGui import QIcon
|
||||
from PyQt5.QtMultimedia import QMediaContent
|
||||
from PyQt5.QtWidgets import QAction, QFileDialog, QStatusBar
|
||||
from PyQt5.QtGui import QPalette, QColor
|
||||
|
||||
print("Qt version {0}".format(QtCore.QT_VERSION_STR))
|
||||
from PyQt5 import QtWidgets
|
||||
import pyqtgraph as pg
|
||||
|
||||
import cv2
|
||||
print("OpenCV version {0} ".format(cv2.__version__))
|
||||
|
||||
|
||||
from uiwidget import widgetplayer, widgettimeline, widgetgaze, widgetspeaking, widgetpose, widgetfacialexpression, widgetobject, widgetvideosettings
|
||||
from processing import Processor
|
||||
|
||||
|
||||
class MainWindow(QtWidgets.QMainWindow):
|
||||
sendPath = pyqtSignal(str)
|
||||
sendSegments = pyqtSignal(np.ndarray)
|
||||
|
||||
def __init__(self, verbose=False, parent=None):
|
||||
super(MainWindow, self).__init__(parent)
|
||||
self.setWindowTitle('Conversation Analysis')
|
||||
|
||||
PyQt5.uic.loadUi('gui.ui', self) # Load the .ui file
|
||||
self.show() # Show the GUI
|
||||
|
||||
self.resize(1600, 900)
|
||||
self.dataContainer = None
|
||||
self.verbose = verbose
|
||||
self.movieDir = ''
|
||||
self.segments = []
|
||||
self.frames_bitmask = None
|
||||
self.frame = None
|
||||
self.frameCount = None
|
||||
self.fps = None
|
||||
|
||||
self._setupMenuBar()
|
||||
self.status = QStatusBar()
|
||||
self.setStatusBar(self.status)
|
||||
self.status.showMessage('Welcome', 1000)
|
||||
|
||||
self.wPlayer = self.findChild(widgetplayer.WidgetPlayer, "video")
|
||||
self.wTimeLine = self.findChild(widgettimeline.WidgetTimeLine, "time")
|
||||
self.wTab = self.findChild(QtWidgets.QTabWidget, "tab")
|
||||
self.wGaze = self.findChild(widgetgaze.WidgetGaze, "gaze")
|
||||
self.wSpeaking = self.findChild(widgetspeaking.WidgetSpeaking, "speak")
|
||||
self.wPose = self.findChild(widgetpose.WidgetPose, "pose")
|
||||
self.wFace = self.findChild(widgetfacialexpression.WidgetFacialExpression, "face")
|
||||
self.wObject = self.findChild(widgetobject.WidgetObject, "object")
|
||||
self.wVideoSettings = self.findChild(widgetvideosettings.WidgetVideoSettings, "vsettings")
|
||||
|
||||
self.wPlayer.mediaPlayer.positionChanged.connect(self.wTimeLine.updateSlider)
|
||||
self.wPlayer.sendState.connect(self.wTimeLine.mediaStateChanged)
|
||||
|
||||
self.wTimeLine.signalSetPosition.connect(self.wPlayer.mediaPlayer.setPosition)
|
||||
self.wTimeLine.signalPlay.connect(self.wPlayer.play)
|
||||
|
||||
|
||||
self.processor = Processor()
|
||||
self.processor.signalInit.connect(self.wPose.initMovementGraph)
|
||||
self.processor.signalInit.connect(self.wSpeaking.initSpeakingGraph)
|
||||
self.processor.signalInit.connect(self.wGaze.initGazeGraph)
|
||||
self.processor.signalInitTags.connect(self.wObject.setInit)
|
||||
|
||||
self.processor.signalInit.connect(self.wFace.setInit)
|
||||
self.processor.signalInit.connect(self.wVideoSettings.setInit)
|
||||
|
||||
self.wPlayer.frameAvailable.connect(self.processor.saveCurrentFrameData)
|
||||
|
||||
self.processor.signalPoseSetInit.connect(self.wPose.setInit)
|
||||
self.processor.signalGazeSetInit.connect(self.wGaze.setInit)
|
||||
self.processor.signalSpeakerSetInit.connect(self.wSpeaking.setInit)
|
||||
self.processor.signalUpdateHandVelocity.connect(self.wPose.updateHandVelocity)
|
||||
self.processor.signalUpdateMovementGraph.connect(self.wPose.updateMovementGraph)
|
||||
self.processor.signalUpdateSpeakGraph.connect(self.wSpeaking.updateSpeakingGraph)
|
||||
self.processor.signalUpdateFaceAus.connect(self.wFace.updateFrame)
|
||||
self.processor.signalUpdateFaceImgs.connect(self.wFace.updateImages)
|
||||
self.processor.signalUpdateTagGraph.connect(self.wObject.updateTagGraph)
|
||||
self.processor.signalVideoLabel.connect(self.wPlayer.draw_labels)
|
||||
self.processor.signalPosePoints.connect(self.wPlayer.draw_pose)
|
||||
self.processor.signalPoseChangedLabels.connect(self.wPose.updateLables)
|
||||
self.processor.signalSpeakChangedLabels.connect(self.wSpeaking.updateLables)
|
||||
self.processor.signalUpdateGazeGraph.connect(self.wGaze.updateGazeGraph)
|
||||
self.processor.signalUpdateGazeMap.connect(self.wPlayer.draw_gaze)
|
||||
self.processor.signalUpdateTags.connect(self.wPlayer.draw_tags)
|
||||
|
||||
self.wVideoSettings.signalVisualize.connect(self.processor.onVisualize)
|
||||
self.wVideoSettings.signalSelectID.connect(self.wPose.onSelectedID)
|
||||
self.wVideoSettings.signalSelectID.connect(self.wGaze.onSelectedID)
|
||||
self.wVideoSettings.signalSelectID.connect(self.wPlayer.onSelectedID)
|
||||
self.wVideoSettings.signalSelectID.connect(self.processor.onSelectedID)
|
||||
|
||||
self.processor.signalClearLabels.connect(self.wPlayer.clear_labels)
|
||||
self.processor.signalClearPose.connect(self.wPlayer.clear_pose)
|
||||
self.processor.signalClearGaze.connect(self.wPlayer.clear_gaze)
|
||||
self.processor.signalClearTags.connect(self.wPlayer.clear_tags)
|
||||
|
||||
self.processor.signalDeactivatePoseTab.connect(self.togglePoseTab)
|
||||
self.processor.signalDeactivateFaceTab.connect(self.toggleFaceTab)
|
||||
self.processor.signalDeactivateGazeTab.connect(self.toggleGazeTab)
|
||||
self.processor.signalDeactivateSpeakingTab.connect(self.toggleSpeakingTab)
|
||||
self.processor.signalDeactivateObjectTab.connect(self.toggleObjectTab)
|
||||
|
||||
self.wTab.setCurrentIndex(0)
|
||||
self.wTab.currentChanged.connect(self.processor.tabChanged)
|
||||
|
||||
def openProject(self, movie_fileName, data_fileName):
|
||||
self.status.showMessage('Reading data...')
|
||||
self.processor.readData(movie_fileName, data_fileName)
|
||||
self.processor.calculateMovementMeasures()
|
||||
self.processor.calculateSpeakingMeasures()
|
||||
self.wPlayer.mediaPlayer.positionChanged.connect(self.processor.updateFrame)
|
||||
self.wPlayer.mediaPlayer.positionChanged.connect(self.updateFrame)
|
||||
self.fps = self.processor.getFPS()
|
||||
|
||||
# Variables for segment updates --> bool array for all frames
|
||||
self.frame = 0
|
||||
self.frameCount = self.processor.getFrameCount()
|
||||
self.frames_bitmask = np.ones(self.frameCount)
|
||||
self.segments = [(0, self.frameCount)]
|
||||
|
||||
self.wTimeLine.setInit(self.processor.getFrameCount(), self.processor.getNumberIDs(), self.processor.getFPS())
|
||||
self.wPlayer.setInit(self.processor.getVideo(), self.processor.getFPS(),
|
||||
self.processor.getOriginalVideoResolution(), self.processor.getNumberIDs(),
|
||||
self.processor.getColors(), self.processor.getTags(), self.processor.getTagColors())
|
||||
|
||||
self.wTimeLine.rangeslider.segmentsChanged.connect(self.segmentsChanged)
|
||||
self.sendSegments.connect(self.processor._updateSegments)
|
||||
self.processor.setReady(True)
|
||||
self.status.showMessage('Ready')
|
||||
|
||||
def segmentsChanged(self, segments, last_segment_changed):
|
||||
self.segments = segments
|
||||
self.frames_bitmask = np.zeros(self.frameCount)
|
||||
for segment in segments:
|
||||
start, end = segment
|
||||
self.frames_bitmask[start:end] = np.ones((end - start))
|
||||
self.sendSegments.emit(self.frames_bitmask)
|
||||
|
||||
start, end = segments[last_segment_changed]
|
||||
|
||||
if start > self.frame:
|
||||
self.wPlayer.setFrame(start)
|
||||
|
||||
if end < self.frame:
|
||||
self.wPlayer.pause()
|
||||
self.wPlayer.setFrame(end)
|
||||
|
||||
"""
|
||||
def closeEvent(self, event):
|
||||
self.wPlayer.stop()
|
||||
|
||||
#quit_msg = "Are you sure you want to exit the program?"
|
||||
# reply = QtGui.QMessageBox.question(self, 'Message',
|
||||
# quit_msg, QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
|
||||
#
|
||||
# if reply == QtGui.QMessageBox.Yes:
|
||||
# event.accept()
|
||||
# else:
|
||||
# event.ignore()
|
||||
"""
|
||||
|
||||
def open(self):
|
||||
movie_fileName, _ = QFileDialog.getOpenFileName(self, "Open Movie File", "", "Files (*.mp4)", self.movieDir)
|
||||
data_fileName, _ = QFileDialog.getOpenFileName(
|
||||
self, "Open Preprocessed File", "", "Files (*.dat)", self.movieDir)
|
||||
|
||||
if movie_fileName and data_fileName and os.path.isfile(movie_fileName) and os.path.isfile(data_fileName):
|
||||
self.status.showMessage('Loading data... ')
|
||||
self.openProject(movie_fileName, data_fileName)
|
||||
else:
|
||||
print("ERROR: select two files")
|
||||
|
||||
def export(self):
|
||||
self.status.showMessage('Exporting data... This might take some time.')
|
||||
# has to be defined in the main window, and then call the processing function
|
||||
self.processor.export()
|
||||
self.status.showMessage('Exported data successfully!')
|
||||
|
||||
def _setupMenuBar(self):
|
||||
self.mainMenu = self.menuBar()
|
||||
|
||||
fileMenu = self.mainMenu.addMenu('&File')
|
||||
|
||||
openAct = QAction('&Open', self)
|
||||
openAct.setStatusTip('Open files')
|
||||
openAct.setShortcut('Ctrl+O')
|
||||
openAct.triggered.connect(self.open)
|
||||
fileMenu.addAction(openAct)
|
||||
|
||||
exportAct = QAction('&Export', self)
|
||||
exportAct.setStatusTip('Export calculations')
|
||||
exportAct.setShortcut('Ctrl+E')
|
||||
exportAct.triggered.connect(self.export)
|
||||
fileMenu.addAction(exportAct)
|
||||
|
||||
exitAct = QAction('&Exit', self)
|
||||
exitAct.setStatusTip('Exit application')
|
||||
exitAct.setShortcut('Ctrl+Q')
|
||||
exitAct.triggered.connect(self.close)
|
||||
fileMenu.addAction(exitAct)
|
||||
|
||||
#@QtCore.pyqtSlot(str)
|
||||
# def statusUpdate(self, message):
|
||||
# self.status.showMessage(message)
|
||||
|
||||
def updateFrame(self, position):
|
||||
self.frame = int((position / 1000.0) * self.fps)
|
||||
if self.frames_bitmask[self.frame] == 0:
|
||||
for segment in self.segments:
|
||||
if segment[0] > self.frame:
|
||||
self.wPlayer.setFrame(segment[0] + 1)
|
||||
break
|
||||
|
||||
@QtCore.pyqtSlot(bool)
|
||||
def togglePoseTab(self, deactivate):
|
||||
self.wTab.setTabEnabled(2, not deactivate)
|
||||
|
||||
@QtCore.pyqtSlot(bool)
|
||||
def toggleGazeTab(self, deactivate):
|
||||
self.wTab.setTabEnabled(0, not deactivate)
|
||||
|
||||
@QtCore.pyqtSlot(bool)
|
||||
def toggleFaceTab(self, deactivate):
|
||||
self.wTab.setTabEnabled(3, not deactivate)
|
||||
|
||||
@QtCore.pyqtSlot(bool)
|
||||
def toggleSpeakingTab(self, deactivate):
|
||||
self.wTab.setTabEnabled(1, not deactivate)
|
||||
|
||||
@QtCore.pyqtSlot(bool)
|
||||
def toggleObjectTab(self, deactivate):
|
||||
self.wTab.setTabEnabled(4, not deactivate)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
app = QtWidgets.QApplication(sys.argv)
|
||||
# print(sys.argv)
|
||||
app.setWindowIcon(QtGui.QIcon('icons/logo.png'))
|
||||
|
||||
app.setStyle("Fusion")
|
||||
|
||||
# Now use a palette to switch to dark colors:
|
||||
palette = QPalette()
|
||||
palette.setColor(QPalette.Window, QColor(53, 53, 53))
|
||||
palette.setColor(QPalette.WindowText, QtCore.Qt.white)
|
||||
palette.setColor(QPalette.Base, QColor(25, 25, 25))
|
||||
palette.setColor(QPalette.AlternateBase, QColor(53, 53, 53))
|
||||
palette.setColor(QPalette.ToolTipBase, QtCore.Qt.black)
|
||||
palette.setColor(QPalette.ToolTipText, QtCore.Qt.white)
|
||||
palette.setColor(QPalette.Text, QtCore.Qt.white)
|
||||
palette.setColor(QPalette.Button, QColor(53, 53, 53))
|
||||
palette.setColor(QPalette.ButtonText, QtCore.Qt.white)
|
||||
palette.setColor(QPalette.BrightText, QtCore.Qt.red)
|
||||
palette.setColor(QPalette.Link, QColor(42, 130, 218))
|
||||
palette.setColor(QPalette.Highlight, QColor(42, 130, 218))
|
||||
palette.setColor(QPalette.HighlightedText, QtCore.Qt.black)
|
||||
|
||||
app.setPalette(palette)
|
||||
|
||||
|
||||
|
||||
if ("--verbose" in sys.argv):
|
||||
verbose = True
|
||||
print("### Verbose ENABLED")
|
||||
else:
|
||||
verbose = False
|
||||
|
||||
main = MainWindow(verbose=verbose)
|
||||
main.setWindowTitle('ConAn: A Usable Tool for Multimodal Conversation Analysis')
|
||||
main.show()
|
||||
|
||||
sys.exit(app.exec_())
|
1015
processing.py
Normal file
1015
processing.py
Normal file
File diff suppressed because it is too large
Load diff
5
requirements.txt
Normal file
5
requirements.txt
Normal file
|
@ -0,0 +1,5 @@
|
|||
PyQt5==5.14.2
|
||||
opencv-python==4.2.0.34
|
||||
pandas
|
||||
pyqtgraph
|
||||
matplotlib
|
0
uiwidget/__init__.py
Normal file
0
uiwidget/__init__.py
Normal file
89
uiwidget/widgetfacialexpression.py
Normal file
89
uiwidget/widgetfacialexpression.py
Normal file
|
@ -0,0 +1,89 @@
|
|||
import numpy as np
|
||||
|
||||
import pyqtgraph as pg
|
||||
from PyQt5 import QtWidgets, QtCore, QtGui
|
||||
from PyQt5.QtGui import QColor
|
||||
|
||||
lst_aus = ['AU1: Inner Brow Raiser', 'AU2: Outer Brow Raiser', 'AU4: Brow Lowerer', 'AU5: Upper Lid Raiser',
|
||||
'AU6: Cheek Raiser', 'AU9: Nose Wrinkler', 'AU12: Lip Corner Puller', 'AU15: Lip Corner Depressor',
|
||||
'AU17: Chin Raiser', 'AU20: Lip Stretcher', 'AU25: Lips Part', 'AU26: Jaw Drop']
|
||||
|
||||
class WidgetFacialExpression(QtWidgets.QWidget):
|
||||
def __init__(self, parent=None):
|
||||
super(WidgetFacialExpression, self).__init__(parent)
|
||||
|
||||
self.faceLayout = QtWidgets.QHBoxLayout()
|
||||
self.setLayout(self.faceLayout)
|
||||
|
||||
self.numberIDs = None
|
||||
self.valueLabels = dict()
|
||||
self.imgPlots = dict()
|
||||
|
||||
@QtCore.pyqtSlot(list, int)
|
||||
def setInit(self, colors, numberIDs):
|
||||
self.numberIDs = numberIDs
|
||||
|
||||
for id_no in range(numberIDs):
|
||||
idLayout = QtWidgets.QHBoxLayout()
|
||||
labelNameLayout = QtWidgets.QVBoxLayout()
|
||||
labelValueLayout = QtWidgets.QVBoxLayout()
|
||||
imageLayout = QtWidgets.QVBoxLayout()
|
||||
imageWidget = pg.PlotWidget(background=QColor(53, 53, 53))
|
||||
imageWidget.invertY()
|
||||
imageWidget.hideAxis('bottom'), imageWidget.hideAxis('left')
|
||||
imageWidget.setMaximumHeight(150), imageWidget.setMaximumWidth(150)
|
||||
imageWidget.setAspectLocked(True)
|
||||
self.imgPlots[id_no] = imageWidget
|
||||
|
||||
color = tuple([int(a * 255) for a in colors[id_no]])
|
||||
|
||||
labelID = QtWidgets.QLabel('ID%i' % id_no)
|
||||
labelID.setStyleSheet('font: bold 12px; color: black; background-color: rgb(%i,%i,%i)' % color)
|
||||
|
||||
|
||||
|
||||
#labelID.setFixedWidth(60)
|
||||
|
||||
labelNameLayout.addWidget(labelID)
|
||||
labelID = QtWidgets.QLabel(' ')
|
||||
|
||||
#labelID.setStyleSheet('background-color: rgb(%i,%i,%i)' % color)
|
||||
labelValueLayout.addWidget(labelID)
|
||||
lst = []
|
||||
for au in lst_aus:
|
||||
nLabel = QtWidgets.QLabel(au)
|
||||
labelNameLayout.addWidget(nLabel)
|
||||
|
||||
vLabel = QtWidgets.QLabel(' ')
|
||||
labelValueLayout.addWidget(vLabel)
|
||||
lst.append(vLabel)
|
||||
|
||||
self.valueLabels[id_no] = lst
|
||||
idLayout.addWidget(imageWidget)
|
||||
idLayout.addLayout(labelNameLayout)
|
||||
idLayout.addLayout(labelValueLayout)
|
||||
self.faceLayout.addLayout(idLayout)
|
||||
|
||||
@QtCore.pyqtSlot(dict, int)
|
||||
def updateImages(self, imgs, id_no):
|
||||
if imgs[id_no] is not None:
|
||||
img = np.moveaxis(imgs[id_no], 0, 1)
|
||||
img = pg.ImageItem(img)
|
||||
self.imgPlots[id_no].addItem(img)
|
||||
|
||||
|
||||
@QtCore.pyqtSlot(dict)
|
||||
def updateFrame(self, aus):
|
||||
if self.numberIDs is None:
|
||||
return
|
||||
|
||||
for id_no in range(self.numberIDs):
|
||||
if len(aus[id_no]) > 0:
|
||||
for i, label in enumerate(self.valueLabels[id_no]):
|
||||
if not np.any(np.isnan(np.array(aus[id_no].flatten()[0], dtype=np.float64))):
|
||||
label.setText('%.2f' % aus[id_no].flatten()[0][i])
|
||||
|
||||
|
||||
|
||||
|
||||
|
147
uiwidget/widgetgaze.py
Normal file
147
uiwidget/widgetgaze.py
Normal file
|
@ -0,0 +1,147 @@
|
|||
import numpy as np
|
||||
import pyqtgraph
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Qt5Agg")
|
||||
import matplotlib.animation as animation
|
||||
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
|
||||
from matplotlib.figure import Figure
|
||||
from PyQt5 import QtWidgets, QtGui
|
||||
from PyQt5 import QtCore
|
||||
import pyqtgraph as pg
|
||||
import pyqtgraph.exporters
|
||||
|
||||
from utils.colors import random_colors
|
||||
from utils.util import sperical2equirec
|
||||
|
||||
|
||||
class WidgetGaze(QtWidgets.QWidget):
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super(WidgetGaze, self).__init__(parent)
|
||||
|
||||
layout = QtWidgets.QGridLayout()
|
||||
|
||||
# Setup gaze graph
|
||||
self.gazeGraph = pg.PlotWidget()
|
||||
self.gazeGraph.setBackground('w')
|
||||
self.gazeGraph.setYRange(-1.25, 1.25, padding=0)
|
||||
self.gazeGraph.setXRange(-1.25, 1.25, padding=0)
|
||||
self.gazeGraph.hideAxis('left')
|
||||
self.gazeGraph.hideAxis('bottom')
|
||||
self.gazeGraph.setAspectLocked()
|
||||
self.gazeGraph.getPlotItem().setTitle(title='Top-down View of Gaze')
|
||||
self.gazeGraphPlots = []
|
||||
|
||||
self.measures = QtWidgets.QWidget()
|
||||
self.measuresLayout = QtWidgets.QHBoxLayout()
|
||||
|
||||
# self.gazeMap = QtWidgets.QWidget()
|
||||
# self.heatmapSlider = HeatmapSlider()
|
||||
# self.heatmapSlider.signalSetThreshold.connect(self.setThreshold)
|
||||
# self.heatmapSlider.signalSaveImage.connect(self.gazeMap.saveImage)
|
||||
|
||||
# row, column, row span, column span
|
||||
layout.addWidget(self.measures, 0, 1, 2, 1)
|
||||
layout.addWidget(self.gazeGraph, 0, 0, 2, 1)
|
||||
layout.setColumnStretch(0, 1)
|
||||
layout.setColumnStretch(1, 1)
|
||||
|
||||
self.setLayout(layout)
|
||||
# layout.addWidget(self.gazeMap, 0, 0, 3, 1)
|
||||
# layout.addWidget(self.heatmapSlider, 3, 0, 1, 1)
|
||||
self.gazeLabels = []
|
||||
self.colors = None
|
||||
|
||||
@QtCore.pyqtSlot(dict, list, int)
|
||||
def setInit(self, measures, colors, numberIDs):
|
||||
"""Initialize measure widget with labels for all IDs"""
|
||||
self.colors = colors # Necessary for ID updates
|
||||
idLayout = QtWidgets.QVBoxLayout()
|
||||
labelID = QtWidgets.QLabel(' ')
|
||||
labelID.setFixedWidth(60)
|
||||
labelID.setFixedHeight(20)
|
||||
labelA = QtWidgets.QLabel('LookSomeone: ')
|
||||
labelNoLook = QtWidgets.QLabel('TotalNoLook: ')
|
||||
labelG = QtWidgets.QLabel('TotalWatched: ')
|
||||
labelRatio = QtWidgets.QLabel('RatioWatcherLookSOne: ')
|
||||
label = QtWidgets.QLabel('Tracked: ')
|
||||
# labelVel = QtWidgets.QLabel('totNoLook: ')
|
||||
idLayout.addWidget(labelID)
|
||||
idLayout.addWidget(labelA)
|
||||
idLayout.addWidget(labelNoLook)
|
||||
idLayout.addWidget(labelG)
|
||||
idLayout.addWidget(labelRatio)
|
||||
idLayout.addWidget(label)
|
||||
# idLayout.addWidget(labelVel)
|
||||
self.measuresLayout.insertLayout(-1, idLayout)
|
||||
|
||||
for id_no in range(numberIDs):
|
||||
idLayout = QtWidgets.QVBoxLayout()
|
||||
|
||||
color = tuple([int(a * 255) for a in colors[id_no]])
|
||||
|
||||
labelID = QtWidgets.QLabel('ID%i' % id_no)
|
||||
labelID.setStyleSheet('font: bold 12px; color: black; background-color: rgb(%i,%i,%i)' % color)
|
||||
labelID.setFixedWidth(60)
|
||||
labelID.setFixedHeight(20)
|
||||
# Look Someone
|
||||
labelA = QtWidgets.QLabel('{:.2%}'.format(measures[id_no][1] / measures[id_no][2]))
|
||||
labelNoLook = QtWidgets.QLabel('{:.2%}'.format((measures[id_no][2] - measures[id_no][1]) / measures[id_no][2]))
|
||||
# Total Watched
|
||||
labelG = QtWidgets.QLabel('{:.2%}'.format(measures[id_no][0] / measures[id_no][2]))
|
||||
# ratio totWatcher / lookSomeone
|
||||
labelRatio = QtWidgets.QLabel('{:.2}'.format(measures[id_no][0] / measures[id_no][1]))
|
||||
label = QtWidgets.QLabel('%i frames' % measures[id_no][2])
|
||||
# labelVel = QtWidgets.QLabel('%.2f' % np.random.uniform(0, 1))
|
||||
idLayout.addWidget(labelID)
|
||||
idLayout.addWidget(labelA)
|
||||
idLayout.addWidget(labelNoLook)
|
||||
idLayout.addWidget(labelG)
|
||||
idLayout.addWidget(labelRatio)
|
||||
idLayout.addWidget(label)
|
||||
# idLayout.addWidget(labelVel)
|
||||
# self.gazeLabels.append(labelVel)
|
||||
self.measuresLayout.insertLayout(-1, idLayout)
|
||||
self.measures.setLayout(self.measuresLayout)
|
||||
|
||||
@QtCore.pyqtSlot(list, int)
|
||||
def initGazeGraph(self, colors, numberIDs):
|
||||
""" initialize gaze graph """
|
||||
# Big circle
|
||||
x1, y1 = self.get_circle(radius=1)
|
||||
self.gazeGraph.addItem(self.gazeGraph.plot(x1, y1, pen=pg.mkPen(0.5)))
|
||||
|
||||
# Camera
|
||||
x2, y2 = self.get_circle(radius=0.02)
|
||||
self.gazeGraph.addItem(self.gazeGraph.plot(x2, y2, pen=pg.mkPen(color=(0, 0, 0), width=3)))
|
||||
|
||||
for id_no in range(numberIDs):
|
||||
color = tuple([int(a * 255) for a in colors[id_no]])
|
||||
plt = self.gazeGraph.plot(x=[], y=[], pen=pg.mkPen(color=color, width=2))
|
||||
self.gazeGraphPlots.append(plt)
|
||||
|
||||
@QtCore.pyqtSlot(dict, int)
|
||||
def updateGazeGraph(self, data, numberIDs):
|
||||
""" frame updates for gaze graph """
|
||||
for id_no in range(numberIDs):
|
||||
if data and id_no in data:
|
||||
self.gazeGraphPlots[id_no].setData(data[id_no][0], data[id_no][1])
|
||||
|
||||
def get_circle(self, radius):
|
||||
""" helper function returns circle to x, y coordinates"""
|
||||
theta = np.linspace(0, 2 * np.pi, 100)
|
||||
x = radius * np.cos(theta)
|
||||
y = radius * np.sin(theta)
|
||||
return np.array(x), np.array(y)
|
||||
|
||||
@QtCore.pyqtSlot(list)
|
||||
def onSelectedID(self, lst):
|
||||
"""Change color to None of gaze graph plot if ID should not be visible"""
|
||||
for i, button in enumerate(lst):
|
||||
if not button.isChecked():
|
||||
self.gazeGraphPlots[i].setPen(None)
|
||||
else:
|
||||
color = tuple([int(a * 255) for a in self.colors[i]])
|
||||
pen = pg.mkPen(color=color)
|
||||
self.gazeGraphPlots[i].setPen(pen)
|
81
uiwidget/widgetobject.py
Normal file
81
uiwidget/widgetobject.py
Normal file
|
@ -0,0 +1,81 @@
|
|||
import numpy as np
|
||||
import pyqtgraph
|
||||
from PyQt5 import QtWidgets, QtCore
|
||||
from PyQt5.QtCore import Qt
|
||||
from PyQt5.QtWidgets import QTextEdit
|
||||
import pyqtgraph as pg
|
||||
|
||||
|
||||
class WidgetObject(QtWidgets.QWidget):
|
||||
def __init__(self, parent=None):
|
||||
super(WidgetObject, self).__init__(parent)
|
||||
|
||||
self.frame = 0
|
||||
self.tagFields = QtWidgets.QWidget()
|
||||
self.tagLayout = QtWidgets.QVBoxLayout()
|
||||
|
||||
# Setup Graph Plot Widget
|
||||
self.tagGraph = pg.PlotWidget()
|
||||
self.tagGraph.setBackground('w')
|
||||
self.tagGraph.setYRange(0, 400, padding=0)
|
||||
self.tagGraph.getPlotItem().getAxis('bottom').setTickSpacing(minor=50, major=100)
|
||||
self.tagGraph.getPlotItem().setTitle(title='Movement of Object Tags')
|
||||
self.tagPlots = dict()
|
||||
|
||||
self.tagTextFields = dict()
|
||||
self.plotText = dict()
|
||||
|
||||
layout = QtWidgets.QGridLayout()
|
||||
layout.addWidget(self.tagGraph, 0, 0)
|
||||
layout.addWidget(self.tagFields, 0, 1)
|
||||
self.setLayout(layout)
|
||||
|
||||
@QtCore.pyqtSlot(list, tuple, dict, list)
|
||||
def setInit(self, tags, frameSize, tracked, colors):
|
||||
|
||||
for i, tag in enumerate(tags):
|
||||
label = QtWidgets.QLabel('Object tag #%i:' % tag)
|
||||
label.setStyleSheet('color: black; background-color: rgb(%i,%i,%i)' % colors[i])
|
||||
label.setFixedHeight(20)
|
||||
field = QtWidgets.QTextEdit()
|
||||
field.setFixedHeight(20)
|
||||
field.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
|
||||
field.textChanged.connect(self.tagTextChanged)
|
||||
|
||||
self.tagTextFields[tag] = field
|
||||
trackedLabel = QtWidgets.QLabel('Tracked: {:.0%}'.format(tracked[tag]))
|
||||
oneTagLayout = QtWidgets.QHBoxLayout()
|
||||
oneTagLayout.addWidget(label)
|
||||
oneTagLayout.addWidget(field)
|
||||
oneTagLayout.addWidget(trackedLabel)
|
||||
|
||||
self.tagLayout.insertLayout(-1, oneTagLayout)
|
||||
|
||||
x = list(range(-200, 0)) # 200 time points
|
||||
y = [0 for _ in range(200)] # 200 data points
|
||||
|
||||
dataLine = self.tagGraph.plot(x, y, pen=pg.mkPen(color=colors[i]))
|
||||
self.tagPlots[tag] = dataLine
|
||||
text = pg.TextItem(text='', color=colors[i])
|
||||
text.setAnchor((1, i + 1))
|
||||
|
||||
self.plotText[tag] = text
|
||||
self.tagGraph.addItem(text)
|
||||
|
||||
self.tagFields.setLayout(self.tagLayout)
|
||||
|
||||
@QtCore.pyqtSlot(dict)
|
||||
def updateTagGraph(self, tagData):
|
||||
for tag, values in tagData.items():
|
||||
self.tagPlots[tag].setData(x=values[1], y=values[0]) # Update the data.
|
||||
if tagData:
|
||||
self.tagGraph.setXRange(np.min(values[1]), np.max(values[1]))
|
||||
|
||||
def tagTextChanged(self):
|
||||
for tag, field in self.tagTextFields.items():
|
||||
self.plotText[tag].setText(field.toPlainText())
|
||||
x, y = self.tagPlots[tag].getData()
|
||||
if len(x) > 0 and len(y) > 0:
|
||||
#print(tag, x[-1], y[-1])
|
||||
self.plotText[tag].setPos(x[-1], y[-1])
|
||||
|
309
uiwidget/widgetplayer.py
Normal file
309
uiwidget/widgetplayer.py
Normal file
|
@ -0,0 +1,309 @@
|
|||
from PyQt5 import QtCore, QtGui, QtWidgets, QtMultimedia, QtMultimediaWidgets, Qt
|
||||
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
|
||||
|
||||
class WidgetPlayer(QtWidgets.QWidget):
|
||||
updateFrame = QtCore.pyqtSignal(int)
|
||||
# sendFileName = QtCore.pyqtSignal(str)
|
||||
sendState = QtCore.pyqtSignal(QtMultimedia.QMediaPlayer.State)
|
||||
frameAvailable = QtCore.pyqtSignal(QtGui.QImage)
|
||||
|
||||
labels = list()
|
||||
colors = list()
|
||||
pose_data = list()
|
||||
gaze_data = list()
|
||||
tag_data = dict()
|
||||
tags = list()
|
||||
tag_colors = dict()
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super(WidgetPlayer, self).__init__(parent)
|
||||
self.root = QtCore.QFileInfo(__file__).absolutePath()
|
||||
|
||||
# mediaplayer for decoding the video
|
||||
self.mediaPlayer = QtMultimedia.QMediaPlayer(self, QtMultimedia.QMediaPlayer.VideoSurface)
|
||||
#self.mediaPlayer.setMuted(True)
|
||||
|
||||
# top = graphicsscene, middle = graphiscview, bottom = graphicsvideoitem, lowest = graphisctextitems, ...
|
||||
self._scene = QtWidgets.QGraphicsScene(self)
|
||||
self._scene.setBackgroundBrush(QtGui.QBrush(QtGui.QColor('black')))
|
||||
self._gv = QtWidgets.QGraphicsView(self._scene)
|
||||
self._videoitem = QtMultimediaWidgets.QGraphicsVideoItem()
|
||||
self._videoitem.setPos(0, 0)
|
||||
self._videoitem.setZValue(-1000)
|
||||
self._scene.addItem(self._videoitem)
|
||||
|
||||
if os.name != 'nt':
|
||||
# grab frames to forward them to facial emotion tab
|
||||
probe = QtMultimedia.QVideoProbe(self)
|
||||
probe.videoFrameProbed.connect(self.on_videoFrameProbed)
|
||||
probe.setSource(self.mediaPlayer)
|
||||
|
||||
# disable scrollbars
|
||||
self._gv.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
|
||||
self._gv.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
|
||||
|
||||
# just a holder for the graphics view to expand to maximum to use full size
|
||||
self.lay = QtWidgets.QVBoxLayout(self)
|
||||
self.lay.setContentsMargins(0, 0, 0, 0)
|
||||
self.lay.addWidget(self._gv)
|
||||
|
||||
self.errorLabel = QtWidgets.QLabel()
|
||||
self.errorLabel.setSizePolicy(QtWidgets.QSizePolicy.Preferred,
|
||||
QtWidgets.QSizePolicy.Maximum)
|
||||
|
||||
self.mediaPlayer.setVideoOutput(self._videoitem)
|
||||
self.mediaPlayer.stateChanged.connect(self.on_stateChanged)
|
||||
self.mediaPlayer.positionChanged.connect(self.mediaChangedPosition)
|
||||
# self.mediaPlayer.durationChanged.connect(self.durationChanged)
|
||||
self.mediaPlayer.error.connect(self.handleError)
|
||||
|
||||
self.movieDir = ''
|
||||
self.duration = 0
|
||||
|
||||
def setInit(self, video, fps, originalVideoResolution, number_ids, colors, tags, tag_colors):
|
||||
self.fps = fps
|
||||
self.originalVideoResolution = originalVideoResolution
|
||||
|
||||
f = os.path.abspath(video)
|
||||
self.mediaPlayer.setMedia(QtMultimedia.QMediaContent(QtCore.QUrl.fromLocalFile(f)))
|
||||
self.mediaPlayer.setNotifyInterval(1000 // self.fps)
|
||||
|
||||
# init pose data
|
||||
for i in range(number_ids):
|
||||
self.pose_data.append(self._scene.addPath(QtGui.QPainterPath()))
|
||||
|
||||
# init gaze data
|
||||
for i in range(number_ids):
|
||||
self.gaze_data.append(self._scene.addPath(QtGui.QPainterPath()))
|
||||
|
||||
# init label data
|
||||
for i in range(number_ids):
|
||||
self.labels.append(self._scene.addPath(QtGui.QPainterPath()))
|
||||
|
||||
# init tag data
|
||||
if tags:
|
||||
for i, tag in enumerate(tags):
|
||||
self.tag_data[tag] = self._scene.addPath(QtGui.QPainterPath())
|
||||
self.tag_colors[tag] = tag_colors[i]
|
||||
|
||||
|
||||
self.number_ids = number_ids
|
||||
self.colors = colors
|
||||
self.tags = tags
|
||||
|
||||
def play(self):
|
||||
if self.mediaPlayer.state() == QtMultimedia.QMediaPlayer.PlayingState:
|
||||
self.mediaPlayer.pause()
|
||||
else:
|
||||
self.mediaPlayer.play()
|
||||
self.sendState.emit(self.mediaPlayer.state())
|
||||
|
||||
def pause(self):
|
||||
if self.mediaPlayer.state() == QtMultimedia.QMediaPlayer.PlayingState:
|
||||
self.mediaPlayer.pause()
|
||||
self.sendState.emit(self.mediaPlayer.state())
|
||||
|
||||
def setFrame(self, frame):
|
||||
# RESPECT FPS! position is time in millisconds
|
||||
position = int(frame * 1000 / self.fps)
|
||||
# print("Received", position)
|
||||
self.mediaPlayer.setPosition(position)
|
||||
|
||||
def stop(self):
|
||||
self.mediaPlayer.stop()
|
||||
|
||||
@QtCore.pyqtSlot(QtMultimedia.QMediaPlayer.State)
|
||||
def on_stateChanged(self, state):
|
||||
self.focus_on_video()
|
||||
|
||||
def mediaChangedPosition(self, position):
|
||||
frame = int((position / 1000.0) * self.fps)
|
||||
# print("Video Running %i" % frame)
|
||||
self.updateFrame.emit(frame)
|
||||
self._gv.fitInView(self._videoitem, QtCore.Qt.KeepAspectRatio)
|
||||
|
||||
def handleError(self):
|
||||
# self.playButton.setEnabled(False)
|
||||
print("Error: " + self.mediaPlayer.errorString())
|
||||
|
||||
def createButtons(self):
|
||||
iconSize = QtCore.QSize(28, 28)
|
||||
|
||||
openButton = QtWidgets.QToolButton()
|
||||
openButton.setStyleSheet('border: none;')
|
||||
openButton.setIcon(QtGui.QIcon(self.root + '/icons/open.png'))
|
||||
openButton.setIconSize(iconSize)
|
||||
openButton.setToolTip("Open File")
|
||||
# openButton.clicked.connect(self.open)
|
||||
|
||||
self.playButton = QtWidgets.QToolButton()
|
||||
self.playButton.setStyleSheet('border: none;')
|
||||
self.playButton.setIcon(QtGui.QIcon(self.root + '/icons/play.png'))
|
||||
self.playButton.setIconSize(iconSize)
|
||||
self.playButton.setToolTip("Play movie")
|
||||
self.playButton.clicked.connect(self.play)
|
||||
self.playButton.setEnabled(False)
|
||||
|
||||
self.stopButton = QtWidgets.QToolButton()
|
||||
self.stopButton.setStyleSheet('border: none;')
|
||||
self.stopButton.setIcon(QtGui.QIcon(self.root + '/icons/stop.png'))
|
||||
self.stopButton.setIconSize(iconSize)
|
||||
self.stopButton.setToolTip("Stop movie")
|
||||
self.stopButton.clicked.connect(self.stop)
|
||||
self.stopButton.setEnabled(False)
|
||||
|
||||
@QtCore.pyqtSlot(QtMultimedia.QVideoFrame)
|
||||
def on_videoFrameProbed(self, frame):
|
||||
cloneFrame = QtMultimedia.QVideoFrame(frame)
|
||||
cloneFrame.map(QtMultimedia.QAbstractVideoBuffer.ReadOnly)
|
||||
image = QtGui.QImage(cloneFrame.bits(), cloneFrame.width(), cloneFrame.height(), cloneFrame.bytesPerLine(),
|
||||
QtMultimedia.QVideoFrame.imageFormatFromPixelFormat(cloneFrame.pixelFormat()))
|
||||
self.frameAvailable.emit(image)
|
||||
cloneFrame.unmap()
|
||||
|
||||
def focus_on_video(self):
|
||||
native_video_resolution = self.mediaPlayer.metaData("Resolution")
|
||||
# we also update the sceneview to zoom to the video
|
||||
if native_video_resolution is not None:
|
||||
self._videoitem.setSize(QtCore.QSizeF(native_video_resolution.width(), native_video_resolution.height()))
|
||||
self._gv.fitInView(self._videoitem, QtCore.Qt.KeepAspectRatio)
|
||||
|
||||
# set scale of video to bigger size
|
||||
if self.originalVideoResolution is not None:
|
||||
width_ratio = self.originalVideoResolution[0] / native_video_resolution.width()
|
||||
self._videoitem.setScale(width_ratio)
|
||||
|
||||
@QtCore.pyqtSlot()
|
||||
def clear_tags(self):
|
||||
self.focus_on_video()
|
||||
# clear all tags
|
||||
for tag in self.tags:
|
||||
self._scene.removeItem(self.tag_data[tag])
|
||||
self.tag_data[tag] = self._scene.addPath(QtGui.QPainterPath())
|
||||
|
||||
@QtCore.pyqtSlot(int, list, list)
|
||||
def draw_tags(self, tag, lstX, lstY):
|
||||
# this is removing the old tag data
|
||||
self._scene.removeItem(self.tag_data[tag])
|
||||
|
||||
path = QtGui.QPainterPath()
|
||||
path.setFillRule(Qt.Qt.WindingFill)
|
||||
# set starting points
|
||||
for (x, y) in zip(lstX, lstY):
|
||||
path.addRect(x-50, y-50, 100, 100)
|
||||
|
||||
# by adding it gets converted into an QGraphicsPathItem
|
||||
# save it for later removal
|
||||
self.tag_data[tag] = self._scene.addPath(path)
|
||||
|
||||
# set colors
|
||||
color = self.tag_colors[tag]
|
||||
pen = QtGui.QPen(QtGui.QColor(color[0], color[1], color[2], 255), 2, QtCore.Qt.SolidLine)
|
||||
self.tag_data[tag].setPen(pen)
|
||||
# fill ellipses - alpha value is set to 50%
|
||||
# self.tag_data[tag].setBrush(QtGui.QColor(color[0], color[1], color[2], int(0.5 * 255)))
|
||||
|
||||
@QtCore.pyqtSlot()
|
||||
def clear_labels(self):
|
||||
self.focus_on_video()
|
||||
# clear all labels
|
||||
for id_no in range(self.number_ids):
|
||||
self._scene.removeItem(self.labels[id_no])
|
||||
self.labels[id_no] = self._scene.addPath(QtGui.QPainterPath())
|
||||
|
||||
@QtCore.pyqtSlot(int, int, int)
|
||||
def draw_labels(self, id_no, x, y):
|
||||
# this is removing the old pose data
|
||||
self._scene.removeItem(self.labels[id_no])
|
||||
|
||||
path = QtGui.QPainterPath()
|
||||
# then draw text
|
||||
font = QtGui.QFont("Arial", 70)
|
||||
font.setStyleStrategy(QtGui.QFont.ForceOutline)
|
||||
# sadly there is no easy way to claculate the width of the text so minus 100 is fine, but not ideal
|
||||
# also moving the text up by 500, so that is does not cover the face
|
||||
path.addText(x - 100, y - 300, font, "ID " + str(id_no))
|
||||
|
||||
# by adding it gets converted into an QGraphicsPathItem
|
||||
# save it for later removal
|
||||
self.labels[id_no] = self._scene.addPath(path)
|
||||
|
||||
# set colors
|
||||
color = tuple([int(a * 255) for a in self.colors[id_no]])
|
||||
# alpha value is set to 70%
|
||||
pen = QtGui.QPen(QtGui.QColor(color[0], color[1], color[2], int(0.9 * 255)), 10, QtCore.Qt.SolidLine)
|
||||
self.labels[id_no].setPen(pen)
|
||||
|
||||
@QtCore.pyqtSlot()
|
||||
def clear_pose(self):
|
||||
# empty pose data
|
||||
for id_no in range(self.number_ids):
|
||||
self._scene.removeItem(self.pose_data[id_no])
|
||||
self.pose_data[id_no] = self._scene.addPath(QtGui.QPainterPath())
|
||||
|
||||
@QtCore.pyqtSlot(int, list, list)
|
||||
def draw_pose(self, id_no, lstX, lstY):
|
||||
# this is removing the old pose data
|
||||
self._scene.removeItem(self.pose_data[id_no])
|
||||
|
||||
if len(lstX) > 0 and len(lstY) > 0:
|
||||
path = QtGui.QPainterPath()
|
||||
|
||||
# set starting points
|
||||
path.moveTo(lstX[0], lstY[0])
|
||||
# then draw remaing lines
|
||||
for (x, y) in zip(lstX[1:], lstY[1:]):
|
||||
path.lineTo(x, y)
|
||||
|
||||
# by adding it gets converted into an QGraphicsPathItem
|
||||
# save it for later removal
|
||||
self.pose_data[id_no] = self._scene.addPath(path)
|
||||
|
||||
# set colors
|
||||
color = tuple([int(a * 255) for a in self.colors[id_no]])
|
||||
# alpha value is set to 70%
|
||||
pen = QtGui.QPen(QtGui.QColor(color[0], color[1], color[2], int(0.7 * 255)), 10, QtCore.Qt.SolidLine)
|
||||
self.pose_data[id_no].setPen(pen)
|
||||
else:
|
||||
self.pose_data[id_no] = self._scene.addPath(QtGui.QPainterPath())
|
||||
|
||||
@QtCore.pyqtSlot()
|
||||
def clear_gaze(self):
|
||||
# empty pose data
|
||||
for id_no in range(self.number_ids):
|
||||
self._scene.removeItem(self.gaze_data[id_no])
|
||||
self.gaze_data[id_no] = self._scene.addPath(QtGui.QPainterPath())
|
||||
|
||||
@QtCore.pyqtSlot(int, list, list)
|
||||
def draw_gaze(self, id_no, lstX, lstY):
|
||||
# this is removing the old pose data
|
||||
self._scene.removeItem(self.gaze_data[id_no])
|
||||
|
||||
path = QtGui.QPainterPath()
|
||||
path.setFillRule(Qt.Qt.WindingFill)
|
||||
# set starting points
|
||||
for (x, y) in zip(lstX, lstY):
|
||||
path.addEllipse(x, y, 100, 100)
|
||||
|
||||
# by adding it gets converted into an QGraphicsPathItem
|
||||
# save it for later removal
|
||||
self.gaze_data[id_no] = self._scene.addPath(path)
|
||||
|
||||
# set colors
|
||||
color = tuple([int(a * 255) for a in self.colors[id_no]])
|
||||
# alpha value is set to 50%
|
||||
pen = QtGui.QPen(QtGui.QColor(color[0], color[1], color[2], int(0.5 * 255)), 1, QtCore.Qt.SolidLine)
|
||||
self.gaze_data[id_no].setPen(pen)
|
||||
# fill ellipses
|
||||
self.gaze_data[id_no].setBrush(QtGui.QColor(color[0], color[1], color[2], int(0.5 * 255)))
|
||||
|
||||
@QtCore.pyqtSlot(list)
|
||||
def onSelectedID(self, lst):
|
||||
self.clear_labels()
|
||||
self.clear_gaze()
|
||||
self.clear_pose()
|
||||
|
147
uiwidget/widgetpose.py
Normal file
147
uiwidget/widgetpose.py
Normal file
|
@ -0,0 +1,147 @@
|
|||
import numpy as np
|
||||
import pyqtgraph as pg
|
||||
from PyQt5 import QtWidgets, QtGui, QtCore
|
||||
|
||||
class WidgetPose(QtWidgets.QWidget):
|
||||
|
||||
video_label_signal = QtCore.pyqtSignal(list)
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super(WidgetPose, self).__init__(parent)
|
||||
|
||||
layout = QtWidgets.QGridLayout()
|
||||
self.poseGraph = QtWidgets.QWidget()
|
||||
self.measures = QtWidgets.QWidget()
|
||||
self.measuresLayout = QtWidgets.QHBoxLayout()
|
||||
|
||||
# Setup Movement Graph Plot Widget
|
||||
self.movementGraph = pg.PlotWidget()
|
||||
self.movementGraph.setBackground('w')
|
||||
self.movementGraph.setYRange(0, 400, padding=0)
|
||||
self.movementGraph.getPlotItem().getAxis('bottom').setTickSpacing(minor=50, major=100)
|
||||
self.movementGraph.getPlotItem().setTitle(title='Body Movement over Time')
|
||||
self.movementPlots = []
|
||||
|
||||
layout.addWidget(self.movementGraph, 1, 0, 1, 1)
|
||||
layout.addWidget(self.measures, 1, 1, 1, 1)
|
||||
layout.setColumnStretch(0, 1)
|
||||
layout.setColumnStretch(1, 1)
|
||||
|
||||
self.setLayout(layout)
|
||||
self.colors = None
|
||||
self.labels = dict()
|
||||
|
||||
@QtCore.pyqtSlot(list)
|
||||
def onSelectedID(self, lst):
|
||||
"""Change color of movement graph plot if ID should not be visible"""
|
||||
for i, button in enumerate(lst):
|
||||
if not button.isChecked():
|
||||
self.movementPlots[i].setPen(None)
|
||||
elif self.colors is not None:
|
||||
color = tuple([int(a * 255) for a in self.colors[i]])
|
||||
pen = pg.mkPen(color=color)
|
||||
self.movementPlots[i].setPen(pen)
|
||||
|
||||
@QtCore.pyqtSlot(dict, list, int)
|
||||
def updateMovementGraph(self, data, colors, numberIDs):
|
||||
"""Plot ID specific movement data from processing class
|
||||
data[id: (movements, frames)]
|
||||
"""
|
||||
# handle NaN https://github.com/pyqtgraph/pyqtgraph/issues/1057
|
||||
# downgrade to 5.13 fixes the issue
|
||||
for id_no in range(numberIDs):
|
||||
if data.get(id_no):
|
||||
if not np.all(np.isnan(data.get(id_no)[0])):
|
||||
self.movementPlots[id_no].setData(data.get(id_no)[1], data.get(id_no)[0]) # Update the data.
|
||||
|
||||
self.movementGraph.setXRange(np.min(data.get(id_no)[1]), np.max(data.get(id_no)[1]))
|
||||
|
||||
@QtCore.pyqtSlot(dict, int)
|
||||
def updateHandVelocity(self, data, numberIDs):
|
||||
"""Update Velocity Label
|
||||
data[id: velocity for frame]"""
|
||||
for id_no in range(numberIDs):
|
||||
if data.get(id_no) is not None:
|
||||
self.labels['Velocity'][id_no].setText('%.2f' % data[id_no])
|
||||
else:
|
||||
self.labels['Velocity'][id_no].setText(' ')
|
||||
|
||||
@QtCore.pyqtSlot(list, int)
|
||||
def initMovementGraph(self, colors, numberIDs):
|
||||
"""Initialize plot lines with 0
|
||||
colors: plot color for each ID
|
||||
"""
|
||||
for i in range(numberIDs):
|
||||
x = list(range(-200, 0)) # 100 time points
|
||||
y = [0 for _ in range(200)] # 100 data points
|
||||
color = tuple([int(a * 255) for a in colors[i]])
|
||||
pen = pg.mkPen(color=color)
|
||||
dataLine = self.movementGraph.plot(x, y, pen=pen)
|
||||
self.movementPlots.append(dataLine)
|
||||
|
||||
@QtCore.pyqtSlot(dict, dict, list, int)
|
||||
def setInit(self, mostActivity, hand, colors, numberIDs):
|
||||
self.colors = colors
|
||||
idLayout |