Added GUI

README.md

@@ -1,3 +1,31 @@
-# conan
+# ConAn
+This is the official repository for [ConAn: A Usable Tool for Multimodal <u>Con</u>versation <u>An</u>alysis](https://www.perceptualui.org/publications/penzkofer21_icmi.pdf) <br>
+ConAn – our graphical tool for multimodal conversation analysis – takes 360 degree videos recorded during multiperson group interactions as input. ConAn integrates state-of-the-art models for gaze estimation, active speaker detection,
+facial action unit detection, and body movement detection and can output quantitative reports both at individual and group
+level, as well as different visualizations that provide qualitative insights into group interaction.
 
-ConAn: A Usable Tool for Multimodal <u>Con</u>versation <u>An</u>alysis 
+## Installation 
+For the graphical user interface (GUI) you need python>3.6 to install the [requirements](requirements.txt) via pip: 
+```
+pip install requirements.txt 
+```
+## Get Started 
+To test the GUI you can download our example use case videos from googledrive: <br>
+As well as the respective processed ``.dat`` files which include all the analyses. 
+Run [main.py](main.py) and import the video file you would like to analyze. 
+## Processing
+
+
+
+## Citation
+Please cite this paper if you use ConAn or parts of this publication in your research: 
+```
+@inproceedings{penzkofer21_icmi,
+  author = {Penzkofer, Anna and Müller, Philipp and Bühler, Felix and Mayer, Sven and Bulling, Andreas},
+  title = {ConAn: A Usable Tool for Multimodal Conversation Analysis},
+  booktitle = {Proc. ACM International Conference on Multimodal Interaction (ICMI)},
+  year = {2021},
+  doi = {10.1145/3462244.3479886},
+  video = {https://www.youtube.com/watch?v=H2KfZNgx6CQ}
+}
+```

container.py

@@ -0,0 +1,151 @@
+import numpy as np
+import pandas as pd
+import pickle as pkl
+import os.path
+import cv2
+from PyQt5 import QtCore, QtGui
+
+
+class DataContainer():
+
+    def __init__(self, movie_fileName, data_fileName):
+        self.movie_fileName = movie_fileName
+        self.data_fileName = data_fileName
+
+        self.frameCount = 0
+        self.fps = 0
+        self.frameSize = [0, 0]
+
+        self.dataGaze = None
+        self.dataMovement = None
+        self.number_ids = None
+        self.dataRTGene = None
+        self.image = None
+
+    def readData(self, verbose=False, rtgene=False):
+        if (verbose):
+            print("## Start Reading Data")
+
+        # Read Video Data
+        f = self.movie_fileName
+        print(f)
+        if os.path.isfile(f):
+            cap = cv2.VideoCapture(f)
+            ret, frame = cap.read()
+            self.image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+            h, w, ch = self.image.shape
+            bytesPerLine = ch * w
+            convertToQtFormat = QtGui.QImage(self.image.data, w, h, bytesPerLine, QtGui.QImage.Format_RGB888)
+
+            self.fps = cap.get(cv2.CAP_PROP_FPS)
+            self.frameCount = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+            width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)  # float
+            height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)  # float
+            self.frameSize = [width, height]
+            print(self.fps, self.frameCount)
+
+            if (verbose):
+                print("Video frameCount %i" % self.frameCount)
+                duration = self.frameCount / self.fps
+                minutes = int(duration / 60)
+                seconds = duration % 60
+                print('Video duration (M:S) = ' + str(minutes) + ':' + str(seconds))
+        else:
+            print("WARNING: no video avaibale.")
+
+        # read data file
+        with open(self.data_fileName, 'rb') as f:
+            data = pkl.load(f)
+
+        # Read RT-Gene Gaze
+        if rtgene:
+            f = "%s_Gaze.pkl" % self.dataPath[0]
+            df_gt = pd.read_pickle('exampledata/G2_VID1_GroundTruth.pkl')
+            df_gt = df_gt[['Frame', 'ID0_target_spher', 'ID1_target_spher']]
+            if os.path.isfile(f):
+                df = pd.read_pickle(f)
+                self.number_ids = len(df.PId.unique())
+
+                self.dataRTGene = df.pivot(index='Frame', columns="PId", values=["GazeTheta", "GazePhi", "HeadCenter",
+                                                                                 "HeadPoseYaw", "HeadPoseTheta",
+                                                                                 "Phi", "Theta"])
+                lst = []
+                for label in ["GazeTheta", "GazePhi", "Head", "HeadPoseYaw", "HeadPoseTheta", "Phi", "Theta"]:
+                    for head_id in range(self.number_ids):
+                        lst.append("ID%i_%s" % (head_id, label))
+                self.dataRTGene.columns = lst
+                self.dataRTGene = self.dataRTGene.reset_index()
+                self.dataRTGene = pd.merge(self.dataRTGene, df_gt, on=['Frame'])
+                self.dataRTGene = self.dataRTGene.rename(
+                    columns={'ID0_target_spher': 'ID1_target', 'ID1_target_spher': 'ID0_target'},
+                    errors='raise')
+
+                print('Detected %i IDs in video' % self.number_ids)
+                if (verbose):
+                    print("Gaze sample count %i" % len(self.dataRTGene))
+            else:
+                print("WARNING: no RT-Gene data avaibale.")
+
+        # Read Gaze Data
+        if "HeadPose" in data:
+            self.dataGaze = data["HeadPose"]
+            self.number_ids = len([col for col in self.dataGaze.columns if 'head' in col])
+
+            print('Detected %i IDs in video' % self.number_ids)
+            if (verbose):
+                print("Gaze sample count %i" % len(self.dataGaze))
+        else:
+            print("WARNING: no gaze data avaibale.")
+
+        """
+        # Read OpenPose Data
+        f = self.__get_filename_with_substring("OpenPose")
+        if os.path.isfile(f):
+            self.dataPose = pd.read_pickle(f)
+            if (verbose):
+                print("Pose sample count %i" % len(self.dataPose))
+        else:
+            print("WARNING: no pose data avaibale.")
+        """
+
+        # Read Movement Data
+        if "BodyMovement" in data:
+            self.dataMovement = data["BodyMovement"]
+            if verbose:
+                print('Body movement sample count %i' % len(self.dataMovement))
+        else:
+            print('WARNING: no body movement data available.')
+
+    def getFrameCount(self):
+        return self.frameCount
+
+    def getFrameSize(self):
+        return self.frameSize
+
+    def getFPS(self):
+        return self.fps
+
+    def getVideo(self):
+        return self.movie_fileName
+
+    def getGazeData(self):
+        return self.dataGaze
+
+    def getFrame(self, frameIdx):
+        return frameIdx
+
+    def getFrameCurrent(self):
+        return 1
+
+    def getNumberIDs(self):
+        return self.number_ids
+
+    def getMovementData(self):
+        return self.dataMovement
+
+    def getRTGeneData(self):
+        return self.dataRTGene
+
+    def getImage(self):
+        return self.image

gui.ui

@@ -0,0 +1,183 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>ConAn</class>
+ <widget class="QMainWindow" name="ConAn">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>800</width>
+    <height>450</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>MainWindow</string>
+  </property>
+  <widget class="QWidget" name="centralwidget">
+   <layout class="QGridLayout" name="gridLayout">
+    <item row="4" column="0" colspan="2">
+     <widget class="QTabWidget" name="tab">
+      <property name="sizePolicy">
+       <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
+        <horstretch>4</horstretch>
+        <verstretch>1</verstretch>
+       </sizepolicy>
+      </property>
+      <property name="minimumSize">
+       <size>
+        <width>0</width>
+        <height>300</height>
+       </size>
+      </property>
+      <property name="currentIndex">
+       <number>4</number>
+      </property>
+      <widget class="WidgetGaze" name="gaze">
+       <attribute name="title">
+        <string>Eye Gaze</string>
+       </attribute>
+      </widget>
+      <widget class="WidgetSpeaking" name="speak">
+       <attribute name="title">
+        <string>Speaking Activity</string>
+       </attribute>
+      </widget>
+      <widget class="WidgetPose" name="pose">
+       <property name="sizePolicy">
+        <sizepolicy hsizetype="Preferred" vsizetype="Ignored">
+         <horstretch>0</horstretch>
+         <verstretch>0</verstretch>
+        </sizepolicy>
+       </property>
+       <attribute name="title">
+        <string>Body Pose</string>
+       </attribute>
+      </widget>
+      <widget class="WidgetFacialExpression" name="face">
+       <attribute name="title">
+        <string>Facial Expression</string>
+       </attribute>
+      </widget>
+      <widget class="WidgetObject" name="object">
+       <attribute name="title">
+        <string>Object Detection</string>
+       </attribute>
+      </widget>
+     </widget>
+    </item>
+    <item row="2" column="0" colspan="2">
+     <widget class="WidgetTimeLine" name="time" native="true">
+      <property name="sizePolicy">
+       <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
+        <horstretch>4</horstretch>
+        <verstretch>1</verstretch>
+       </sizepolicy>
+      </property>
+      <property name="minimumSize">
+       <size>
+        <width>0</width>
+        <height>60</height>
+       </size>
+      </property>
+     </widget>
+    </item>
+    <item row="0" column="0">
+     <widget class="WidgetPlayer" name="video" native="true">
+      <property name="sizePolicy">
+       <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
+        <horstretch>3</horstretch>
+        <verstretch>1</verstretch>
+       </sizepolicy>
+      </property>
+     </widget>
+    </item>
+    <item row="0" column="1">
+     <widget class="WidgetVideoSettings" name="vsettings" native="true">
+      <property name="sizePolicy">
+       <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
+        <horstretch>1</horstretch>
+        <verstretch>1</verstretch>
+       </sizepolicy>
+      </property>
+     </widget>
+    </item>
+   </layout>
+  </widget>
+  <widget class="QMenuBar" name="menubar">
+   <property name="geometry">
+    <rect>
+     <x>0</x>
+     <y>0</y>
+     <width>800</width>
+     <height>21</height>
+    </rect>
+   </property>
+  </widget>
+  <widget class="QStatusBar" name="statusbar"/>
+  <action name="actionOpen">
+   <property name="text">
+    <string>Open</string>
+   </property>
+  </action>
+  <action name="actionExit">
+   <property name="text">
+    <string>Exit</string>
+   </property>
+  </action>
+ </widget>
+ <customwidgets>
+  <customwidget>
+   <class>WidgetTimeLine</class>
+   <extends>QWidget</extends>
+   <header>uiwidget/widgettimeline</header>
+   <container>1</container>
+  </customwidget>
+  <customwidget>
+   <class>WidgetPlayer</class>
+   <extends>QWidget</extends>
+   <header>uiwidget/widgetplayer</header>
+   <container>1</container>
+  </customwidget>
+  <customwidget>
+   <class>WidgetPose</class>
+   <extends>QWidget</extends>
+   <header>uiwidget/widgetpose</header>
+   <container>1</container>
+  </customwidget>
+  <customwidget>
+   <class>WidgetSpeaking</class>
+   <extends>QWidget</extends>
+   <header>uiwidget/widgetspeaking</header>
+   <container>1</container>
+  </customwidget>
+  <customwidget>
+   <class>WidgetGaze</class>
+   <extends>QWidget</extends>
+   <header>uiwidget/widgetgaze</header>
+   <container>1</container>
+  </customwidget>
+  <customwidget>
+   <class>WidgetFacialExpression</class>
+   <extends>QWidget</extends>
+   <header>uiwidget/widgetfacialexpression</header>
+   <container>1</container>
+  </customwidget>
+  <customwidget>
+   <class>WidgetVideoSettings</class>
+   <extends>QWidget</extends>
+   <header>uiwidget/widgetvideosettings</header>
+   <container>1</container>
+  </customwidget>
+  <customwidget>
+   <class>WidgetObject</class>
+   <extends>QWidget</extends>
+   <header location="global">uiwidget/widgetobject</header>
+   <container>1</container>
+  </customwidget>
+ </customwidgets>
+ <tabstops>
+  <tabstop>tab</tabstop>
+ </tabstops>
+ <resources/>
+ <connections/>
+</ui>

main.py

@@ -0,0 +1,287 @@
+#!/usr/bin/env python3
+import sys
+import json
+import numpy as np
+import pandas as pd
+import os
+
+import PyQt5
+from PyQt5 import QtGui
+from PyQt5 import QtCore
+from PyQt5.QtCore import QSize, QUrl, pyqtSignal, QFile, QTextStream
+from PyQt5.QtGui import QIcon
+from PyQt5.QtMultimedia import QMediaContent
+from PyQt5.QtWidgets import QAction, QFileDialog, QStatusBar
+from PyQt5.QtGui import QPalette, QColor
+
+print("Qt version {0}".format(QtCore.QT_VERSION_STR))
+from PyQt5 import QtWidgets
+import pyqtgraph as pg
+
+import cv2
+print("OpenCV version {0} ".format(cv2.__version__))
+
+
+from uiwidget import widgetplayer, widgettimeline, widgetgaze, widgetspeaking, widgetpose, widgetfacialexpression, widgetobject, widgetvideosettings
+from processing import Processor
+
+
+class MainWindow(QtWidgets.QMainWindow):
+    sendPath = pyqtSignal(str)
+    sendSegments = pyqtSignal(np.ndarray)
+
+    def __init__(self, verbose=False, parent=None):
+        super(MainWindow, self).__init__(parent)
+        self.setWindowTitle('Conversation Analysis')
+
+        PyQt5.uic.loadUi('gui.ui', self)  # Load the .ui file
+        self.show()  # Show the GUI
+
+        self.resize(1600, 900)
+        self.dataContainer = None
+        self.verbose = verbose
+        self.movieDir = ''
+        self.segments = []
+        self.frames_bitmask = None
+        self.frame = None
+        self.frameCount = None
+        self.fps = None
+
+        self._setupMenuBar()
+        self.status = QStatusBar()
+        self.setStatusBar(self.status)
+        self.status.showMessage('Welcome', 1000)
+
+        self.wPlayer = self.findChild(widgetplayer.WidgetPlayer, "video")
+        self.wTimeLine = self.findChild(widgettimeline.WidgetTimeLine, "time")
+        self.wTab = self.findChild(QtWidgets.QTabWidget, "tab")
+        self.wGaze = self.findChild(widgetgaze.WidgetGaze, "gaze")
+        self.wSpeaking = self.findChild(widgetspeaking.WidgetSpeaking, "speak")
+        self.wPose = self.findChild(widgetpose.WidgetPose, "pose")
+        self.wFace = self.findChild(widgetfacialexpression.WidgetFacialExpression, "face")
+        self.wObject = self.findChild(widgetobject.WidgetObject, "object")
+        self.wVideoSettings = self.findChild(widgetvideosettings.WidgetVideoSettings, "vsettings")
+
+        self.wPlayer.mediaPlayer.positionChanged.connect(self.wTimeLine.updateSlider)
+        self.wPlayer.sendState.connect(self.wTimeLine.mediaStateChanged)
+
+        self.wTimeLine.signalSetPosition.connect(self.wPlayer.mediaPlayer.setPosition)
+        self.wTimeLine.signalPlay.connect(self.wPlayer.play)
+
+
+        self.processor = Processor()
+        self.processor.signalInit.connect(self.wPose.initMovementGraph)
+        self.processor.signalInit.connect(self.wSpeaking.initSpeakingGraph)
+        self.processor.signalInit.connect(self.wGaze.initGazeGraph)
+        self.processor.signalInitTags.connect(self.wObject.setInit)
+
+        self.processor.signalInit.connect(self.wFace.setInit)
+        self.processor.signalInit.connect(self.wVideoSettings.setInit)
+
+        self.wPlayer.frameAvailable.connect(self.processor.saveCurrentFrameData)
+
+        self.processor.signalPoseSetInit.connect(self.wPose.setInit)
+        self.processor.signalGazeSetInit.connect(self.wGaze.setInit)
+        self.processor.signalSpeakerSetInit.connect(self.wSpeaking.setInit)
+        self.processor.signalUpdateHandVelocity.connect(self.wPose.updateHandVelocity)
+        self.processor.signalUpdateMovementGraph.connect(self.wPose.updateMovementGraph)
+        self.processor.signalUpdateSpeakGraph.connect(self.wSpeaking.updateSpeakingGraph)
+        self.processor.signalUpdateFaceAus.connect(self.wFace.updateFrame)
+        self.processor.signalUpdateFaceImgs.connect(self.wFace.updateImages)
+        self.processor.signalUpdateTagGraph.connect(self.wObject.updateTagGraph)
+        self.processor.signalVideoLabel.connect(self.wPlayer.draw_labels)
+        self.processor.signalPosePoints.connect(self.wPlayer.draw_pose)
+        self.processor.signalPoseChangedLabels.connect(self.wPose.updateLables)
+        self.processor.signalSpeakChangedLabels.connect(self.wSpeaking.updateLables)
+        self.processor.signalUpdateGazeGraph.connect(self.wGaze.updateGazeGraph)
+        self.processor.signalUpdateGazeMap.connect(self.wPlayer.draw_gaze)
+        self.processor.signalUpdateTags.connect(self.wPlayer.draw_tags)
+
+        self.wVideoSettings.signalVisualize.connect(self.processor.onVisualize)
+        self.wVideoSettings.signalSelectID.connect(self.wPose.onSelectedID)
+        self.wVideoSettings.signalSelectID.connect(self.wGaze.onSelectedID)
+        self.wVideoSettings.signalSelectID.connect(self.wPlayer.onSelectedID)
+        self.wVideoSettings.signalSelectID.connect(self.processor.onSelectedID)
+
+        self.processor.signalClearLabels.connect(self.wPlayer.clear_labels)
+        self.processor.signalClearPose.connect(self.wPlayer.clear_pose)
+        self.processor.signalClearGaze.connect(self.wPlayer.clear_gaze)
+        self.processor.signalClearTags.connect(self.wPlayer.clear_tags)
+
+        self.processor.signalDeactivatePoseTab.connect(self.togglePoseTab)
+        self.processor.signalDeactivateFaceTab.connect(self.toggleFaceTab)
+        self.processor.signalDeactivateGazeTab.connect(self.toggleGazeTab)
+        self.processor.signalDeactivateSpeakingTab.connect(self.toggleSpeakingTab)
+        self.processor.signalDeactivateObjectTab.connect(self.toggleObjectTab)
+
+        self.wTab.setCurrentIndex(0)
+        self.wTab.currentChanged.connect(self.processor.tabChanged)
+
+    def openProject(self, movie_fileName, data_fileName):
+        self.status.showMessage('Reading data...')
+        self.processor.readData(movie_fileName, data_fileName)
+        self.processor.calculateMovementMeasures()
+        self.processor.calculateSpeakingMeasures()
+        self.wPlayer.mediaPlayer.positionChanged.connect(self.processor.updateFrame)
+        self.wPlayer.mediaPlayer.positionChanged.connect(self.updateFrame)
+        self.fps = self.processor.getFPS()
+
+        # Variables for segment updates --> bool array for all frames
+        self.frame = 0
+        self.frameCount = self.processor.getFrameCount()
+        self.frames_bitmask = np.ones(self.frameCount)
+        self.segments = [(0, self.frameCount)]
+
+        self.wTimeLine.setInit(self.processor.getFrameCount(), self.processor.getNumberIDs(), self.processor.getFPS())
+        self.wPlayer.setInit(self.processor.getVideo(), self.processor.getFPS(),
+                             self.processor.getOriginalVideoResolution(), self.processor.getNumberIDs(),
+                             self.processor.getColors(), self.processor.getTags(), self.processor.getTagColors())
+
+        self.wTimeLine.rangeslider.segmentsChanged.connect(self.segmentsChanged)
+        self.sendSegments.connect(self.processor._updateSegments)
+        self.processor.setReady(True)
+        self.status.showMessage('Ready')
+
+    def segmentsChanged(self, segments, last_segment_changed):
+        self.segments = segments
+        self.frames_bitmask = np.zeros(self.frameCount)
+        for segment in segments:
+            start, end = segment
+            self.frames_bitmask[start:end] = np.ones((end - start))
+        self.sendSegments.emit(self.frames_bitmask)
+
+        start, end = segments[last_segment_changed]
+
+        if start > self.frame:
+            self.wPlayer.setFrame(start)
+
+        if end < self.frame:
+            self.wPlayer.pause()
+            self.wPlayer.setFrame(end)
+
+    """
+    def closeEvent(self, event):
+        self.wPlayer.stop()
+
+        #quit_msg = "Are you sure you want to exit the program?"
+        # reply = QtGui.QMessageBox.question(self, 'Message',
+        #                 quit_msg, QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
+        #
+        # if reply == QtGui.QMessageBox.Yes:
+        #    event.accept()
+        # else:
+        #    event.ignore()
+    """
+
+    def open(self):
+        movie_fileName, _ = QFileDialog.getOpenFileName(self, "Open Movie File", "", "Files (*.mp4)", self.movieDir)
+        data_fileName, _ = QFileDialog.getOpenFileName(
+            self, "Open Preprocessed File", "", "Files (*.dat)", self.movieDir)
+
+        if movie_fileName and data_fileName and os.path.isfile(movie_fileName) and os.path.isfile(data_fileName):
+            self.status.showMessage('Loading data... ')
+            self.openProject(movie_fileName, data_fileName)
+        else:
+            print("ERROR: select two files")
+
+    def export(self):
+        self.status.showMessage('Exporting data... This might take some time.')
+        # has to be defined in the main window, and then call the processing function
+        self.processor.export()
+        self.status.showMessage('Exported data successfully!')
+
+    def _setupMenuBar(self):
+        self.mainMenu = self.menuBar()
+
+        fileMenu = self.mainMenu.addMenu('&File')
+
+        openAct = QAction('&Open', self)
+        openAct.setStatusTip('Open files')
+        openAct.setShortcut('Ctrl+O')
+        openAct.triggered.connect(self.open)
+        fileMenu.addAction(openAct)
+
+        exportAct = QAction('&Export', self)
+        exportAct.setStatusTip('Export calculations')
+        exportAct.setShortcut('Ctrl+E')
+        exportAct.triggered.connect(self.export)
+        fileMenu.addAction(exportAct)
+
+        exitAct = QAction('&Exit', self)
+        exitAct.setStatusTip('Exit application')
+        exitAct.setShortcut('Ctrl+Q')
+        exitAct.triggered.connect(self.close)
+        fileMenu.addAction(exitAct)
+
+    #@QtCore.pyqtSlot(str)
+    # def statusUpdate(self, message):
+        # self.status.showMessage(message)
+
+    def updateFrame(self, position):
+        self.frame = int((position / 1000.0) * self.fps)
+        if self.frames_bitmask[self.frame] == 0:
+            for segment in self.segments:
+                if segment[0] > self.frame:
+                    self.wPlayer.setFrame(segment[0] + 1)
+                    break
+
+    @QtCore.pyqtSlot(bool)
+    def togglePoseTab(self, deactivate):
+        self.wTab.setTabEnabled(2, not deactivate)
+
+    @QtCore.pyqtSlot(bool)
+    def toggleGazeTab(self, deactivate):
+        self.wTab.setTabEnabled(0, not deactivate)
+
+    @QtCore.pyqtSlot(bool)
+    def toggleFaceTab(self, deactivate):
+        self.wTab.setTabEnabled(3, not deactivate)
+
+    @QtCore.pyqtSlot(bool)
+    def toggleSpeakingTab(self, deactivate):
+        self.wTab.setTabEnabled(1, not deactivate)
+
+    @QtCore.pyqtSlot(bool)
+    def toggleObjectTab(self, deactivate):
+        self.wTab.setTabEnabled(4, not deactivate)
+
+
+if __name__ == "__main__":
+
+    app = QtWidgets.QApplication(sys.argv)
+    # print(sys.argv)
+    app.setWindowIcon(QtGui.QIcon('icons/logo.png'))
+
+    app.setStyle("Fusion")
+
+    # Now use a palette to switch to dark colors:
+    palette = QPalette()
+    palette.setColor(QPalette.Window, QColor(53, 53, 53))
+    palette.setColor(QPalette.WindowText, QtCore.Qt.white)
+    palette.setColor(QPalette.Base, QColor(25, 25, 25))
+    palette.setColor(QPalette.AlternateBase, QColor(53, 53, 53))
+    palette.setColor(QPalette.ToolTipBase, QtCore.Qt.black)
+    palette.setColor(QPalette.ToolTipText, QtCore.Qt.white)
+    palette.setColor(QPalette.Text, QtCore.Qt.white)
+    palette.setColor(QPalette.Button, QColor(53, 53, 53))
+    palette.setColor(QPalette.ButtonText, QtCore.Qt.white)
+    palette.setColor(QPalette.BrightText, QtCore.Qt.red)
+    palette.setColor(QPalette.Link, QColor(42, 130, 218))
+    palette.setColor(QPalette.Highlight, QColor(42, 130, 218))
+    palette.setColor(QPalette.HighlightedText, QtCore.Qt.black)
+
+    app.setPalette(palette)
+
+
+
+    if ("--verbose" in sys.argv):
+        verbose = True
+        print("### Verbose ENABLED")
+    else:
+        verbose = False
+
+    main = MainWindow(verbose=verbose)
+    main.setWindowTitle('ConAn: A Usable Tool for Multimodal Conversation Analysis')
+    main.show()
+
+    sys.exit(app.exec_())

requirements.txt

@@ -0,0 +1,5 @@
+PyQt5==5.14.2
+opencv-python==4.2.0.34
+pandas
+pyqtgraph
+matplotlib

uiwidget/widgetfacialexpression.py

@@ -0,0 +1,89 @@
+import numpy as np
+
+import pyqtgraph as pg
+from PyQt5 import QtWidgets, QtCore, QtGui
+from PyQt5.QtGui import QColor
+
+lst_aus = ['AU1: Inner Brow Raiser', 'AU2: Outer Brow Raiser', 'AU4: Brow Lowerer', 'AU5: Upper Lid Raiser',
+              'AU6: Cheek Raiser', 'AU9: Nose Wrinkler', 'AU12: Lip Corner Puller', 'AU15: Lip Corner Depressor',
+              'AU17: Chin Raiser',  'AU20: Lip Stretcher', 'AU25: Lips Part',  'AU26: Jaw Drop']
+
+class WidgetFacialExpression(QtWidgets.QWidget):
+    def __init__(self, parent=None):
+        super(WidgetFacialExpression, self).__init__(parent)
+
+        self.faceLayout = QtWidgets.QHBoxLayout()
+        self.setLayout(self.faceLayout)
+
+        self.numberIDs = None
+        self.valueLabels = dict()
+        self.imgPlots = dict()
+
+    @QtCore.pyqtSlot(list, int)
+    def setInit(self, colors, numberIDs):
+        self.numberIDs = numberIDs
+
+        for id_no in range(numberIDs):
+            idLayout = QtWidgets.QHBoxLayout()
+            labelNameLayout = QtWidgets.QVBoxLayout()
+            labelValueLayout = QtWidgets.QVBoxLayout()
+            imageLayout = QtWidgets.QVBoxLayout()
+            imageWidget = pg.PlotWidget(background=QColor(53, 53, 53))
+            imageWidget.invertY()
+            imageWidget.hideAxis('bottom'), imageWidget.hideAxis('left')
+            imageWidget.setMaximumHeight(150), imageWidget.setMaximumWidth(150)
+            imageWidget.setAspectLocked(True)
+            self.imgPlots[id_no] = imageWidget
+
+            color = tuple([int(a * 255) for a in colors[id_no]])
+
+            labelID = QtWidgets.QLabel('ID%i' % id_no)
+            labelID.setStyleSheet('font: bold 12px; color: black; background-color: rgb(%i,%i,%i)' % color)
+
+
+
+            #labelID.setFixedWidth(60)
+
+            labelNameLayout.addWidget(labelID)
+            labelID = QtWidgets.QLabel(' ')
+
+            #labelID.setStyleSheet('background-color: rgb(%i,%i,%i)' % color)
+            labelValueLayout.addWidget(labelID)
+            lst = []
+            for au in lst_aus:
+                nLabel = QtWidgets.QLabel(au)
+                labelNameLayout.addWidget(nLabel)
+
+                vLabel = QtWidgets.QLabel(' ')
+                labelValueLayout.addWidget(vLabel)
+                lst.append(vLabel)
+
+            self.valueLabels[id_no] = lst
+            idLayout.addWidget(imageWidget)
+            idLayout.addLayout(labelNameLayout)
+            idLayout.addLayout(labelValueLayout)
+            self.faceLayout.addLayout(idLayout)
+
+    @QtCore.pyqtSlot(dict, int)
+    def updateImages(self, imgs, id_no):
+        if imgs[id_no] is not None:
+            img = np.moveaxis(imgs[id_no], 0, 1)
+            img = pg.ImageItem(img)
+            self.imgPlots[id_no].addItem(img)
+
+
+    @QtCore.pyqtSlot(dict)
+    def updateFrame(self, aus):
+        if self.numberIDs is None:
+            return
+
+        for id_no in range(self.numberIDs):
+            if len(aus[id_no]) > 0:
+                for i, label in enumerate(self.valueLabels[id_no]):
+                    if not np.any(np.isnan(np.array(aus[id_no].flatten()[0], dtype=np.float64))):
+                        label.setText('%.2f' % aus[id_no].flatten()[0][i])
+
+
+
+
+

uiwidget/widgetgaze.py

@@ -0,0 +1,147 @@
+import numpy as np
+import pyqtgraph
+import matplotlib
+
+matplotlib.use("Qt5Agg")
+import matplotlib.animation as animation
+from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
+from matplotlib.figure import Figure
+from PyQt5 import QtWidgets, QtGui
+from PyQt5 import QtCore
+import pyqtgraph as pg
+import pyqtgraph.exporters
+
+from utils.colors import random_colors
+from utils.util import sperical2equirec
+
+
+class WidgetGaze(QtWidgets.QWidget):
+
+    def __init__(self, parent=None):
+        super(WidgetGaze, self).__init__(parent)
+
+        layout = QtWidgets.QGridLayout()
+
+        # Setup gaze graph
+        self.gazeGraph = pg.PlotWidget()
+        self.gazeGraph.setBackground('w')
+        self.gazeGraph.setYRange(-1.25, 1.25, padding=0)
+        self.gazeGraph.setXRange(-1.25, 1.25, padding=0)
+        self.gazeGraph.hideAxis('left')
+        self.gazeGraph.hideAxis('bottom')
+        self.gazeGraph.setAspectLocked()
+        self.gazeGraph.getPlotItem().setTitle(title='Top-down View of Gaze')
+        self.gazeGraphPlots = []
+
+        self.measures = QtWidgets.QWidget()
+        self.measuresLayout = QtWidgets.QHBoxLayout()
+
+        # self.gazeMap = QtWidgets.QWidget()
+        # self.heatmapSlider = HeatmapSlider()
+        # self.heatmapSlider.signalSetThreshold.connect(self.setThreshold)
+        # self.heatmapSlider.signalSaveImage.connect(self.gazeMap.saveImage)
+
+        # row, column, row span, column span
+        layout.addWidget(self.measures, 0, 1, 2, 1)
+        layout.addWidget(self.gazeGraph, 0, 0, 2, 1)
+        layout.setColumnStretch(0, 1)
+        layout.setColumnStretch(1, 1)
+
+        self.setLayout(layout)
+        # layout.addWidget(self.gazeMap, 0, 0, 3, 1)
+        # layout.addWidget(self.heatmapSlider, 3, 0, 1, 1)
+        self.gazeLabels = []
+        self.colors = None
+
+    @QtCore.pyqtSlot(dict, list, int)
+    def setInit(self, measures, colors, numberIDs):
+        """Initialize measure widget with labels for all IDs"""
+        self.colors = colors  # Necessary for ID updates
+        idLayout = QtWidgets.QVBoxLayout()
+        labelID = QtWidgets.QLabel('      ')
+        labelID.setFixedWidth(60)
+        labelID.setFixedHeight(20)
+        labelA = QtWidgets.QLabel('LookSomeone: ')
+        labelNoLook = QtWidgets.QLabel('TotalNoLook: ')
+        labelG = QtWidgets.QLabel('TotalWatched: ')
+        labelRatio = QtWidgets.QLabel('RatioWatcherLookSOne: ')
+        label = QtWidgets.QLabel('Tracked: ')
+        # labelVel = QtWidgets.QLabel('totNoLook: ')
+        idLayout.addWidget(labelID)
+        idLayout.addWidget(labelA)
+        idLayout.addWidget(labelNoLook)
+        idLayout.addWidget(labelG)
+        idLayout.addWidget(labelRatio)
+        idLayout.addWidget(label)
+        # idLayout.addWidget(labelVel)
+        self.measuresLayout.insertLayout(-1, idLayout)
+
+        for id_no in range(numberIDs):
+            idLayout = QtWidgets.QVBoxLayout()
+
+            color = tuple([int(a * 255) for a in colors[id_no]])
+
+            labelID = QtWidgets.QLabel('ID%i' % id_no)
+            labelID.setStyleSheet('font: bold 12px; color: black; background-color: rgb(%i,%i,%i)' % color)
+            labelID.setFixedWidth(60)
+            labelID.setFixedHeight(20)
+            # Look Someone
+            labelA = QtWidgets.QLabel('{:.2%}'.format(measures[id_no][1] / measures[id_no][2]))
+            labelNoLook = QtWidgets.QLabel('{:.2%}'.format((measures[id_no][2] - measures[id_no][1]) / measures[id_no][2]))
+            # Total Watched
+            labelG = QtWidgets.QLabel('{:.2%}'.format(measures[id_no][0] / measures[id_no][2]))
+            # ratio totWatcher / lookSomeone
+            labelRatio = QtWidgets.QLabel('{:.2}'.format(measures[id_no][0] / measures[id_no][1]))
+            label = QtWidgets.QLabel('%i frames' % measures[id_no][2])
+            # labelVel = QtWidgets.QLabel('%.2f' % np.random.uniform(0, 1))
+            idLayout.addWidget(labelID)
+            idLayout.addWidget(labelA)
+            idLayout.addWidget(labelNoLook)
+            idLayout.addWidget(labelG)
+            idLayout.addWidget(labelRatio)
+            idLayout.addWidget(label)
+            # idLayout.addWidget(labelVel)
+            # self.gazeLabels.append(labelVel)
+            self.measuresLayout.insertLayout(-1, idLayout)
+        self.measures.setLayout(self.measuresLayout)
+
+    @QtCore.pyqtSlot(list, int)
+    def initGazeGraph(self, colors, numberIDs):
+        """ initialize gaze graph """
+        # Big circle
+        x1, y1 = self.get_circle(radius=1)
+        self.gazeGraph.addItem(self.gazeGraph.plot(x1, y1, pen=pg.mkPen(0.5)))
+
+        # Camera
+        x2, y2 = self.get_circle(radius=0.02)
+        self.gazeGraph.addItem(self.gazeGraph.plot(x2, y2, pen=pg.mkPen(color=(0, 0, 0), width=3)))
+
+        for id_no in range(numberIDs):
+            color = tuple([int(a * 255) for a in colors[id_no]])
+            plt = self.gazeGraph.plot(x=[], y=[], pen=pg.mkPen(color=color, width=2))
+            self.gazeGraphPlots.append(plt)
+
+    @QtCore.pyqtSlot(dict, int)
+    def updateGazeGraph(self, data, numberIDs):
+        """ frame updates for gaze graph """
+        for id_no in range(numberIDs):
+            if data and id_no in data:
+                self.gazeGraphPlots[id_no].setData(data[id_no][0], data[id_no][1])
+
+    def get_circle(self, radius):
+        """ helper function returns circle to x, y coordinates"""
+        theta = np.linspace(0, 2 * np.pi, 100)
+        x = radius * np.cos(theta)
+        y = radius * np.sin(theta)
+        return np.array(x), np.array(y)
+
+    @QtCore.pyqtSlot(list)
+    def onSelectedID(self, lst):
+        """Change color to None of gaze graph plot if ID should not be visible"""
+        for i, button in enumerate(lst):
+            if not button.isChecked():
+                self.gazeGraphPlots[i].setPen(None)
+            else:
+                color = tuple([int(a * 255) for a in self.colors[i]])
+                pen = pg.mkPen(color=color)
+                self.gazeGraphPlots[i].setPen(pen)

uiwidget/widgetobject.py

@@ -0,0 +1,81 @@
+import numpy as np
+import pyqtgraph
+from PyQt5 import QtWidgets, QtCore
+from PyQt5.QtCore import Qt
+from PyQt5.QtWidgets import QTextEdit
+import pyqtgraph as pg
+
+
+class WidgetObject(QtWidgets.QWidget):
+    def __init__(self, parent=None):
+        super(WidgetObject, self).__init__(parent)
+
+        self.frame = 0
+        self.tagFields = QtWidgets.QWidget()
+        self.tagLayout = QtWidgets.QVBoxLayout()
+
+        # Setup Graph Plot Widget
+        self.tagGraph = pg.PlotWidget()
+        self.tagGraph.setBackground('w')
+        self.tagGraph.setYRange(0, 400, padding=0)
+        self.tagGraph.getPlotItem().getAxis('bottom').setTickSpacing(minor=50, major=100)
+        self.tagGraph.getPlotItem().setTitle(title='Movement of Object Tags')
+        self.tagPlots = dict()
+
+        self.tagTextFields = dict()
+        self.plotText = dict()
+
+        layout = QtWidgets.QGridLayout()
+        layout.addWidget(self.tagGraph, 0, 0)
+        layout.addWidget(self.tagFields, 0, 1)
+        self.setLayout(layout)
+
+    @QtCore.pyqtSlot(list, tuple, dict, list)
+    def setInit(self, tags, frameSize, tracked, colors):
+
+        for i, tag in enumerate(tags):
+            label = QtWidgets.QLabel('Object tag #%i:' % tag)
+            label.setStyleSheet('color: black; background-color: rgb(%i,%i,%i)' % colors[i])
+            label.setFixedHeight(20)
+            field = QtWidgets.QTextEdit()
+            field.setFixedHeight(20)
+            field.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
+            field.textChanged.connect(self.tagTextChanged)
+
+            self.tagTextFields[tag] = field
+            trackedLabel = QtWidgets.QLabel('Tracked: {:.0%}'.format(tracked[tag]))
+            oneTagLayout = QtWidgets.QHBoxLayout()
+            oneTagLayout.addWidget(label)
+            oneTagLayout.addWidget(field)
+            oneTagLayout.addWidget(trackedLabel)
+
+            self.tagLayout.insertLayout(-1, oneTagLayout)
+
+            x = list(range(-200, 0))  # 200 time points
+            y = [0 for _ in range(200)]  # 200 data points
+
+            dataLine = self.tagGraph.plot(x, y, pen=pg.mkPen(color=colors[i]))
+            self.tagPlots[tag] = dataLine
+            text = pg.TextItem(text='', color=colors[i])
+            text.setAnchor((1, i + 1))
+
+            self.plotText[tag] = text
+            self.tagGraph.addItem(text)
+
+        self.tagFields.setLayout(self.tagLayout)
+
+    @QtCore.pyqtSlot(dict)
+    def updateTagGraph(self, tagData):
+        for tag, values in tagData.items():
+            self.tagPlots[tag].setData(x=values[1], y=values[0])  # Update the data.
+        if tagData:
+            self.tagGraph.setXRange(np.min(values[1]), np.max(values[1]))
+
+    def tagTextChanged(self):
+        for tag, field in self.tagTextFields.items():
+            self.plotText[tag].setText(field.toPlainText())
+            x, y = self.tagPlots[tag].getData()
+            if len(x) > 0 and len(y) > 0:
+                #print(tag, x[-1], y[-1])
+                self.plotText[tag].setPos(x[-1], y[-1])
+

uiwidget/widgetplayer.py

@@ -0,0 +1,309 @@
+from PyQt5 import QtCore, QtGui, QtWidgets, QtMultimedia, QtMultimediaWidgets, Qt
+
+import os
+import numpy as np
+
+
+
+class WidgetPlayer(QtWidgets.QWidget):
+    updateFrame = QtCore.pyqtSignal(int)
+    # sendFileName = QtCore.pyqtSignal(str)
+    sendState = QtCore.pyqtSignal(QtMultimedia.QMediaPlayer.State)
+    frameAvailable = QtCore.pyqtSignal(QtGui.QImage)
+
+    labels = list()
+    colors = list()
+    pose_data = list()
+    gaze_data = list()
+    tag_data = dict()
+    tags = list()
+    tag_colors = dict()
+
+    def __init__(self, parent=None):
+        super(WidgetPlayer, self).__init__(parent)
+        self.root = QtCore.QFileInfo(__file__).absolutePath()
+
+        # mediaplayer for decoding the video
+        self.mediaPlayer = QtMultimedia.QMediaPlayer(self, QtMultimedia.QMediaPlayer.VideoSurface)
+        #self.mediaPlayer.setMuted(True)
+
+        # top = graphicsscene, middle = graphiscview, bottom = graphicsvideoitem, lowest = graphisctextitems, ...
+        self._scene = QtWidgets.QGraphicsScene(self)
+        self._scene.setBackgroundBrush(QtGui.QBrush(QtGui.QColor('black')))
+        self._gv = QtWidgets.QGraphicsView(self._scene)
+        self._videoitem = QtMultimediaWidgets.QGraphicsVideoItem()
+        self._videoitem.setPos(0, 0)
+        self._videoitem.setZValue(-1000)
+        self._scene.addItem(self._videoitem)
+
+        if os.name != 'nt':
+            # grab frames to forward them to facial emotion tab
+            probe = QtMultimedia.QVideoProbe(self)
+            probe.videoFrameProbed.connect(self.on_videoFrameProbed)
+            probe.setSource(self.mediaPlayer)
+
+        # disable scrollbars
+        self._gv.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
+        self._gv.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
+
+        # just a holder for the graphics view to expand to maximum to use full size
+        self.lay = QtWidgets.QVBoxLayout(self)
+        self.lay.setContentsMargins(0, 0, 0, 0)
+        self.lay.addWidget(self._gv)
+
+        self.errorLabel = QtWidgets.QLabel()
+        self.errorLabel.setSizePolicy(QtWidgets.QSizePolicy.Preferred,
+                                      QtWidgets.QSizePolicy.Maximum)
+
+        self.mediaPlayer.setVideoOutput(self._videoitem)
+        self.mediaPlayer.stateChanged.connect(self.on_stateChanged)
+        self.mediaPlayer.positionChanged.connect(self.mediaChangedPosition)
+        # self.mediaPlayer.durationChanged.connect(self.durationChanged)
+        self.mediaPlayer.error.connect(self.handleError)
+
+        self.movieDir = ''
+        self.duration = 0
+
+    def setInit(self, video, fps, originalVideoResolution, number_ids, colors, tags, tag_colors):
+        self.fps = fps
+        self.originalVideoResolution = originalVideoResolution
+
+        f = os.path.abspath(video)
+        self.mediaPlayer.setMedia(QtMultimedia.QMediaContent(QtCore.QUrl.fromLocalFile(f)))
+        self.mediaPlayer.setNotifyInterval(1000 // self.fps)
+
+        # init pose data
+        for i in range(number_ids):
+            self.pose_data.append(self._scene.addPath(QtGui.QPainterPath()))
+
+        # init gaze data
+        for i in range(number_ids):
+            self.gaze_data.append(self._scene.addPath(QtGui.QPainterPath()))
+
+        # init label data
+        for i in range(number_ids):
+            self.labels.append(self._scene.addPath(QtGui.QPainterPath()))
+
+        # init tag data
+        if tags:
+            for i, tag in enumerate(tags):
+                self.tag_data[tag] = self._scene.addPath(QtGui.QPainterPath())
+                self.tag_colors[tag] = tag_colors[i]
+
+
+        self.number_ids = number_ids
+        self.colors = colors
+        self.tags = tags
+
+    def play(self):
+        if self.mediaPlayer.state() == QtMultimedia.QMediaPlayer.PlayingState:
+            self.mediaPlayer.pause()
+        else:
+            self.mediaPlayer.play()
+        self.sendState.emit(self.mediaPlayer.state())
+
+    def pause(self):
+        if self.mediaPlayer.state() == QtMultimedia.QMediaPlayer.PlayingState:
+            self.mediaPlayer.pause()
+            self.sendState.emit(self.mediaPlayer.state())
+
+    def setFrame(self, frame):
+        # RESPECT FPS! position is time in millisconds
+        position = int(frame * 1000 / self.fps)
+        # print("Received", position)
+        self.mediaPlayer.setPosition(position)
+
+    def stop(self):
+        self.mediaPlayer.stop()
+
+    @QtCore.pyqtSlot(QtMultimedia.QMediaPlayer.State)
+    def on_stateChanged(self, state):
+        self.focus_on_video()
+
+    def mediaChangedPosition(self, position):
+        frame = int((position / 1000.0) * self.fps)
+        # print("Video Running %i" % frame)
+        self.updateFrame.emit(frame)
+        self._gv.fitInView(self._videoitem, QtCore.Qt.KeepAspectRatio)
+
+    def handleError(self):
+        # self.playButton.setEnabled(False)
+        print("Error: " + self.mediaPlayer.errorString())
+
+    def createButtons(self):
+        iconSize = QtCore.QSize(28, 28)
+
+        openButton = QtWidgets.QToolButton()
+        openButton.setStyleSheet('border: none;')
+        openButton.setIcon(QtGui.QIcon(self.root + '/icons/open.png'))
+        openButton.setIconSize(iconSize)
+        openButton.setToolTip("Open File")
+        # openButton.clicked.connect(self.open)
+
+        self.playButton = QtWidgets.QToolButton()
+        self.playButton.setStyleSheet('border: none;')
+        self.playButton.setIcon(QtGui.QIcon(self.root + '/icons/play.png'))
+        self.playButton.setIconSize(iconSize)
+        self.playButton.setToolTip("Play movie")
+        self.playButton.clicked.connect(self.play)
+        self.playButton.setEnabled(False)
+
+        self.stopButton = QtWidgets.QToolButton()
+        self.stopButton.setStyleSheet('border: none;')
+        self.stopButton.setIcon(QtGui.QIcon(self.root + '/icons/stop.png'))
+        self.stopButton.setIconSize(iconSize)
+        self.stopButton.setToolTip("Stop movie")
+        self.stopButton.clicked.connect(self.stop)
+        self.stopButton.setEnabled(False)
+
+    @QtCore.pyqtSlot(QtMultimedia.QVideoFrame)
+    def on_videoFrameProbed(self, frame):
+        cloneFrame = QtMultimedia.QVideoFrame(frame)
+        cloneFrame.map(QtMultimedia.QAbstractVideoBuffer.ReadOnly)
+        image = QtGui.QImage(cloneFrame.bits(), cloneFrame.width(), cloneFrame.height(), cloneFrame.bytesPerLine(),
+                             QtMultimedia.QVideoFrame.imageFormatFromPixelFormat(cloneFrame.pixelFormat()))
+        self.frameAvailable.emit(image)
+        cloneFrame.unmap()
+
+    def focus_on_video(self):
+        native_video_resolution = self.mediaPlayer.metaData("Resolution")
+        # we also update the sceneview to zoom to the video
+        if native_video_resolution is not None:
+            self._videoitem.setSize(QtCore.QSizeF(native_video_resolution.width(), native_video_resolution.height()))
+            self._gv.fitInView(self._videoitem, QtCore.Qt.KeepAspectRatio)
+
+            # set scale of video to bigger size
+            if self.originalVideoResolution is not None:
+                width_ratio = self.originalVideoResolution[0] / native_video_resolution.width()
+                self._videoitem.setScale(width_ratio)
+
+    @QtCore.pyqtSlot()
+    def clear_tags(self):
+        self.focus_on_video()
+        # clear all tags
+        for tag in self.tags:
+            self._scene.removeItem(self.tag_data[tag])
+            self.tag_data[tag] = self._scene.addPath(QtGui.QPainterPath())
+
+    @QtCore.pyqtSlot(int, list, list)
+    def draw_tags(self, tag, lstX, lstY):
+        # this is removing the old tag data
+        self._scene.removeItem(self.tag_data[tag])
+
+        path = QtGui.QPainterPath()
+        path.setFillRule(Qt.Qt.WindingFill)
+        # set starting points
+        for (x, y) in zip(lstX, lstY):
+            path.addRect(x-50, y-50, 100, 100)
+
+        # by adding it gets converted into an QGraphicsPathItem
+        # save it for later removal
+        self.tag_data[tag] = self._scene.addPath(path)
+
+        # set colors
+        color = self.tag_colors[tag]
+        pen = QtGui.QPen(QtGui.QColor(color[0], color[1], color[2], 255), 2, QtCore.Qt.SolidLine)
+        self.tag_data[tag].setPen(pen)
+        # fill ellipses - alpha value is set to 50%
+        # self.tag_data[tag].setBrush(QtGui.QColor(color[0], color[1], color[2], int(0.5 * 255)))
+
+    @QtCore.pyqtSlot()
+    def clear_labels(self):
+        self.focus_on_video()
+        # clear all labels
+        for id_no in range(self.number_ids):
+            self._scene.removeItem(self.labels[id_no])
+            self.labels[id_no] = self._scene.addPath(QtGui.QPainterPath())
+
+    @QtCore.pyqtSlot(int, int, int)
+    def draw_labels(self, id_no, x, y):
+        # this is removing the old pose data
+        self._scene.removeItem(self.labels[id_no])
+
+        path = QtGui.QPainterPath()
+        # then draw text
+        font = QtGui.QFont("Arial", 70)
+        font.setStyleStrategy(QtGui.QFont.ForceOutline)
+        # sadly there is no easy way to claculate the width of the text so minus 100 is fine, but not ideal
+        # also moving the text up by 500, so that is does not cover the face
+        path.addText(x - 100, y - 300, font, "ID " + str(id_no))
+
+        # by adding it gets converted into an QGraphicsPathItem
+        # save it for later removal
+        self.labels[id_no] = self._scene.addPath(path)
+
+        # set colors
+        color = tuple([int(a * 255) for a in self.colors[id_no]])
+        # alpha value is set to 70%
+        pen = QtGui.QPen(QtGui.QColor(color[0], color[1], color[2], int(0.9 * 255)), 10, QtCore.Qt.SolidLine)
+        self.labels[id_no].setPen(pen)
+
+    @QtCore.pyqtSlot()
+    def clear_pose(self):
+        # empty pose data
+        for id_no in range(self.number_ids):
+            self._scene.removeItem(self.pose_data[id_no])
+            self.pose_data[id_no] = self._scene.addPath(QtGui.QPainterPath())
+
+    @QtCore.pyqtSlot(int, list, list)
+    def draw_pose(self, id_no, lstX, lstY):
+        # this is removing the old pose data
+        self._scene.removeItem(self.pose_data[id_no])
+
+        if len(lstX) > 0 and len(lstY) > 0:
+            path = QtGui.QPainterPath()
+
+            # set starting points
+            path.moveTo(lstX[0], lstY[0])
+            # then draw remaing lines
+            for (x, y) in zip(lstX[1:], lstY[1:]):
+                path.lineTo(x, y)
+
+            # by adding it gets converted into an QGraphicsPathItem
+            # save it for later removal
+            self.pose_data[id_no] = self._scene.addPath(path)
+
+            # set colors
+            color = tuple([int(a * 255) for a in self.colors[id_no]])
+            # alpha value is set to 70%
+            pen = QtGui.QPen(QtGui.QColor(color[0], color[1], color[2], int(0.7 * 255)), 10, QtCore.Qt.SolidLine)
+            self.pose_data[id_no].setPen(pen)
+        else:
+            self.pose_data[id_no] = self._scene.addPath(QtGui.QPainterPath())
+
+    @QtCore.pyqtSlot()
+    def clear_gaze(self):
+        # empty pose data
+        for id_no in range(self.number_ids):
+            self._scene.removeItem(self.gaze_data[id_no])
+            self.gaze_data[id_no] = self._scene.addPath(QtGui.QPainterPath())
+
+    @QtCore.pyqtSlot(int, list, list)
+    def draw_gaze(self, id_no, lstX, lstY):
+        # this is removing the old pose data
+        self._scene.removeItem(self.gaze_data[id_no])
+
+        path = QtGui.QPainterPath()
+        path.setFillRule(Qt.Qt.WindingFill)
+        # set starting points
+        for (x, y) in zip(lstX, lstY):
+            path.addEllipse(x, y, 100, 100)
+
+        # by adding it gets converted into an QGraphicsPathItem
+        # save it for later removal
+        self.gaze_data[id_no] = self._scene.addPath(path)
+
+        # set colors
+        color = tuple([int(a * 255) for a in self.colors[id_no]])
+        # alpha value is set to 50%
+        pen = QtGui.QPen(QtGui.QColor(color[0], color[1], color[2], int(0.5 * 255)), 1, QtCore.Qt.SolidLine)
+        self.gaze_data[id_no].setPen(pen)
+        # fill ellipses
+        self.gaze_data[id_no].setBrush(QtGui.QColor(color[0], color[1], color[2], int(0.5 * 255)))
+
+    @QtCore.pyqtSlot(list)
+    def onSelectedID(self, lst):
+        self.clear_labels()
+        self.clear_gaze()
+        self.clear_pose()
+

uiwidget/widgetpose.py

@@ -0,0 +1,147 @@
+import numpy as np
+import pyqtgraph as pg
+from PyQt5 import QtWidgets, QtGui, QtCore
+
+class WidgetPose(QtWidgets.QWidget):
+
+    video_label_signal = QtCore.pyqtSignal(list)
+
+    def __init__(self, parent=None):
+        super(WidgetPose, self).__init__(parent)
+
+        layout = QtWidgets.QGridLayout()
+        self.poseGraph = QtWidgets.QWidget()
+        self.measures = QtWidgets.QWidget()
+        self.measuresLayout = QtWidgets.QHBoxLayout()
+
+        # Setup Movement Graph Plot Widget
+        self.movementGraph = pg.PlotWidget()
+        self.movementGraph.setBackground('w')
+        self.movementGraph.setYRange(0, 400, padding=0)
+        self.movementGraph.getPlotItem().getAxis('bottom').setTickSpacing(minor=50, major=100)
+        self.movementGraph.getPlotItem().setTitle(title='Body Movement over Time')
+        self.movementPlots = []
+
+        layout.addWidget(self.movementGraph, 1, 0, 1, 1)
+        layout.addWidget(self.measures, 1, 1, 1, 1)
+        layout.setColumnStretch(0, 1)
+        layout.setColumnStretch(1, 1)
+
+        self.setLayout(layout)
+        self.colors = None
+        self.labels = dict()
+
+    @QtCore.pyqtSlot(list)
+    def onSelectedID(self, lst):
+        """Change color of movement graph plot if ID should not be visible"""
+        for i, button in enumerate(lst):
+            if not button.isChecked():
+                self.movementPlots[i].setPen(None)
+            elif self.colors is not None:
+                color = tuple([int(a * 255) for a in self.colors[i]])
+                pen = pg.mkPen(color=color)
+                self.movementPlots[i].setPen(pen)
+
+    @QtCore.pyqtSlot(dict, list, int)
+    def updateMovementGraph(self, data, colors, numberIDs):
+        """Plot ID specific movement data from processing class
+        data[id: (movements, frames)]
+        """
+        # handle NaN https://github.com/pyqtgraph/pyqtgraph/issues/1057
+        # downgrade to 5.13 fixes the issue
+        for id_no in range(numberIDs):
+            if data.get(id_no):
+                if not np.all(np.isnan(data.get(id_no)[0])):
+                    self.movementPlots[id_no].setData(data.get(id_no)[1], data.get(id_no)[0])  # Update the data.
+
+        self.movementGraph.setXRange(np.min(data.get(id_no)[1]), np.max(data.get(id_no)[1]))
+
+    @QtCore.pyqtSlot(dict, int)
+    def updateHandVelocity(self, data, numberIDs):
+        """Update Velocity Label
+        data[id: velocity for frame]"""
+        for id_no in range(numberIDs):
+            if data.get(id_no) is not None:
+                self.labels['Velocity'][id_no].setText('%.2f' % data[id_no])
+            else:
+                self.labels['Velocity'][id_no].setText(' ')
+
+    @QtCore.pyqtSlot(list, int)
+    def initMovementGraph(self, colors, numberIDs):
+        """Initialize plot lines with 0
+        colors: plot color for each ID
+        """
+        for i in range(numberIDs):
+            x = list(range(-200, 0))  # 100 time points
+            y = [0 for _ in range(200)]  # 100 data points
+            color = tuple([int(a * 255) for a in colors[i]])
+            pen = pg.mkPen(color=color)
+            dataLine = self.movementGraph.plot(x, y, pen=pen)
+            self.movementPlots.append(dataLine)
+
+    @QtCore.pyqtSlot(dict, dict, list, int)
+    def setInit(self, mostActivity, hand, colors, numberIDs):
+        self.colors = colors
+        idLayout = QtWidgets.QVBoxLayout()
+
+        labelID = QtWidgets.QLabel('      ')
+        labelID.setFixedWidth(60)
+        labelID.setFixedHeight(20)
+        label = QtWidgets.QLabel('Most body activity in frame: ')
+        labelA = QtWidgets.QLabel('Hands above table (relative): ')
+        labelG = QtWidgets.QLabel('Gestures (relative): ')
+        labelVel = QtWidgets.QLabel('Hand velocity: ')
+        idLayout.addWidget(labelID)
+        idLayout.addWidget(label)
+        idLayout.addWidget(labelA)
+        idLayout.addWidget(labelG)
+        idLayout.addWidget(labelVel)
+        self.measuresLayout.insertLayout(-1, idLayout)
+
+        activityLabel = []
+        aboveLabel = []
+        gestureLabel = []
+        velLabel = []
+        for id_no in range(numberIDs):
+            idLayout = QtWidgets.QVBoxLayout()
+
+            [total, tracked, high_vel] = hand.get(id_no)
+
+            color = tuple([int(a * 255) for a in colors[id_no]])
+
+            labelID = QtWidgets.QLabel('ID%i' % id_no)
+            labelID.setStyleSheet('font: bold 12px; color: black; background-color: rgb(%i,%i,%i)' % color)
+            labelID.setFixedWidth(60)
+            labelID.setFixedHeight(20)
+
+            label = QtWidgets.QLabel('%i' % mostActivity.get(id_no))
+            activityLabel.append(label)
+            labelG = QtWidgets.QLabel('%.2f' % (high_vel / total))
+            gestureLabel.append(labelG)
+            labelA = QtWidgets.QLabel('%.2f' % (tracked / total))
+            aboveLabel.append(labelA)
+            labelVel = QtWidgets.QLabel(' ')
+            velLabel.append(labelVel)
+            idLayout.addWidget(labelID)
+            idLayout.addWidget(label)
+            idLayout.addWidget(labelA)
+            idLayout.addWidget(labelG)
+            idLayout.addWidget(labelVel)
+
+            self.measuresLayout.insertLayout(-1, idLayout)
+
+        # Velocity will be updated each frame, rest is updated in _updateLabels
+        self.labels['Velocity'] = velLabel
+        self.labels['Above'] = aboveLabel
+        self.labels['Gesture'] = gestureLabel
+        self.labels['Activity'] = activityLabel
+        self.measures.setLayout(self.measuresLayout)
+
+    @QtCore.pyqtSlot(dict, dict, int)
+    def updateLables(self, mostActivity, hand, numberIDs):
+        """ Update above hands, gestures and most activity labels when segment was changed"""
+        for id_no in range(numberIDs):
+            [total, tracked, high_vel] = hand[id_no]
+            self.labels['Activity'][id_no].setText('%i' % mostActivity[id_no])
+            self.labels['Above'][id_no].setText('%.2f' % (tracked / total))
+            self.labels['Gesture'][id_no].setText('%.2f' % (high_vel / total))

uiwidget/widgetspeaking.py

@@ -0,0 +1,157 @@
+import numpy as np
+import pyqtgraph as pg
+from PyQt5 import QtWidgets, QtCore
+from utils.util import get_circle
+
+
+class WidgetSpeaking(QtWidgets.QWidget):
+    def __init__(self, parent=None):
+        super(WidgetSpeaking, self).__init__(parent)
+
+        layout = QtWidgets.QGridLayout()
+
+        self.measures = QtWidgets.QWidget()
+        self.measuresLayout = QtWidgets.QHBoxLayout()
+
+        # Setup Speaking Graph Plot Widget
+        self.speakingGraph = pg.PlotWidget()
+        self.speakingGraph.setBackground('w')
+        self.speakingGraph.hideAxis('left')
+        self.speakingGraph.hideAxis('bottom')
+        self.speakingGraph.setAspectLocked()
+        self.speakingGraph.setYRange(-2.25, 2.25, padding=0)
+        self.speakingGraph.setXRange(-2.25, 2.25, padding=0)
+        self.speakingGraph.getPlotItem().setTitle(title='Speaking Distribution')
+
+        self.speakingGraphPlots = []
+
+        layout.addWidget(self.speakingGraph, 1, 0, 1, 1)
+        layout.addWidget(self.measures, 1, 1, 1, 2)
+        layout.setColumnStretch(0, 1)
+        layout.setColumnStretch(1, 2)
+
+        self.setLayout(layout)
+        self.colors = None
+        self.labels = dict()
+        self.speakBlob = None
+        self.positions = dict()
+        self.idBlobs = dict()
+
+    @QtCore.pyqtSlot(dict, int)
+    def updateLables(self, speak, numberIDs):
+        """ Update labels when segment was changed"""
+        for id_no in range(numberIDs):
+            self.labels['RST'][id_no].setText('%.2f' % speak[id_no][1])
+            # self.labels['Turns'][id_no].setText('%i' % speak[id_no][2])
+            # self.labels['TurnLength'][id_no].setText('%.2f sec' % speak[id_no][3])
+            # self.labels['TurnPerMin'][id_no].setText('%.2f /min' % speak[id_no][4])
+
+    @QtCore.pyqtSlot(dict, list, int)
+    def setInit(self, speak, colors, numberIDs):
+        self.colors = colors
+        idLayout = QtWidgets.QVBoxLayout()
+
+        labelID = QtWidgets.QLabel('      ')
+        labelID.setFixedWidth(60)
+        labelID.setFixedHeight(20)
+        labelRST = QtWidgets.QLabel('Relative speaking time:')
+        labelTurn = QtWidgets.QLabel('Number of speaking turns:')
+        labelTurnLen = QtWidgets.QLabel('Average length of turn:')
+        labelTurnMin = QtWidgets.QLabel('Average number of turns:')
+        idLayout.addWidget(labelID)
+        idLayout.addWidget(labelRST)
+        idLayout.addWidget(labelTurn)
+        idLayout.addWidget(labelTurnLen)
+        idLayout.addWidget(labelTurnMin)
+
+        self.measuresLayout.insertLayout(-1, idLayout)
+
+        rstLabels = []
+        turnLabels = []
+        turnLenLabels = []
+        turnMinLabels = []
+        for id_no in range(numberIDs):
+            idLayout = QtWidgets.QVBoxLayout()
+            color = tuple([int(a * 255) for a in colors[id_no]])
+
+            labelID = QtWidgets.QLabel('ID%i' % id_no)
+            labelID.setFixedWidth(60)
+            labelID.setFixedHeight(20)
+            labelID.setStyleSheet('font: bold 12px; color: black; background-color: rgb(%i,%i,%i)' % color)
+
+            labelRST = QtWidgets.QLabel('%.2f' % speak[id_no][1])
+            rstLabels.append(labelRST)
+            labelTurn = QtWidgets.QLabel('%i' % speak[id_no][2])
+            turnLabels.append(labelTurn)
+            labelTurnLen = QtWidgets.QLabel('%.2f sec' % speak[id_no][3])
+            turnLenLabels.append(labelTurnLen)
+            labelTurnMin = QtWidgets.QLabel('%.2f /min' % speak[id_no][4])
+            turnMinLabels.append(labelTurnMin)
+
+            idLayout.addWidget(labelID)
+            idLayout.addWidget(labelRST)
+            idLayout.addWidget(labelTurn)
+            idLayout.addWidget(labelTurnLen)
+            idLayout.addWidget(labelTurnMin)
+
+            self.measuresLayout.insertLayout(-1, idLayout)
+
+        self.labels['RST'] = rstLabels
+        self.labels['Turns'] = turnLabels
+        self.labels['TurnLength'] = turnLenLabels
+        self.labels['TurnPerMin'] = turnMinLabels
+
+        self.measures.setLayout(self.measuresLayout)
+
+    @QtCore.pyqtSlot(list, int)
+    def initSpeakingGraph(self, colors, numberIDs):
+        """ initialize speaking graph """
+
+        self.speakBlob = pg.ScatterPlotItem([0], [0], size=0.3, pxMode=False,
+                           pen=pg.mkPen(color=0.5, width=2), brush=pg.mkBrush(255, 255, 255))
+        self.speakBlob.setZValue(100)
+        self.speakingGraph.addItem(self.speakBlob)
+
+        pos = 360 // numberIDs
+        for id_no in range(numberIDs):
+            color = tuple([int(a * 255) for a in colors[id_no]])
+            idPos = pos * id_no
+            x = 2 * np.cos(np.deg2rad(idPos))
+            y = 2 * np.sin(np.deg2rad(idPos))
+            scatterPlot = pg.ScatterPlotItem([x], [y], size=0.2, pxMode=False,
+                                             pen=pg.mkPen(color=color, width=2), brush=pg.mkBrush(*color))
+            self.speakingGraph.addItem(scatterPlot)
+            self.positions[id_no] = [x, y]
+            self.idBlobs[id_no] = scatterPlot
+
+            plt = self.speakingGraph.plot(x=[], y=[], pen=pg.mkPen(color=color, width=1))
+            self.speakingGraphPlots.append(plt)
+
+    @QtCore.pyqtSlot(dict, int, int)
+    def updateSpeakingGraph(self, rst, active_speaker, numberIDs):
+        """ frame updates for gaze graph """
+        blobPos = np.array([0, 0])
+        for id_no in range(numberIDs):
+            diff = rst[id_no] - (1.0 / numberIDs)
+            if diff > 0:
+                blobPos = blobPos + 2 * diff * np.array(self.positions[id_no])
+        self.speakBlob.setData([blobPos[0]], [blobPos[1]])
+
+        for id_no in range(numberIDs):
+            color = tuple([int(a * 255) for a in self.colors[id_no]])
+            pen = pg.mkPen(color=color, width=(rst[id_no]*4))
+
+            x = [self.positions[id_no][0], blobPos[0]]
+            y = [self.positions[id_no][1], blobPos[1]]
+            self.speakingGraphPlots[id_no].setData(x, y)
+            self.speakingGraphPlots[id_no].setPen(pen)
+
+            if id_no == active_speaker:
+                self.idBlobs[id_no].setData([self.positions[id_no][0]], [self.positions[id_no][1]],
+                                            pen=pg.mkPen(color=(0, 0, 0), width=2))
+            else:
+                self.idBlobs[id_no].setData([self.positions[id_no][0]], [self.positions[id_no][1]],
+                                            pen=pg.mkPen(color=color, width=2))
+
+
+

uiwidget/widgetstatic.py

@@ -0,0 +1,15 @@
+import numpy as np
+import pyqtgraph
+from PyQt5 import QtWidgets
+from PyQt5.QtWidgets import QLabel
+
+
+class WidgetStatic(QtWidgets.QWidget):
+    labels = list()
+
+    def __init__(self, parent=None):
+        super(WidgetStatic, self).__init__(parent)
+
+        self.frame = 0
+
+        layout = QtWidgets.QGridLayout()

uiwidget/widgettimeline.py

@@ -0,0 +1,451 @@
+from PyQt5 import QtWidgets
+from PyQt5 import QtCore
+from PyQt5 import QtGui
+from PyQt5.QtCore import QSize, QFileInfo, pyqtSlot
+from PyQt5.QtGui import QIcon, QKeySequence
+from PyQt5.QtMultimedia import QMediaPlayer
+
+import sys
+
+__all__ = ['QRangeSlider']
+
+
+def scale(val, src, dst):
+    """
+    Scale the given value from the scale of src to the scale of dst.
+    """
+    return int(((val - src[0]) / float(src[1] - src[0])) * (dst[1] - dst[0]) + dst[0])
+
+
+class WidgetTimeLine(QtWidgets.QWidget):
+
+    signalSetPosition = QtCore.pyqtSignal(int)
+    signalPlay = QtCore.pyqtSignal()
+    signalStop = QtCore.pyqtSignal()
+    signalSelectID = QtCore.pyqtSignal(list)
+
+    def __init__(self, parent=None):
+        super(WidgetTimeLine, self).__init__(parent)
+        self.root = QFileInfo(__file__).absolutePath()
+
+        self.setFixedHeight(110)
+        self.frame = 0
+
+        layout = QtWidgets.QGridLayout()
+
+        iconSize = QSize(28, 28)
+
+        self.playButton = QtWidgets.QToolButton()
+        self.playButton.setStyleSheet('border: none;')
+        self.playButton.setIcon(QIcon(self.root + '/../icons/play.png'))
+        self.playButton.setIconSize(iconSize)
+        self.playButton.setToolTip("Play movie")
+        self.playButton.clicked.connect(self.play)
+
+        # See https://stackoverflow.com/questions/50880660/how-to-change-space-bar-behaviour-in-pyqt5-python3
+        # self.shortcut_play = QtWidgets.QShortcut(QKeySequence(Qt.Key_Space), self)
+        # self.shortcut_play.activated.connect(self.play)
+
+        self.labelFrame = QtWidgets.QLabel('Frame\n')
+        self.labelFrame.setAlignment(QtCore.Qt.AlignRight)
+        self.labelFrame.setFixedWidth(40)
+        self.sl = QtWidgets.QSlider(QtCore.Qt.Horizontal)
+        # Directly connect: Slider value changed to player set position
+        self.sl.valueChanged.connect(self.signalSetPosition.emit)
+
+        self.rangeslider = QMultiRangeSlider()
+        self.rangeslider.setFixedHeight(30)
+        self.rangeslider.setMin(0)
+        self.rangeslider.setMax(200)
+        self.rangeslider.setRanges([(5, 25), (30, 50), (70, 90)])
+        # self.rangeslider.setBackgroundStyle(
+        #    'background: qlineargradient(x1:0, y1:0, x2:0, y2:1, stop:0 #222, stop:1 #333);')
+        # szlf.rangeslider.setHandleStyle(
+        #    'background: qlineargradient(x1:0, y1:0, x2:0, y2:1, stop:0 #289, stop:1 #289);')
+
+        layout.addWidget(self.rangeslider, 1, 1, 1, 6)
+        layout.addWidget(self.labelFrame, 1, 0, 2, 1)
+        layout.addWidget(self.sl, 2, 1, 1, 6)
+        layout.addWidget(self.playButton, 2, 0, 1, 1)
+
+        self.setLayout(layout)
+        self.number_ids = None
+        self.fps = None
+
+    def setInit(self, frames, number_ids, fps):
+        self.fps = fps
+        self.frame = 0
+        self.sl.setMinimum(0)
+        # Use position = (frame * 1000) / FPS for all slider and player values!
+        self.sl.setMaximum(int((frames * 1000) / self.fps))
+        self.sl.setValue(0)
+        self.sl.setTickPosition(QtWidgets.QSlider.TicksBelow)
+        self.sl.setTickInterval(5)
+        self.labelFrame.setText('Frame\n %i' % self.frame)
+        self.playButton.setEnabled(True)
+        self.number_ids = number_ids
+
+        self.rangeslider.setMin(0)
+        self.rangeslider.setMax(frames)
+        self.rangeslider.setRanges([(0, frames)])
+
+    @pyqtSlot(QMediaPlayer.State)
+    def mediaStateChanged(self, state):
+        if state == QMediaPlayer.PlayingState:
+            print('MediaState changed...')
+            self.playButton.setIcon(QIcon(self.root + '/../icons/pause.png'))
+            self.playButton.setToolTip('Pause movie')
+        else:
+            self.playButton.setIcon(QIcon(self.root + '/../icons/play.png'))
+            self.playButton.setToolTip('Play movie')
+
+    def play(self):
+        self.signalPlay.emit()
+
+    def stop(self):
+        self.signalStop.emit()
+
+    def updateSlider(self, position):
+        # Directly connect Player position changed to slider set value
+        frame = int((position / 1000.0) * self.fps)
+        self.frame = frame
+        # Disable the events to prevent updating triggering a setPosition event (can cause stuttering).
+        self.sl.blockSignals(True)
+        self.sl.setValue(position)
+        self.sl.blockSignals(False)
+
+        self.labelFrame.setText('Frame\n %i' % frame)
+
+
+class QMultiRangeSlider(QtWidgets.QWidget):
+
+    minValueChanged = QtCore.pyqtSignal(int)
+    maxValueChanged = QtCore.pyqtSignal(int)
+
+    segmentsChanged = QtCore.pyqtSignal(list, int)
+
+    segments = []
+    last_update_segment = 0
+
+    def __init__(self, parent=None):
+        """Create a new QRangeSlider instance.
+
+            :param parent: QWidget parent
+            :return: New QRangeSlider instance.
+
+        """
+        super(QMultiRangeSlider, self).__init__(parent)
+        self.setMinimumSize(1, 30)
+        self.padding = 5
+        self.hande_width = 4
+
+        self.selected_segment = None
+
+        self.setMin(0)
+        self.setMax(99)
+        self.setRanges([(20, 30), [40, 90]])
+
+    def setMin(self, value):
+        """sets minimum value"""
+        assert type(value) is int
+        setattr(self, '__min', value)
+        self.minValueChanged.emit(value)
+        self.segments = []
+
+    def setMax(self, value):
+        """sets maximum value"""
+        assert type(value) is int
+        setattr(self, '__max', value)
+        self.maxValueChanged.emit(value)
+        self.segments = []
+
+    def initSegment(self):
+        self.segments = []
+
+        if (self.min() != None & self.max() != None):
+            self.setRanges([(self.min(), self.max())])
+
+    def keyPressEvent(self, event):
+        """overrides key press event to move range left and right"""
+        key = event.key()
+        if key == QtCore.Qt.Key_Left:
+            s = self.start() - 1
+            e = self.end() - 1
+        elif key == QtCore.Qt.Key_Right:
+            s = self.start() + 1
+            e = self.end() + 1
+        else:
+            event.ignore()
+            return
+        event.accept()
+        if s >= self.parent().min() and e <= self.max():
+            self.setRange(s, e)
+
+    def min(self):
+        """:return: minimum value"""
+        return getattr(self, '__min', None)
+
+    def max(self):
+        """:return: maximum value"""
+        return getattr(self, '__max', None)
+
+    def _setStart(self, value):
+        """stores the start value only"""
+        setattr(self, '__start', value)
+        self.startValueChanged.emit(value)
+        # self.segmentChanged.emit(value, self.end())
+
+    def setStart(self, values):
+        """sets the range slider start value"""
+        # assert type(value) is int
+
+        for i, s in enumerate(self.segments):
+            s.setStart(values[i])
+
+    def setEnd(self, values):
+        """set the range slider end value"""
+        # assert type(value) is int
+
+        for i, s in enumerate(self.segments):
+            s.setEnd(values[i])
+
+    def getRanges(self):
+        """:return: the start and end values as a tuple"""
+        ret = []
+        for i, s in enumerate(self.segments):
+            ret.append(s.getRange())
+
+        return ret
+
+    def setRanges(self, values):
+        """set the start and end values"""
+
+        while len(self.segments) < len(values):
+            self.segments.append(QRangeSliderSegment(self))
+
+        while len(self.segments) > len(values):
+            self.segments.remove(-1)
+
+        for i, (s, e) in enumerate(values):
+            self.segments[i].setStart(s)
+            self.segments[i].setEnd(e)
+
+        self._trigger_refresh(0)
+
+    def mouseDoubleClickEvent(self, e):
+        self.selected_segment = None
+
+        d_width = self.width() - (self.padding * 2)
+        step_size = d_width / (self.max() - self.min())
+        pos = (e.x() - (self.padding)) / step_size
+
+        # removing existing segment
+        for (i, s) in enumerate(self.segments):
+            if (s.start() < pos) and (pos < s.end()):
+                # always keep one segment
+                if len(self.segments) > 1:
+                    self.segments.remove(s)
+                # since one segement exists all the time the first can not be -1
+                if i == 0:
+                    self._trigger_refresh(0)
+                else:
+                    self._trigger_refresh(i - 1)
+                return
+
+        # if new segment in front
+        if (pos < self.segments[0].end()):
+            start = 0
+            end = self.segments[0].start()
+            diff = end - start
+            start += .25 * diff
+            end -= .25 * diff
+            start = int(start)
+            end = int(end)
+            seg = QRangeSliderSegment(self)
+            seg.setStart(start)
+            seg.setEnd(end)
+            self.segments.insert(0, seg)
+            self._trigger_refresh(0)
+            return
+
+        # if new segment in back
+        if (pos > self.segments[-1].end()):
+            start = self.segments[-1].end()
+            end = self.max()
+            diff = end - start
+            start += .25 * diff
+            end -= .25 * diff
+            start = int(start)
+            end = int(end)
+            seg = QRangeSliderSegment(self)
+            seg.setStart(start)
+            seg.setEnd(end)
+            self.segments.append(seg)
+            self._trigger_refresh(len(self.segments) - 1)
+            return
+
+        # if new segment in between
+        for i in range(len(self.segments) - 1):
+            if (self.segments[i].end() < pos) and (pos < self.segments[i + 1].start()):
+                start = self.segments[i].end()
+                end = self.segments[i + 1].start()
+                diff = end - start
+                start += .25 * diff
+                end -= .25 * diff
+                start = int(start)
+                end = int(end)
+                seg = QRangeSliderSegment(self)
+                seg.setStart(start)
+                seg.setEnd(end)
+                self.segments.insert(i + 1, seg)
+                self._trigger_refresh(i + 1)
+                return
+
+    def mousePressEvent(self, e):
+        # d_height = painter.device().height() - (self.padding * 2)
+        d_width = self.width() - (self.padding * 2)
+        step_size = d_width / (self.max() - self.min())
+        pos = (e.x() - (self.padding)) / step_size
+
+        distance = sys.maxsize
+
+        for i, s in enumerate(self.segments):
+            if (abs(s.start() - pos) < distance):
+                distance = abs(s.start() - pos)
+                if (distance * step_size < 10):
+                    self.selected_segment = {"Seg": s, "Type": "start", "Index": i}
+
+            if (abs(s.end() - pos) < distance):
+                distance = abs(s.end() - pos)
+                if (distance * step_size < 10):
+                    self.selected_segment = {"Seg": s, "Type": "end", "Index": i}
+
+        self._trigger_refresh(self.last_update_segment)
+
+    def mouseMoveEvent(self, e):
+        d_width = self.width() - (self.padding * 2)
+        step_size = d_width / (self.max() - self.min())
+        pos = int(round((e.x() - (self.padding)) / step_size))
+
+        if self.selected_segment is None:
+            return
+
+        s = self.selected_segment["Seg"]
+        if (self.selected_segment["Type"] == "start"):
+            if ((self.selected_segment["Index"] == 0) & (0 <= pos)):
+                s.setStart(pos)
+            elif (self.segments[self.selected_segment["Index"] - 1].end() < pos - 5) & (s.end() > pos + 5):
+                s.setStart(pos)
+        elif (self.selected_segment["Type"] == "end"):
+            if (self.selected_segment["Index"] == len(self.segments) - 1) & (pos <= self.max()):
+                s.setEnd(pos)
+            elif self.selected_segment["Index"] + 1 < len(self.segments) and (self.segments[self.selected_segment["Index"] + 1].start() > pos + 5) and (s.start() < pos - 5):
+                s.setEnd(pos)
+
+        self._trigger_refresh(self.selected_segment["Index"])
+
+    def mouseReleaseEvent(self, e):
+        self.selected_segment = None
+
+    def _trigger_refresh(self, last_update_segment):
+        self.last_update_segment = last_update_segment
+        self.update()
+        self.segmentsChanged.emit(self.getRanges(), last_update_segment)
+
+    def paintEvent(self, e):
+        painter = QtGui.QPainter(self)
+        brush = QtGui.QBrush()
+        brush.setColor(QtGui.QColor(53, 53, 53))
+        brush.setStyle(QtCore.Qt.SolidPattern)
+        rect = QtCore.QRect(0, 0, painter.device().width(), painter.device().height())
+        painter.fillRect(rect, brush)
+
+        # Define our canvas.
+        d_height = painter.device().height() - (self.padding * 2)
+        d_width = painter.device().width() - (self.padding * 2)
+        step_size = d_width / (self.max() - self.min())
+
+        for s in self.segments:
+
+            brush.setColor(QtGui.QColor('white'))
+            rect = QtCore.QRect(
+                self.padding + s.start() * step_size,
+                self.padding * 2,
+                (s.end() - s.start()) * step_size,
+                d_height - 2 * self.padding
+            )
+            painter.fillRect(rect, brush)
+
+            brush.setColor(QtGui.QColor('red'))
+            rect = QtCore.QRect(
+                self.padding + s.end() * step_size - (self.hande_width / 2),
+                self.padding,
+                (self.hande_width / 2),
+                d_height
+            )
+            painter.fillRect(rect, brush)
+
+            brush.setColor(QtGui.QColor('green'))
+            rect = QtCore.QRect(
+                self.padding + s.start() * step_size - (self.hande_width / 2),
+                self.padding,
+                (self.hande_width / 2),
+                d_height
+            )
+            painter.fillRect(rect, brush)
+
+
+class QRangeSliderSegment(QtWidgets.QWidget):
+
+    # signals
+    # startValueChanged = QtCore.pyqtSignal(int)
+    # endValueChanged = QtCore.pyqtSignal(int)
+    # segmentChanged = QtCore.pyqtSignal(int, int)
+
+    def __init__(self, parent=None):
+        """Create a new QRangeSlider instance.
+
+            :param parent: QWidget parent
+            :return: New QRangeSlider instance.
+
+        """
+        super(QRangeSliderSegment, self).__init__(parent)
+
+    def start(self):
+        """:return: range slider start value"""
+        return getattr(self, '__start', None)
+
+    def end(self):
+        """:return: range slider end value"""
+        return getattr(self, '__end', None)
+
+    def getRange(self):
+        """:return: the start and end values as a tuple"""
+        return (self.start(), self.end())
+
+    def setRange(self, start, end):
+        """set the start and end values"""
+        self.setStart(start)
+        self.setEnd(end)
+
+    def _setEnd(self, value):
+        """stores the end value only"""
+        setattr(self, '__end', value)
+        # self.endValueChanged.emit(value)
+        # self.segmentChanged.emit(self.start(), value)
+
+    def setEnd(self, value):
+        """set the range slider end value"""
+        assert type(value) is int
+        self._setEnd(value)
+
+    def _setStart(self, value):
+        """stores the start value only"""
+        setattr(self, '__start', value)
+        # self.startValueChanged.emit(value)
+        # self.segmentChanged.emit(value, self.end())
+
+    def setStart(self, value):
+        """sets the range slider start value"""
+        assert type(value) is int
+        self._setStart(value)

uiwidget/widgetvideosettings.py

@@ -0,0 +1,63 @@
+from PyQt5 import QtWidgets, QtCore, QtGui
+
+
+class WidgetVideoSettings(QtWidgets.QWidget):
+    signalVisualize = QtCore.pyqtSignal(dict)
+    signalSelectID = QtCore.pyqtSignal(list)
+
+    def __init__(self, parent=None):
+        super(WidgetVideoSettings, self).__init__(parent)
+
+        self.setFixedWidth(200)
+
+        layout = QtWidgets.QGridLayout()
+
+        iconSize = QtCore.QSize(28, 28)
+
+        self.checkBoxes = QtWidgets.QWidget()
+        self.checkBoxesLayout = QtWidgets.QVBoxLayout()
+
+        label = QtWidgets.QLabel('Visualize')
+        label.setFixedWidth(100)
+
+        self.checkBoxes.setLayout(self.checkBoxesLayout)
+        self.checkBoxesList = dict()
+
+        for data in ['Pose', 'Gaze', 'Label', 'Tags']:
+            box = QtWidgets.QCheckBox(data)
+            box.setChecked(True)
+            box.stateChanged.connect(lambda: self.boxstate(box))
+            self.checkBoxesLayout.addWidget(box)
+            self.checkBoxesList[data] = box
+
+        self.idButtons = QtWidgets.QWidget()
+        self.buttonsLayout = QtWidgets.QVBoxLayout()
+
+        labelID = QtWidgets.QLabel('Showing')
+        labelID.setFixedWidth(100)
+
+        self.idButtons.setLayout(self.buttonsLayout)
+        self.buttonsList = []
+
+        layout.addWidget(label, 0, 0)
+        layout.addWidget(labelID, 0, 1)
+        layout.addWidget(self.checkBoxes, 1, 0)
+        layout.addWidget(self.idButtons, 1, 1)
+        self.setLayout(layout)
+
+    @QtCore.pyqtSlot(list, int)
+    def setInit(self, colors, numberIDs):
+        # Emit initial state
+        self.signalVisualize.emit(self.checkBoxesList)
+        for id_no in range(numberIDs):
+            button = QtWidgets.QCheckBox("ID%i" % id_no)
+            button.setChecked(True)
+            button.stateChanged.connect(lambda: self.btnstate(button))
+            self.buttonsLayout.addWidget(button)
+            self.buttonsList.append(button)
+
+    def btnstate(self, b):
+        self.signalSelectID.emit(self.buttonsList)
+
+    def boxstate(self, b):
+        self.signalVisualize.emit(self.checkBoxesList)

utils/colors.py

@@ -0,0 +1,13 @@
+import colorsys
+
+
+def random_colors(N, bright=True):
+    """
+    Generate random colors.
+    To get visually distinct colors, generate them in HSV space then
+    convert to RGB.
+    """
+    brightness = 1.0 if bright else 0.7
+    hsv = [(i / N, 1, brightness) for i in range(N)]
+    colors = list(map(lambda c: colorsys.hsv_to_rgb(*c), hsv))
+    return colors

utils/util.py

@@ -0,0 +1,31 @@
+import numpy as np
+
+
+def sperical2equirec(theta, phi, img_w, img_h):
+    u = np.deg2rad(theta) / (2 * np.pi)
+    v = np.deg2rad(phi) / np.pi
+
+    x = u * img_w
+    y = v * img_h
+    return x, y
+
+
+def equirec2spherical(x, y, img_w, img_h):
+    # Normalize coordinates
+    u = x / img_w
+    v = y / img_h
+
+    # Calculate spherical coordinates from normalized coordinates
+    # theta is horizontal angle, phi is polar angle
+    theta = u * 2 * np.pi
+    phi = v * np.pi
+
+    return np.rad2deg(theta), np.rad2deg(phi)
+
+
+def get_circle(radius):
+    """ helper function returns circle to x, y coordinates"""
+    theta = np.linspace(0, 2 * np.pi, 100)
+    x = radius * np.cos(theta)
+    y = radius * np.sin(theta)
+    return np.array(x), np.array(y)