Commit cc90d980 authored by burcharr's avatar burcharr 💬
Browse files

automatic writing commit ...

parent 666be086
...@@ -1001,7 +1001,7 @@ Subject\_term\_id: computational-neuroscience;computational-science;computer-sci ...@@ -1001,7 +1001,7 @@ Subject\_term\_id: computational-neuroscience;computational-science;computer-sci
booktitle = {2017 International Conference on Engineering and Technology ({ICET})}, booktitle = {2017 International Conference on Engineering and Technology ({ICET})},
author = {Albawi, Saad and Mohammed, Tareq Abed and Al-Zawi, Saad}, author = {Albawi, Saad and Mohammed, Tareq Abed and Al-Zawi, Saad},
date = {2017-08}, date = {2017-08},
keywords = {artificial neural networks, computer vision, Convolution, convolutional neural networks, Convolutional neural networks, deep learning, Feature extraction, Image edge detection, Image recognition, machine learning, Neurons}, keywords = {Feature extraction, deep learning, artificial neural networks, computer vision, Convolution, convolutional neural networks, Convolutional neural networks, Image edge detection, Image recognition, machine learning, Neurons},
file = {IEEE Xplore Full Text PDF:/home/robin/Zotero/storage/GI55HP88/Albawi et al. - 2017 - Understanding of a convolutional neural network.pdf:application/pdf;IEEE Xplore Abstract Record:/home/robin/Zotero/storage/JLJQ2QPK/8308186.html:text/html}, file = {IEEE Xplore Full Text PDF:/home/robin/Zotero/storage/GI55HP88/Albawi et al. - 2017 - Understanding of a convolutional neural network.pdf:application/pdf;IEEE Xplore Abstract Record:/home/robin/Zotero/storage/JLJQ2QPK/8308186.html:text/html},
} }
...@@ -1018,7 +1018,7 @@ Subject\_term\_id: computational-neuroscience;computational-science;computer-sci ...@@ -1018,7 +1018,7 @@ Subject\_term\_id: computational-neuroscience;computational-science;computer-sci
author = {Lawrence, S. and Giles, C.L. and Tsoi, Ah Chung and Back, A.D.}, author = {Lawrence, S. and Giles, C.L. and Tsoi, Ah Chung and Back, A.D.},
date = {1997-01}, date = {1997-01},
note = {Conference Name: {IEEE} Transactions on Neural Networks}, note = {Conference Name: {IEEE} Transactions on Neural Networks},
keywords = {Face recognition, Feature extraction, Humans, Image databases, Image sampling, Karhunen-Loeve transforms, Multilayer perceptrons, Neural networks, Quantization, Spatial databases}, keywords = {Neural networks, Humans, Feature extraction, Face recognition, Image databases, Image sampling, Karhunen-Loeve transforms, Multilayer perceptrons, Quantization, Spatial databases},
file = {IEEE Xplore Full Text PDF:/home/robin/Zotero/storage/I72QVX2C/Lawrence et al. - 1997 - Face recognition a convolutional neural-network a.pdf:application/pdf;IEEE Xplore Abstract Record:/home/robin/Zotero/storage/2ALTIFHU/554195.html:text/html}, file = {IEEE Xplore Full Text PDF:/home/robin/Zotero/storage/I72QVX2C/Lawrence et al. - 1997 - Face recognition a convolutional neural-network a.pdf:application/pdf;IEEE Xplore Abstract Record:/home/robin/Zotero/storage/2ALTIFHU/554195.html:text/html},
} }
...@@ -1047,7 +1047,7 @@ Subject\_term\_id: computational-neuroscience;computational-science;computer-sci ...@@ -1047,7 +1047,7 @@ Subject\_term\_id: computational-neuroscience;computational-science;computer-sci
booktitle = {10th International Conference on Pattern Recognition [1990] Proceedings}, booktitle = {10th International Conference on Pattern Recognition [1990] Proceedings},
author = {Le Cun, Y. and Matan, O. and Boser, B. and Denker, J.S. and Henderson, D. and Howard, R.E. and Hubbard, W. and Jacket, L.D. and Baird, H.S.}, author = {Le Cun, Y. and Matan, O. and Boser, B. and Denker, J.S. and Henderson, D. and Howard, R.E. and Hubbard, W. and Jacket, L.D. and Baird, H.S.},
date = {1990-06}, date = {1990-06},
keywords = {Computer networks, Data mining, Feature extraction, Handwriting recognition, Neural networks, Nonhomogeneous media, Pattern recognition, Postal services, Spatial databases, Testing}, keywords = {Neural networks, Feature extraction, Spatial databases, Computer networks, Data mining, Handwriting recognition, Nonhomogeneous media, Pattern recognition, Postal services, Testing},
file = {IEEE Xplore Abstract Record:/home/robin/Zotero/storage/LEHAD9HH/119325.html:text/html;IEEE Xplore Full Text PDF:/home/robin/Zotero/storage/5NV33HI6/Le Cun et al. - 1990 - Handwritten zip code recognition with multilayer n.pdf:application/pdf}, file = {IEEE Xplore Abstract Record:/home/robin/Zotero/storage/LEHAD9HH/119325.html:text/html;IEEE Xplore Full Text PDF:/home/robin/Zotero/storage/5NV33HI6/Le Cun et al. - 1990 - Handwritten zip code recognition with multilayer n.pdf:application/pdf},
} }
...@@ -1061,4 +1061,29 @@ Subject\_term\_id: computational-neuroscience;computational-science;computer-sci ...@@ -1061,4 +1061,29 @@ Subject\_term\_id: computational-neuroscience;computational-science;computer-sci
urldate = {2021-11-03}, urldate = {2021-11-03},
date = {2012}, date = {2012},
file = {Full Text PDF:/home/robin/Zotero/storage/MRFGVD3R/Krizhevsky et al. - 2012 - ImageNet Classification with Deep Convolutional Ne.pdf:application/pdf}, file = {Full Text PDF:/home/robin/Zotero/storage/MRFGVD3R/Krizhevsky et al. - 2012 - ImageNet Classification with Deep Convolutional Ne.pdf:application/pdf},
}
@online{scholl_pyav_2021,
title = {{PyAV}},
rights = {{BSD}-3-Clause},
url = {https://github.com/pscholl/PyAV},
abstract = {Pythonic bindings for {FFmpeg}'s libraries.},
author = {Scholl, Philipp M.},
urldate = {2021-11-07},
date = {2021-05-05},
note = {original-date: 2019-03-25T14:00:49Z},
}
@article{szegedy_rethinking_2015,
title = {Rethinking the Inception Architecture for Computer Vision},
url = {http://arxiv.org/abs/1512.00567},
abstract = {Convolutional networks are at the core of most state-of-the-art computer vision solutions for a wide variety of tasks. Since 2014 very deep convolutional networks started to become mainstream, yielding substantial gains in various benchmarks. Although increased model size and computational cost tend to translate to immediate quality gains for most tasks (as long as enough labeled data is provided for training), computational efficiency and low parameter count are still enabling factors for various use cases such as mobile vision and big-data scenarios. Here we explore ways to scale up networks in ways that aim at utilizing the added computation as efficiently as possible by suitably factorized convolutions and aggressive regularization. We benchmark our methods on the {ILSVRC} 2012 classification challenge validation set demonstrate substantial gains over the state of the art: 21.2\% top-1 and 5.6\% top-5 error for single frame evaluation using a network with a computational cost of 5 billion multiply-adds per inference and with using less than 25 million parameters. With an ensemble of 4 models and multi-crop evaluation, we report 3.5\% top-5 error on the validation set (3.6\% error on the test set) and 17.3\% top-1 error on the validation set.},
journaltitle = {{arXiv}:1512.00567 [cs]},
author = {Szegedy, Christian and Vanhoucke, Vincent and Ioffe, Sergey and Shlens, Jonathon and Wojna, Zbigniew},
urldate = {2021-11-07},
date = {2015-12-11},
eprinttype = {arxiv},
eprint = {1512.00567},
keywords = {Computer Science - Computer Vision and Pattern Recognition},
file = {arXiv Fulltext PDF:/home/robin/Zotero/storage/NEA2CRUI/Szegedy et al. - 2015 - Rethinking the Inception Architecture for Computer.pdf:application/pdf;arXiv.org Snapshot:/home/robin/Zotero/storage/7BBTB7CG/1512.html:text/html},
} }
\ No newline at end of file
This diff is collapsed.
...@@ -14,28 +14,28 @@ Added to that, hand washing using soap or disinfectants is also part of the work ...@@ -14,28 +14,28 @@ Added to that, hand washing using soap or disinfectants is also part of the work
In order to monitor the effectiveness and frequency of hand washing, we could use a sensor-based computer system to detect the activity of hand washing and its duration. Further advanced systems could also be used to predict the quality of the hand washing. These systems could then be used to reduce the risk of contaminations or infections by ameliorating the hygiene of their users. In order to monitor the effectiveness and frequency of hand washing, we could use a sensor-based computer system to detect the activity of hand washing and its duration. Further advanced systems could also be used to predict the quality of the hand washing. These systems could then be used to reduce the risk of contaminations or infections by ameliorating the hygiene of their users.
### Obsessive-Compulsive Disorders ### Obsessive-Compulsive disorder
While it is usually helpful and a basic part of hygiene, hand washing can also be overdone, i.e. be too frequent or be done too thoroughly. One example of persons for which overly excessive hand washing is a problem, is the small percentage of humans suffering from Obsessive-Compulsive Disorders (OCD). OCD affects about $1-3\,\%$ of humans during their life @valleni-basile_frequency_1994, @fawcett_women_2020. OCD appears in the form of obsessions, that lead to compulsive behavior. There are multiple subgroups of obsessions and compulsions, including contamination concerns, symmetry and precision concerns, saving concerns and more @stein_obsessive-compulsive_2002. These concerns lead to respective compulsive behavior: Symmetry and precision concerns lead to arranging and ordering, saving concerns lead to hoarding and contamination concerns can lead to excessive washing, bathing and showering, including compulsive hand washing. This work will focus on detecting hand washing and will also try to tell apart hand washing from compulsive hand washing of OCD patients. While it is usually helpful and a basic part of hygiene, hand washing can also be overdone, i.e. be too frequent or be done too thoroughly. One example of persons for which overly excessive hand washing is a problem, is the small percentage of humans suffering from Obsessive-Compulsive Disorder (OCD). OCD affects about $1-3\,\%$ of humans during their life @valleni-basile_frequency_1994, @fawcett_women_2020. OCD appears in the form of obsessions, that lead to compulsive behavior. There are multiple subgroups of obsessions and compulsions, including contamination concerns, symmetry and precision concerns, saving concerns and more @stein_obsessive-compulsive_2002. These concerns lead to respective compulsive behavior: Symmetry and precision concerns lead to arranging and ordering, saving concerns lead to hoarding and contamination concerns can lead to excessive washing, bathing and showering, including compulsive hand washing. This work will focus on detecting hand washing and will also try to tell apart hand washing from compulsive hand washing of OCD patients.
One method of treatment for clinical cases of OCD is exposure and response prevention (ERP) therapy @meyer_modification_1966 @whittal_treatment_2005. Using this method, patients that suffer from OCD are exposed to situations in which their obsessions are stimulated and they are helped at preventing compulsive reactions to the stimulation. The patients can then "get used" to the situation in a sense, and thus the reaction to the stimulation will be weakened over time. This means that their quality of life is improved, as the severity of their OCD declines. One method of treatment for clinical cases of OCD is exposure and response prevention (ERP) therapy @meyer_modification_1966 @whittal_treatment_2005. Using this method, patients that suffer from OCD are exposed to situations in which their obsessions are stimulated and they are helped at preventing compulsive reactions to the stimulation. The patients can then "get used" to the situation in a sense, and thus the reaction to the stimulation will be weakened over time. This means that their quality of life is improved, as the severity of their OCD declines.
A successful, i.e. reliable and accurate system for compulsive hand washing detection could be used to intervene, whenever the compulsive hand washing is detected. It could therefore help psychologists and their patients in the treatment of the symptoms. It could help the user to stop the compulsive behavior by issuing a warning. Such a warning could be a vibration of the device, or a sound that is played upon the detection of compulsive behavior. However, the hypothesis of usefulness is yet to be tested, as no such systems exists as of now. Therefore we want to develop a system that can not only detect hand washing with low latency and in real time, but also discriminate between usual hand washing and obsessive-compulsive hand washing at the same time. The system could then, as described, be used in ERP therapy sessions, but also in everyday life, to prevent compulsive hand washing. A successful, i.e. reliable and accurate system for compulsive hand washing detection could be used to intervene, whenever the compulsive hand washing is detected. It could therefore help psychologists and their patients in the treatment of the symptoms. It could help the user to stop the compulsive behavior by issuing a warning. Such a warning could be a vibration of the device, or a sound that is played upon the detection of compulsive behavior. However, the hypothesis of usefulness is yet to be tested, as no such systems exists as of now. Therefore we want to develop a system that can not only detect hand washing with low latency and in real-time, but also discriminate between usual hand washing and obsessive-compulsive hand washing at the same time. The system could then, as described, be used in ERP therapy sessions, but also in everyday life, to prevent compulsive hand washing.
The separation of compulsive hand washing from ordinary hand washing is an even harder problem than just hand washing detection itself. It is unclear, whether it is possible to predict the type of hand washing with high probability, as there is no previous work in this area. It is reasonable to assume, that there are strong similarities between compulsive hand washing and non-compulsive hand washing, as well as subtle differences, e.g. in intensity and duration of the washing. The separation of compulsive hand washing from ordinary hand washing could be an even harder problem than just hand washing detection itself. It is unclear, whether it is possible to predict the type of hand washing with high probability, as there is no previous work in this area. It is reasonable to assume, that there are strong similarities between compulsive hand washing and non-compulsive hand washing, as well as subtle differences, e.g. in intensity and duration of the washing.
### Wrist worn sensors ### Wrist worn sensors
Different types of sensors can be used to detect activities such as hand washing. It is possible to detect hand washing from RGB camera data to some extent. However, for this to work, we would need to place a camera at every place and room a subject could want to wash their hands at. This is unfeasible for most applications of hand washing detection and could be very expensive. Furthermore, it might be problematic to place cameras inside wash- or bathrooms for privacy reasons. Thus, a better alternative could be body worn, camera-less devices. Different types of sensors can be used to detect activities such as hand washing. It is possible to detect hand washing from RGB camera data to some extent. However, for this to work, we would need to place a camera at every place and room a subject could want to wash their hands at. This is unfeasible for most applications of hand washing detection and could be very expensive. Furthermore, it might be problematic to place cameras inside wash- or bathrooms for privacy reasons. Thus, a better alternative could be body worn, camera-less devices.
Inertial measurement units (IMUs) can measure different types of time series movement data, e.g. the acceleration or angular velocity of the device they are embedded in. IMUs are embedded in most modern smart phones and smart watches, which makes them easily available. For hand washing detection, especially the movement of the hands and wrists can contain information that can help us classify hand washing. Therefore, we can use a smart watch and its embedded IMU to try to predict whether a user is washing their hands or not. Added to that, if the user is washing their hands, we could try to predict if they are washing them in an obsessive-compulsive way or not. Another advantage of using a smart watch would be, that they usually have in-built vibration motors or even speakers. These means could be used to intervene, whenever compulsive hand washing is detected, as described above. Therefore, wrist worn sensors, especially those embedded in smart watch systems, are used in this work. The wrist worn devices can also be used to execute machine learning models in real time, using publicly available libraries, e.g. on smart watches running Wear OS. Inertial measurement units (IMUs) can measure different types of time series movement data, e.g. the acceleration or angular velocity of the device they are embedded in. IMUs are embedded in most modern smart phones and smart watches, which makes them easily available. For hand washing detection, especially the movement of the hands and wrists can contain information that can help us classify the activity. Therefore, we can use a smart watch and its embedded IMU to try to predict whether a user is washing their hands or not. Added to that, if the user is washing their hands, we could try to predict if they are washing them in an obsessive-compulsive way or not. Another advantage of using a smart watch would be, that they usually have in-built vibration motors or even speakers. These means could be used to intervene, whenever compulsive hand washing is detected, as described above. Therefore, wrist worn sensors, especially those embedded in smart watch systems, are used in this work. The wrist worn devices can also be used to execute machine learning models in real-time, using publicly available libraries, e.g. on smart watches running Wear OS.
## Goals ## Goals
In this work, we want to develop several neural network based machine learning methods for the real time detection of hand washing and compulsive hand washing on inertial sensor data of wrist worn devices. We also want to test the methods and report meaningful statistics for their performance. Further, we want to test parts of the developed methods in a real-world scenario. We then want to draw conclusions on the applicability of the developed systems in the real-world. In this work, we want to develop several neural network based machine learning methods for the real-time detection of hand washing and compulsive hand washing on inertial sensor data of wrist worn devices. We also want to test the methods and report meaningful statistics for their performance. Further, we want to test parts of the developed methods in a real-world scenario. We then want to draw conclusions on the applicability of the developed systems in the real-world.
### Detection of hand washing in real time utilizing inertial measurement sensors ### Detection of hand washing in real-time utilizing inertial measurement sensors
We want to show that neural network based classification methods can be applied to the recognition of hand washing. We want to base our method on sensor data from inertial measurement sensors in smart watches or other wrist worn IMU-equipped devices. We want to detect the hand washing in real time and directly on the mobile, i.e. on a wrist wearable device, such as a smart watch. Doing so, we would be able to give instant real time feedback to the user of the device. We want to show that neural network based classification methods can be applied to the recognition of hand washing. We want to base our method on sensor data from inertial measurement sensors in smart watches or other wrist worn IMU-equipped devices. We want to detect the hand washing in real-time and directly on the mobile, i.e. on a wrist wearable device, such as a smart watch. Doing so, we would be able to give instant real-time feedback to the user of the device.
### Separation of hand washing and compulsive hand washing ### Separation of hand washing and compulsive hand washing
On top of the detection of hand washing, the detection of obsessive-compulsive hand washing is part of our goals. We want to be able to separate compulsive hand washing from non-compulsive hand washing, based on inertial motion data. Especially for the scenario of possible interventions used for the treatment of OCD, this separation is crucial, as OCD patients do also wash their hands in non-compulsive ways and we do not want to intervene for these kinds of hand washing procedures. On top of the detection of hand washing, the detection of obsessive-compulsive hand washing is part of our goals. We want to be able to separate compulsive hand washing from non-compulsive hand washing, based on inertial motion data. Especially for the scenario of possible interventions used for the treatment of OCD, this separation is crucial, as OCD patients do also wash their hands in non-compulsive ways and we do not want to intervene for these ordinary hand washing procedures.
### Real-world evaluation ### Real-world evaluation
We want to evaluate the most promising of the developed models in a real-world evaluation, in order to obtain a realistic estimate of its applicability in the task of hand washing detection. We want to report results of an evaluation with multiple subjects to obtain a meaningful performance estimation. From this estimation we want to draw conclusions on the applicability of the developed system in real world therapy scenarios. Added to that, we want to derive future improvements, that could be applied to the system. We want to evaluate the most promising of the developed models in a real-world evaluation, in order to obtain a realistic estimate of its applicability in the task of hand washing detection. We want to report results of an evaluation with multiple subjects to obtain a meaningful performance estimation. From this estimation we want to draw conclusions on the applicability of the developed system in real-world therapy scenarios. Added to that, we want to derive future improvements, that could be applied to the system.
...@@ -34,6 +34,6 @@ reviewer2: "Prof. Dr. Thomas Brox" ...@@ -34,6 +34,6 @@ reviewer2: "Prof. Dr. Thomas Brox"
declaration: Hiermit erkläre ich, dass ich diese Arbeit selbstständig verfasst habe, keine anderen als die angegebenen Quellen/Hilfsmittel verwendet habe und alle Stellen, die wörtlich oder sinngemäß aus veröffentlichten Schriften entnommen wurden, als solche kenntlich gemacht habe. Darüber hinaus erkläre ich, dass diese Arbeit nicht, auch nicht auszugsweise, bereits für eine andere Prüfung angefertigt wurde. declaration: Hiermit erkläre ich, dass ich diese Arbeit selbstständig verfasst habe, keine anderen als die angegebenen Quellen/Hilfsmittel verwendet habe und alle Stellen, die wörtlich oder sinngemäß aus veröffentlichten Schriften entnommen wurden, als solche kenntlich gemacht habe. Darüber hinaus erkläre ich, dass diese Arbeit nicht, auch nicht auszugsweise, bereits für eine andere Prüfung angefertigt wurde.
#abstract #abstract
abstract-de: Die automatische Erkennung von Händewaschen und zwanghaftem Händewaschen hat mehrere Anwendungsbereiche in Arbeits- und medizinischen Umgebungen. Die Erkennung kann zur Überprüfung der Einhaltung von Hygieneregeln eingesetzt werden, da das Händewaschen eine der wichtigsten Komponenten der persönlichen Hygiene ist. Allerdings kann das Waschen auch übertrieben werden, was bedeutet, dass es für die Haut und die allgemeine Gesundheit schädlich sein kann. Manche Patienten mit Zwangsstörungen waschen sich zwanghaft und zu häufig die Hände auf diese schädliche Weise. Die automatische Erkennung von zwanghaftem Händewaschen kann bei der Behandlung dieser Patienten helfen. Ziel dieser Arbeit ist es, auf neuronalen Netzen basierende Methoden zu entwickeln, die in der Lage sind, Händewaschen und zwanghaftes Händewaschen in Echtzeit auf einem am Handgelenk getragenen Gerät zu erkennen, wobei die Daten der Bewegungssensoren des am Handgelenk getragenen Geräts verwendet werden. Die entwickelte Methode erreicht eine hohe Genauigkeit für beide Aufgaben und Teile der Arbeit wurden mit Probanden in einem realen Experiment evaluiert, um die starke theoretische Leistung (F1 score von 89,2 % bzw. 96,6 %) zu bestätigen. abstract-de: Die automatische Erkennung von Händewaschen und zwanghaftem Händewaschen hat mehrere Anwendungsbereiche in Arbeitsumgebungen und im medizinischen Bereich. Die Erkennung kann zur Überprüfung der Einhaltung von Hygieneregeln eingesetzt werden, da das Händewaschen eine der wichtigsten Komponenten der persönlichen Hygiene ist. Allerdings kann das Waschen auch übertrieben werden, was bedeutet, dass es für die Haut und die allgemeine Gesundheit schädlich sein kann. Manche Patienten mit Zwangsstörungen waschen sich zwanghaft zu häufig und intensiv die Hände auf diese schädliche Weise. Die automatische Erkennung von zwanghaftem Händewaschen kann bei der Behandlung dieser Patienten helfen. Ziel dieser Arbeit ist es, auf neuronalen Netzen basierende Methoden zu entwickeln, die in der Lage sind, Händewaschen und zwanghaftes Händewaschen in Echtzeit auf einem am Handgelenk getragenen Gerät zu erkennen, wobei die Daten der Bewegungssensoren des Geräts verwendet werden. Die entwickelte Methode erreicht eine hohe Genauigkeit für beide Aufgaben. Sie erreicht einen F1 score von 89,2 % für die Erkennung von Händewaschen bzw. 96,6 % für die Erkennung von zwanghaftem Händewaschen. Teile der Arbeit wurden mit Probanden in einem realen Experiment evaluiert, um die starke theoretische Leistung zu bestätigen.
abstract-en: The automatic detection of hand washing and compulsive hand washing has multiple areas of application in work and medical environments. The detection can be used in compliance and hygiene scenarios, as hand washing is one of the main components of personal hygiene. However, the washing can also be overdone, which means it can be unhealthy for the skin and general health. Patients with obsessive-compulsive disorder sometimes compulsively wash their hands in such a harmful way. In order to help with their treatment, the automatic detection of compulsive hand washing can possibly be applied. This thesis aims to develop neural network based methods which are able to detect hand washing as well as compulsive hand washing in real time on a wrist worn device using inertial motion sensor data of said wrist worn device. We achieve high accuracy for both tasks and evaluate parts of the work on subjects in a real world experiment, in order to confirm the strong theoretical performance (F1 score of 89.2 % and 96.6 %) achieved. abstract-en: The automatic detection of hand washing and compulsive hand washing has multiple areas of application in work and medical environments. The detection can be used in compliance and hygiene scenarios, as hand washing is one of the main components of personal hygiene. However, the washing can also be overdone, which means it can be unhealthy for the skin and general health. Patients with obsessive-compulsive disorder sometimes compulsively wash their hands in such a harmful way. In order to help with their treatment, the automatic detection of compulsive hand washing can possibly be applied. This thesis aims to develop neural network based methods which are able to detect hand washing as well as compulsive hand washing in real time on a wrist-worn device using inertial motion sensor data of the device. We achieve high accuracy for both tasks. We reach an F1 score of 89.2 % for hand washing detection and 96.6 % for compulsive hand washing detection. We evaluate parts of the work on subjects in a real world experiment, in order to confirm the strong theoretical performance achieved.
--- ---
This diff is collapsed.
...@@ -6,24 +6,24 @@ Automatically detecting the current activity of a human being is a wide research ...@@ -6,24 +6,24 @@ Automatically detecting the current activity of a human being is a wide research
In the area of gesture recognition, we try to detect and classify specific, and narrowly defined gestures. In the area of gesture recognition, we try to detect and classify specific, and narrowly defined gestures.
The defined gestures can e.g. be used to actively control a system @saini_human_2020. This kind of approach is not directly applicable to our task of detecting hand washing. However, it could be possible to adapt algorithms from this field to the detection of a new gesture or a new set of gestures related to hand washing. The defined gestures can e.g. be used to actively control a system @saini_human_2020. This kind of approach is not directly applicable to our task of detecting hand washing. However, it could be possible to adapt algorithms from this field to the detection of a new gesture or a new set of gestures related to hand washing.
There are camera-based approaches and physical measurement-based approaches @saini_human_2020. The camera-based approaches were out of scope for this work. As explained in the introduction, in our setting, wrist worn devices have significant advantages over camera-based solutions that would have to be stationary, i.e. in fixed locations. There are camera-based approaches and physical measurement-based approaches @saini_human_2020. The camera-based approaches were out of scope for this work. As explained in the introduction, in our setting, wrist-worn devices have significant advantages over camera-based solutions that would have to be stationary, i.e. in fixed locations.
There also exist approaches based on inertial measurement sensors. These sensors measure movement related physical values, such as the force or acceleration, angular velocity or orientation in space. There also exist approaches based on inertial measurement sensors. These sensors measure movement related physical values, such as the acceleration, the angular velocity or the orientation in space.
Gesture recognition, in general, uses similar methods as the more difficult human activity recognition @saini_human_2020, which will be explained below. Gesture recognition, in general, uses similar methods as the more difficult human activity recognition @saini_human_2020, which will be explained below.
## Human activity recognition ## Human activity recognition
\label{section:har} \label{section:har}
Recognizing more than one gesture or body movement in combination in a temporal context and deriving the current activity of the user is called human activity recognition (HAR). In this task, we want to detect more general activities, compared to the shorter and simpler gestures. An activity can include many distinguishable gestures. However, the same activity will not always include all of the same gestures and the gestures included could be in a different order for every repetition. Activities are less repetitive than gestures, and harder to detect in general @zhu_wearable_2011. However, Zhu et al. have shown that the combined detection of multiple different gestures can be used in HAR tasks too @zhu_wearable_2011, which makes sense, because a human activity can consist of many gestures. Nevertheless, most methods used for HAR consist of more direct applications of machine learning to the data, without the detour of detecting specific gestures contained in the execution of an activity. Recognizing more than one gesture or body movement in combination in a temporal context and deriving the current activity of the user is called human activity recognition (HAR). In this task, we want to detect more general activities, compared to the shorter and simpler gestures. An activity can include many distinguishable gestures. However, the same activity will not always include all of the same gestures and the gestures that are implicitly included could be in a different order for every repetition. Activities are less repetitive than gestures, and harder to detect in general @zhu_wearable_2011. However, Zhu et al. have shown that the combined detection of multiple different gestures can be used in HAR tasks too @zhu_wearable_2011, which makes sense, because a human activity can consist of many gestures. Nevertheless, most methods used for HAR consist of more direct applications of machine learning to the data, without the detour of detecting specific gestures contained in the execution of an activity.
Methods used in HAR include classical machine learning methods as well as deep learning @liu_overview_2021 @bulling_tutorial_2014. The classical machine learning methods rely on features of the data obtained by feature engineering. The required feature engineering is the creation of meaningful statistics or calculations based on the time frame for which the activity should be predicted. The features can be frequency-domain-based and time-domain-based, but usually both are used at the same time to train these conventional models @liu_overview_2021. The classical machine learning methods include but are not limited to Random Forests (RFC), Hidden Markov Models (HMM), Support Vector Machines (SVM), the $k$-nearest neighbors algorithm and more. Methods used in HAR include classical machine learning methods as well as deep learning @liu_overview_2021 @bulling_tutorial_2014. The classical machine learning methods rely on features of the data obtained by feature engineering. The required feature engineering is the creation of meaningful statistics or calculations based on the time frame for which the activity should be predicted. The features can be frequency-domain-based and time-domain-based, but usually both are used at the same time to train these conventional models @liu_overview_2021. The classical machine learning methods include but are not limited to Random Forests (RFC), Hidden Markov Models (HMM), Support Vector Machines (SVM), the $k$-nearest neighbors algorithm and more.
#### Deep neural networks #### Deep neural networks
Recently, deep neural networks have taken over the role of the state-of-the-art machine learning method in the area of human activity recognition @bock_improving_2021, @liu_overview_2021. Deep neural networks are universal function approximators @bishop_pattern_2006 and are known for being easy to use on "raw" data. They are "artificial neural networks" consisting of multiple layers, where each layer contains a certain number of nodes that are connected to the nodes of the following layer. The connections are each assigned a weight, and the weighted sum over the values of all the previous connected nodes is used to calculate the value of a node in the next layer. Simple neural networks where all nodes of a layer are connected to all nodes in the following layer are often called "fully connected neural networks" (FC-NN or FC). Recently, deep neural networks have taken over the role of the state-of-the-art machine learning method in the area of human activity recognition @bock_improving_2021, @liu_overview_2021. Deep neural networks are universal function approximators @bishop_pattern_2006 and are known for being easy to use on "raw" data. They are "artificial neural networks" consisting of multiple layers, where each layer contains a certain number of nodes that are connected to the nodes of the following layer. The connections are each assigned a weight, and the weighted sum over the values of all the previous connected nodes is used to calculate the value of a node in the next layer. Simple neural networks where all nodes of a layer are connected to all nodes in the following layer are often called "fully connected neural networks" (FC-NN or FC).
The connections' parameters are optimized using forward passes through the network of nodes, followed by the execution of the backpropagation algorithm, and an optimization step. We can accumulate all the gradients with regard to a loss function for each of the parameters and for a small subset of the data passed and perform "stochastic gradient decent" (SGD). SGD or alternative similar optimization methods like the commonly used ADAM @kingma_adam_2017 optimizer perform a parameter update step. After many such updates and if the training works well, the network parameters will have been updated to values that lead to a lower value of the loss function for the training data. However, there is no guarantee of convergence whatsoever. As mentioned above, deep neural networks can, in theory, be used to approximate arbitrary functions. Nevertheless, the parameters for the perfect approximation cannot be easily found, and empirical testing has revealed that neural networks do need a lot of training data in order to perform well, compared to classical machine learning methods. In return, with enough data, deep neural networks often outperform classical machine learning methods. The connections' parameters are optimized using forward passes through the network of nodes, followed by the execution of the backpropagation algorithm, and an optimization step. We can accumulate all the gradients with regard to a loss function for each of the parameters and for a small subset of the data (mini-batch) passed and perform "stochastic gradient decent" (SGD). SGD or alternative similar optimization methods like the commonly used ADAM @kingma_adam_2017 optimizer perform a parameter update step. After many such updates and if the training works well, the network parameters will have been updated to values that lead to a lower value of the loss function for the training data. However, there is no guarantee of convergence whatsoever. As mentioned above, deep neural networks can, in theory, be used to approximate arbitrary functions. Nevertheless, the parameters for the perfect approximation cannot be easily found, and empirical testing has revealed that neural networks do need a lot of training data in order to perform well, compared to classical machine learning methods. In return, with enough data, deep neural networks often outperform classical machine learning methods.
###### Convolutional neural networks (CNNs) ###### Convolutional neural networks (CNNs)
are neural networks that are not fully connected, but work by using convolutions with a kernel, that we slide over the input. CNNs were first introduced for hand-written character recognition @lecun_backpropagation_1989 @le_cun_handwritten_1990 (1989, 1990), but were later revived for computer vision tasks @krizhevsky_imagenet_2012 (2012), after more computational power was available on modern devices to train them. Since the rise of CNNs in computer vision, most computer vision problems are solved with their help. The convolutions work by moving filter windows with learnable parameters (also called kernels) over the input @albawi_understanding_2017. Opposed to a fully connected network, the weights are shared over many of the nodes, because the same filters are applied over the full size of the input. CNNs have less parameters to train than a fully connected network with the same number of nodes, which makes them easier to train. They are generally expected to perform better than FC networks, especially on image related tasks. The filters can be 2-dimensional (2d), like for images (e.g. a 5x5 filter moved across the two axes of an image) or 1-dimensional (1d), which can e.g. be used to slide a kernel along the time dimension of a sensor recording. Even in the 1-dimensional case, less parameters are needed compared to the application of a fully connected network. Thus, the 1-dimensional CNN is expected to be easier to train and achieve a better performance. are neural networks that are not fully connected, but work by using convolutions with a kernel, that we slide over the input. CNNs were first introduced for hand-written character recognition @lecun_backpropagation_1989 @le_cun_handwritten_1990 (1989, 1990), but were later revived for computer vision tasks @krizhevsky_imagenet_2012 (2012), after more computational power was available on modern devices to train them. Since the rise of CNNs in computer vision, most computer vision problems are solved with their help. The convolutions work by moving filter windows with learnable parameters (also called kernels) over the input @albawi_understanding_2017. Opposed to a fully connected network, the weights are shared over many of the nodes, because the same filters are applied over the full size of the input. CNNs have less parameters to train than a fully connected network with the same number of nodes, which makes them easier to train. They are generally expected to perform better than FC networks, especially on image related tasks. The filters can be 2-dimensional (2d), like for images (e.g. a 5x5 filter moved across the two axes of an image) or 1-dimensional (1d), which can e.g. be used to slide a kernel along the time dimension of a sensor recording. Even in the 1-dimensional case, less parameters are needed compared to the application of a fully connected network. Thus, the 1-dimensional CNN is expected to be easier to train and achieve a better performance on time series sensor data than a fully connected network.
###### Recurrent neural networks (RNNs) ###### Recurrent neural networks (RNNs)
...@@ -52,14 +52,17 @@ The inputs to the cell are the external inputs $\mathbf{x}_t$ (from the previous ...@@ -52,14 +52,17 @@ The inputs to the cell are the external inputs $\mathbf{x}_t$ (from the previous
\caption*{($\odot$ marking element-wise multiplication)} \caption*{($\odot$ marking element-wise multiplication)}
\end{figure} \end{figure}
![LSTM Cell, by Guillaume Chevalier CC BY-SA 4.0, with added labeling for the gates)](img/LSTM_Cell.png){#fig:lstm_cell width=85%} \begin{minipage}{\textwidth}
The four LSTMs' gates are: The four LSTMs' gates are:
\begin{itemize}
\item forget gate
\item new memory gate
\item input gate
\item output gate
\end{itemize}
\end{minipage}
- forget gate ![LSTM Cell, by Guillaume Chevalier CC BY-SA 4.0, with added labeling for the gates)](img/LSTM_Cell.png){#fig:lstm_cell width=85%}
- new memory gate
- input gate
- output gate
These gates are fully connected neural network layers (marked in orange and with the corresponding activation functions in @fig:lstm_cell) with respective weights and biases and serve a functionality from which their names are derived. The weights and biases must be learned during the training phase of the neural network. The forget gate allows the LSTM to only apply part of the "remembered" cell memory $\mathbf{c}_{t-1}$ in the current step, i.e. which bits should be used to which extent with regard to the current new input data $\mathbf{x}_t$ and the hidden state from the last time step $\mathbf{h}_{t-1}$. The output of the forget gate, $\mathbf{f}_t$, multiplied element-wise with $\mathbf{c}_{t-1}$ is considered the "remembered" information from the last step. The new memory gate and the input gate are used to decide which new data is added to the cell state. These two layers are also given the previous step's hidden state $\mathbf{h}_{t-1}$ and the current step's input $\mathbf{x}_t$. In combination, the new memory network output $\tilde{\mathbf{c}}_t$ and the input gates' output $\mathbf{i}_t$ decide which components of the current input and hidden state will be taken into the new memory state $\mathbf{c}_{t}$. The memory state is passed on to the next step. The output gate will generate $\mathbf{o}_t$, which will be combined with $tanh(\mathbf{c}_{t})$ by element-wise matrix multiplication to form the new hidden state $\mathbf{h}_{t}$. These gates are fully connected neural network layers (marked in orange and with the corresponding activation functions in @fig:lstm_cell) with respective weights and biases and serve a functionality from which their names are derived. The weights and biases must be learned during the training phase of the neural network. The forget gate allows the LSTM to only apply part of the "remembered" cell memory $\mathbf{c}_{t-1}$ in the current step, i.e. which bits should be used to which extent with regard to the current new input data $\mathbf{x}_t$ and the hidden state from the last time step $\mathbf{h}_{t-1}$. The output of the forget gate, $\mathbf{f}_t$, multiplied element-wise with $\mathbf{c}_{t-1}$ is considered the "remembered" information from the last step. The new memory gate and the input gate are used to decide which new data is added to the cell state. These two layers are also given the previous step's hidden state $\mathbf{h}_{t-1}$ and the current step's input $\mathbf{x}_t$. In combination, the new memory network output $\tilde{\mathbf{c}}_t$ and the input gates' output $\mathbf{i}_t$ decide which components of the current input and hidden state will be taken into the new memory state $\mathbf{c}_{t}$. The memory state is passed on to the next step. The output gate will generate $\mathbf{o}_t$, which will be combined with $tanh(\mathbf{c}_{t})$ by element-wise matrix multiplication to form the new hidden state $\mathbf{h}_{t}$.
...@@ -90,7 +93,7 @@ Note that the calculation of $\alpha_t$ is done with the softmax function as sho ...@@ -90,7 +93,7 @@ Note that the calculation of $\alpha_t$ is done with the softmax function as sho
Zeng et al. evaluate their approach on 3 data sets and report a state-of-the-art performance, beating the initial DeepConvLSTM. Zeng et al. evaluate their approach on 3 data sets and report a state-of-the-art performance, beating the initial DeepConvLSTM.
\label{deepconvlstm_att} \label{deepconvlstm_att}
Another study by Singh et al. combines DeepConvLSTM with a self-attention mechanism @singh_deep_2021. The attention mechanism is very similar to the one used by Zeng et al. @zeng_understanding_2018, where the mechanism consists of a layer that follows the LSTM layers in the DeepConvLSTM network. Instead of utilizing a score layer which uses the relation of each $h_t$ to $h_T$, Singh et al. find the weights $\mathbf{\alpha}$ by applying the softmax function to the output of a fully connected layer through which they pass the concatenated $h_t$ values. Instead of considering only the relations of each $h_t$ to $h_T$ separately, they use one layer to jointly calculate all the attention weights. Other than that, the two attention mechanisms are similar. Singh et al. also report a statistically significant increase in performance compared to the initial DeepConvLSTM, although the evaluate their approach on different data sets than Zeng et al.. Another study by Singh et al. combines DeepConvLSTM with a self-attention mechanism @singh_deep_2021. The attention mechanism is very similar to the one used by Zeng et al. @zeng_understanding_2018, where the mechanism consists of a layer that follows the LSTM layers in the DeepConvLSTM network. Instead of utilizing a score layer which uses the relation of each $h_t$ to $h_T$, Singh et al. find the weights $\mathbf{\alpha}$ for the same weighted sum by applying the softmax function to the output of a fully connected layer through which they pass the concatenated $h_t$ values. Instead of considering only the relations of each $h_t$ to $h_T$ separately, they use one layer to jointly calculate all the attention weights. Other than that, the two attention mechanisms are similar. Singh et al. also report a statistically significant increase in performance compared to the initial DeepConvLSTM, although the evaluate their approach on different data sets than Zeng et al..
For HAR, DeepConvLSTM and the models derived from it are the state-of-the-art machine learning methods, as they consistently outperform other model architectures on the available benchmarks and data sets. For HAR, DeepConvLSTM and the models derived from it are the state-of-the-art machine learning methods, as they consistently outperform other model architectures on the available benchmarks and data sets.
...@@ -107,7 +110,7 @@ In order to separate hand washing from other activities, Mondol et al. employ a ...@@ -107,7 +110,7 @@ In order to separate hand washing from other activities, Mondol et al. employ a
![Steps of HAWAD for parameter estimation and inference, taken from @sayeed_mondol_hawad_2020](img/HAWAD_filter.png){width=98% #fig:HAWAD} ![Steps of HAWAD for parameter estimation and inference, taken from @sayeed_mondol_hawad_2020](img/HAWAD_filter.png){width=98% #fig:HAWAD}
They use the said features of all positive class samples to calculate the mean $\boldsymbol{\mu}$ and covariance matrix $\mathbf{S}$ of the feature distribution. Based on these measures, one can compute each sample's distance to the distribution using the Mahalanobis distance (as seen in equation \ref{eqn:mahala}). If during test time, the model predicts a sample to belong to the positive class, the distance is calculated. If the distance is bigger than a threshold ($d_{th}$), the sample is classified as a negative. The threshold $d_{th}$ can be derived by selecting it fittingly in order to include almost all positive samples seen during training. The parameter estimation and hand washing steps performed in the HAWAD paper can be seen in @fig:HAWAD. On their own data set (HAWAD data set) they reach F1-Scores of over 90% for hand washing detection. They use the said features of all positive class samples to calculate the estimated mean $\boldsymbol{\hat{\mu}}$ and covariance matrix $\mathbf{\hat{\sum}}$ of the feature distribution. Based on these measures, one can compute each sample's distance to the distribution using the Mahalanobis distance (as seen in equation \ref{eqn:mahala}). If during test time, the model predicts a sample to belong to the positive class, the distance is calculated. If the distance is bigger than a threshold ($d_{th}$), the sample is classified as a negative. The threshold $d_{th}$ can be derived by selecting it fittingly in order to include almost all positive samples seen during training. The parameter estimation and hand washing steps performed in the HAWAD paper can be seen in @fig:HAWAD. On their own data set (HAWAD data set) they reach F1-Scores of over 90% for hand washing detection.
\begin{figure} \begin{figure}
\begin{align} \begin{align}
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
\label{sec:results} \label{sec:results}
This chapter will report the evaluation results from both the theoretical evaluation and the practical evaluation. This chapter will report the evaluation results from both the theoretical evaluation and the practical evaluation.
## Theoretical Evaluation ## Theoretical evaluation
For the theoretical evaluation, we report the results separately, split by the tasks 1.-3. described in Section \ref{sec:classification_problems} For the theoretical evaluation, we report the results separately, split by the tasks 1.-3. described in Section \ref{sec:classification_problems}
In all tables of this chapter, the best values for a specific metric will be highlighted in bold font. In all tables of this chapter, the best values for a specific metric will be highlighted in bold font.
...@@ -27,7 +27,6 @@ The models running on normalized data also profit from the label smoothing, howe ...@@ -27,7 +27,6 @@ The models running on normalized data also profit from the label smoothing, howe
For the special case of the models initially trained on problem 3 which were then binarized and run on problem 1 (without smoothing), we only report some results in this section. The full results can be found in the appendix. Surprisingly, the models trained on problem 3 reach similar F1 scores on the test data of problem 1 as the models trained on problem 1. DeepConvLSTM achieves an F1 score of $0.857$, DeepConvLSTM-A achieves $0.847$. The F1 score of DeepConvLSTM is even higher than the highest F1 score of the models trained for problem 1 by $0.004$. However, for the S score metric, the models trained for problem 3 can only reach up to $0.704$ (CNN) or $0.671$ (DeepConvLSTM-A), which is lower by $0.052$ than the best performing model trained for problem 1. For the special case of the models initially trained on problem 3 which were then binarized and run on problem 1 (without smoothing), we only report some results in this section. The full results can be found in the appendix. Surprisingly, the models trained on problem 3 reach similar F1 scores on the test data of problem 1 as the models trained on problem 1. DeepConvLSTM achieves an F1 score of $0.857$, DeepConvLSTM-A achieves $0.847$. The F1 score of DeepConvLSTM is even higher than the highest F1 score of the models trained for problem 1 by $0.004$. However, for the S score metric, the models trained for problem 3 can only reach up to $0.704$ (CNN) or $0.671$ (DeepConvLSTM-A), which is lower by $0.052$ than the best performing model trained for problem 1.
\FloatBarrier
### Distinguishing compulsive hand washing from non-compulsive hand washing ### Distinguishing compulsive hand washing from non-compulsive hand washing
The results without smoothing of predictions for the second task, distinguishing compulsive hand washing from non-compulsive hand washing can be seen in table \ref{tbl:only_conv_hw}. In @fig:p2_metrics, the results with and without smoothing are shown. In terms of the F1 score metric, the LSTM model performs best ($0.926$). It is closely followed by DeepConvLSTM-A ($0.922$) and DeepConvLSTM ($0.918$). However, the RFC also performs surprisingly well, with an F1 score of $0.891$, even beating the CNN ($0.883$) and FC networks ($0.886$). Due to the imbalance of classes in the test set ($70.6\,\%$ of samples correspond to the positive class), the majority classifier reaches an F1 score of $0.828$. The S score is best for DeepConvLSTM ($0.869$) and LSTM ($0.862$), followed by LSTM-A ($0.848$) and DeepConvLSTM-A ($0.846$). The baseline methods RFC ($0.734$) and SVM ($0.701$) fail to reach similar S scores as the neural network based methods. The results without smoothing of predictions for the second task, distinguishing compulsive hand washing from non-compulsive hand washing can be seen in table \ref{tbl:only_conv_hw}. In @fig:p2_metrics, the results with and without smoothing are shown. In terms of the F1 score metric, the LSTM model performs best ($0.926$). It is closely followed by DeepConvLSTM-A ($0.922$) and DeepConvLSTM ($0.918$). However, the RFC also performs surprisingly well, with an F1 score of $0.891$, even beating the CNN ($0.883$) and FC networks ($0.886$). Due to the imbalance of classes in the test set ($70.6\,\%$ of samples correspond to the positive class), the majority classifier reaches an F1 score of $0.828$. The S score is best for DeepConvLSTM ($0.869$) and LSTM ($0.862$), followed by LSTM-A ($0.848$) and DeepConvLSTM-A ($0.846$). The baseline methods RFC ($0.734$) and SVM ($0.701$) fail to reach similar S scores as the neural network based methods.
...@@ -60,20 +59,36 @@ The confusion matrices of the non-normalized models in the right column do not a ...@@ -60,20 +59,36 @@ The confusion matrices of the non-normalized models in the right column do not a
As for problem 1 and for problem 2, we obtain the result, that normalization seems to decrease the performance of all the neural network based classifiers. For this problem, the FC network also has a decreased performance when normalized input data is used. As for problem 1 and for problem 2, we obtain the result, that normalization seems to decrease the performance of all the neural network based classifiers. For this problem, the FC network also has a decreased performance when normalized input data is used.
![Confusion matrices for all baseline classifiers with and without normalization of the sensor data](img/confusion_baselines.pdf){#fig:confusion_baselines width=98%} The respective confusion matrices for the baseline classifiers, i.e. RFC, SVM, majority classifier and random classifier are displayed in @fig:confusion_baselines. For both SVM and RFC, and both the normalized and the non-normalized versions thereof, the confusion matrices show, that the Null class was predicted most. In the non-normalized version, $94\,\%$ of samples belonging to the Null class are predicted correctly by both of these methods. However, they also predict most of the samples belonging to the other classes as Null. The HW class is detected to be Null in $71\,\%$ (SVM) and $67\,\%$ (RFC) of its samples, with only $15\,\%$ (SVM) and $22\,\%$ (RFC) being identified correctly. The accuracy is better for the HW-C class, where a ratio of correct predictions of $0.42$ (SVM) and $0.43$ (RFC) is reached, although there is again a high amount of misclassifications into the Null class (SVM: $0.56$, RFC: $0.54$).
The respective confusion matrices for the baseline classifiers, i.e. RFC, SVM, majority classifier and random classifier are displayed in @fig:confusion_baselines. For both SVM and RFC, and both the normalized and the non-normalized versions thereof, the confusion matrices show, that the Null class was predicted most. In the non-normalized version, $94\,\%$ of samples belonging to the Null class are predicted correctly by both of these methods. However, they also predict most of the samples belonging to the other classes as Null. The HW class is detected to be Null in $71\,\%$ (SVM) and $67\,\%$ of its samples, with only $15\,\%$ (SVM) and $22\,\%$ (RFC) being identified correctly. The accuracy is better for the HW-C class, where a ratio of correct predictions of $0.42$ (SVM) and $0.43$ (RFC) is reached, although there is again a high amount of misclassifications into the Null class (SVM: $0.56$, RFC: $0.54$).
The majority classifier classifies all the samples into the Null class, which leads to its accuracy on the samples belonging to the Null class being $1.0$ and $0.0$ for all other samples belonging to the HW and HW-C classes. The majority classifier classifies all the samples into the Null class, which leads to its accuracy on the samples belonging to the Null class being $1.0$ and $0.0$ for all other samples belonging to the HW and HW-C classes.
The random classifier also does not perform well and reaches values around $0.33$ for each of the fields of the confusion matrix. The random classifier also does not perform well and reaches values around $0.33$ for each of the fields of the confusion matrix.
\newpage
\begin{figure}[H]
\centering
\includegraphics[width=0.98\textwidth]{img/confusion_baselines.pdf}
\caption{Confusion matrices for all baseline classifiers with and without normalization of the sensor data}
\label{fig:confusion_baselines}
\end{figure}
![Confusion matrix of the chained best classifiers for problem 1 (DeepConvLSTM-A) and problem 2 (DeepConvLSTM), applied to problem 3](img/chained_confusion.pdf){#fig:chained_confusion width=60%}
#### Chained model #### Chained model
The best classifier in terms of the S score for problem 1 was DeepConvLSTM-A. Likewise, for problem 2 it was the DeepConvLSTM. Thus, these two classes were selected to run in a chained model as described in section \ref{chained_model}. The results of this chained run of 2 models on the testing data for problem 3 can be seen in @fig:chained_confusion. The results were achieved on non-normalized data. The main difference of the results of the chained models compared with the best performing original models for problem 3 is that the ratio of correct predictions for the Null class by the chained model is significantly higher ($0.69$ instead of around $0.5$). In return, its ratio of correct predictions for the HW class is only $0.67$ instead of around $0.79$ for the DeepConvLSTM-based models, and $0.8$ instead of around $0.85$ for the HW-C class. The best classifier in terms of the S score for problem 1 was DeepConvLSTM-A. Likewise, for problem 2 it was the DeepConvLSTM. Thus, these two classes were selected to run in a chained model as described in section \ref{chained_model}. The results of this chained run of 2 models on the testing data for problem 3 can be seen in @fig:chained_confusion. The results were achieved on non-normalized data. The main difference of the results of the chained models compared with the best performing original models for problem 3 is that the ratio of correct predictions for the Null class by the chained model is significantly higher ($0.69$ instead of around $0.5$). In return, its ratio of correct predictions for the HW class is only $0.67$ instead of around $0.79$ for the DeepConvLSTM-based models, and $0.8$ instead of around $0.85$ for the HW-C class.
\begin{figure}[H]
\centering
\includegraphics[width=0.6\textwidth]{img/chained_confusion.pdf}
\caption{Confusion matrix of the chained best classifiers for problem 1 (DeepConvLSTM-A) and problem 2 (DeepConvLSTM), applied to problem 3}
\label{fig:chained_confusion}
\end{figure}
\input{tables/separate.tex} \input{tables/separate.tex}
Aside from showing and discussing the confusion matrices, we also report the multiclass F1 score, the multiclass S score, and the mean diagonal value of the confusion matrices. These results can be seen in table Aside from showing and discussing the confusion matrices, we also report the multiclass F1 score, the multiclass S score, and the mean diagonal value of the confusion matrices. These results can be seen in table
...@@ -88,14 +103,13 @@ The mean diagonal value of the confusion matrix upholds almost the same ordering ...@@ -88,14 +103,13 @@ The mean diagonal value of the confusion matrix upholds almost the same ordering
\FloatBarrier \FloatBarrier
\newpage
## Practical Evaluation ## Practical evaluation
### Scenario 1: One day of evaluation ### Scenario 1: One day of evaluation
In the first scenario, the 5 subjects reported an average of $4.75$ hand washing procedures on the day on which they evaluated the system. In the first scenario, the 5 subjects reported an average of $4.75$ ($\pm\,3.3$) hand washing procedures on the day on which they evaluated the system. Out of those, $1.75$ ($\pm\,2.06\,\%$) were correctly identified. The accuracy per subject was $28.33\,\%$ ($\pm\,37.9\,\%$). The highest accuracy for a subject was $80\,\%$ out of 5 hand washes, the lowest was $0\,\%$ out of 4 hand washes. Of all hand washing procedures conducted over the day by the subjects, $35.8\,\%$ were detected correctly.
Per subject, there were $4.75$ ($\pm\,3.3$) hand washing procedures. Out of those, $1.75$ ($\pm\,2.06\,\%$) were correctly identified. The accuracy per subject was $28.33\,\%$ ($\pm\,37.9\,\%$). The highest accuracy for a subject was $80\,\%$ out of 5 hand washes, the lowest was $0\,\%$ out of 4 hand washes. Of all hand washing procedures conducted over the day by the subjects, $35.8\,\%$ were detected correctly.
Some subjects wore the smart watch on the right wrist instead of the left wrist and reported worse results for that. Leaving out hand washes conducted with the smart watch worn on the right wrist, the detection sensitivity rises to $50\,\%$. Two subjects wore the smart watch on the right wrist instead of the left wrist and reported worse results for that. Leaving out hand washes conducted with the smart watch worn on the right wrist, the detection sensitivity rises to $50\,\%$.
The duration and intensity of the hand washing process also played a role. The duration and intensity of the hand washing process also played a role.
The correlation of duration of the hand washing with the detection rate is $-0.039$. However, the raw data does only contain 2 "longer" hand washes over 30 seconds, the rest being in the range of 10 to 25 seconds. The correlation of duration of the hand washing with the detection rate is $-0.039$. However, the raw data does only contain 2 "longer" hand washes over 30 seconds, the rest being in the range of 10 to 25 seconds.
...@@ -119,5 +133,5 @@ Some subjects also reported difficulties with the smart watch application (not p ...@@ -119,5 +133,5 @@ Some subjects also reported difficulties with the smart watch application (not p
### Scenario 2: Controlled intensive hand washing ### Scenario 2: Controlled intensive hand washing
In scenario 2, the subjects each washed their hands at least 3 times. Some subjects voluntarily agreed to perform more repetitions, which leads to more than 3 washing detection results per subject. The detection accuracy per subject was $76\,\%$ ($\pm\,25\,\%$), with the highest being $100\,\%$ and the lowest being $50\,\%$. In scenario 2, the subjects each washed their hands at least 3 times. Some subjects voluntarily agreed to perform more repetitions, which leads to more than 3 washing detection results per subject. The detection accuracy per subject was $76\,\%$ ($\pm\,25\,\%$), with the highest being $100\,\%$ and the lowest being $50\,\%$.
The mean accuracy over all repetitions and not split by subjects was $73.7\,\%$. For scenario 2, one user moved the smart watch from the right wrist to the left wrist after two repetitions. The first two repetitions were not detected, while the two repetitions with the smart watch worn on the right wrist were detected correctly. Leaving out hand washes conducted with the smart watch worn on the right wrist, the detection sensitivity rises to $78.6\,\%$, and the detection accuracy per subject is $82.5\,\%$ ($\pm\,23.6\,\%$). The mean accuracy over all repetitions and not split by subjects was $73.7\,\%$. For scenario 2, one user moved the smart watch from the right wrist to the left wrist after two repetitions. The first two repetitions were not detected, while the two repetitions with the smart watch worn on the left wrist were detected correctly. If we leave out hand washes conducted with the smart watch worn on the right wrist, the detection sensitivity rises to $78.6\,\%$, and the detection accuracy per subject is $82.5\,\%$ ($\pm\,23.6\,\%$).
No preview for this file type
...@@ -9,6 +9,13 @@ ...@@ -9,6 +9,13 @@
\usepackage{multirow} \usepackage{multirow}
\usepackage[section]{placeins} \usepackage[section]{placeins}
% hyphenation
\tolerance=9999
\emergencystretch=10pt
\hyphenpenalty=10000
\exhyphenpenalty=100
% Hurenkinder und Schusterjungen verhindern % Hurenkinder und Schusterjungen verhindern
\clubpenalties=4 10000 10000 10000 0 \clubpenalties=4 10000 10000 10000 0
\widowpenalties=4 10000 10000 10000 0 \widowpenalties=4 10000 10000 10000 0
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment