Commit 1e3db523 authored by Alexander Henkel's avatar Alexander Henkel
Browse files

feedback impl

parent 660c9d2a
......@@ -880,4 +880,31 @@ DOI = {10.3390/s16010115}
publisher = {Springer},
address = {Berlin, Germany},
doi = {10.1007/978-3-540-85563-7_14}
}
@article{Xiong2016Nov,
author = {Xiong, Peng and Wang, Hongrui and Liu, Ming and Lin, Feng and Hou, Zengguang and Liu, Xiuling},
title = {{A stacked contractive denoising auto-encoder for ECG signal denoising}},
journal = {Physiol. Meas.},
volume = {37},
number = {12},
pages = {2214--2230},
year = {2016},
month = nov,
issn = {0967-3334},
publisher = {IOP Publishing},
doi = {10.1088/0967-3334/37/12/2214}
}
@article{Xiong2016Jun,
author = {Xiong, Peng and Wang, Hongrui and Liu, Ming and Zhou, Suiping and Hou, Zengguang and Liu, Xiuling},
title = {{ECG signal enhancement based on improved denoising auto-encoder}},
journal = {Eng. Appl. Artif. Intell.},
volume = {52},
pages = {194--202},
year = {2016},
month = jun,
issn = {0952-1976},
publisher = {Pergamon},
doi = {10.1016/j.engappai.2016.02.015}
}
\ No newline at end of file
\chapter*{Abstract}
Wearable sensors like smartwatches offer a good opportunity for human activity recognition (HAR). They are available to a wide user base and can be used in everyday life. Due to the variety of users, the detection model must be able to recognize different movement patterns. Recent research has demonstrated, that a personalized recognition tends to perform better than a general one. However, additional labeled data from the user is required which can be time consuming and labor intensive. While common personalization approaches try to reduce the necessary labeled training data, the labeling process remains dependent on some user interaction.
Wearable sensors like smartwatches offer a good opportunity for human activity recognition (HAR). They are available to a wide user base and can be used in everyday life. Due to the variety of users, the detection model must be able to recognize different movement patterns. Recent research has demonstrated that a personalized recognition tends to perform better than a general one. However, additional labeled data from the user is required which can be time consuming and labor intensive. While common personalization approaches try to reduce the necessary labeled training data, the labeling process remains dependent on some user interaction.
In this work, I present a personalization approach in which training data labels are derived from inexplicit user feedback obtained during the usual use of a HAR application. The general model predicts labels which are then refined by various denoising filters based on Convolutional Neural Networks and Autoencoders. This process is assisted by the previously obtained user feedback. High confidence data is then used for fine tuning the recognition model via transfer learning. No changes to the model architecture are required and thus personalization can easily be added to an existing application.
Analysis in the context of hand wash detection demonstrate, that a significant performance increase can be achieved. More over I compare my approach with a traditional personalization method to confirm the robustness. Finally I evaluate the process in a real world experiment where participants wore a smart watch on a daily basis for a month.
Analysis in the context of hand wash detection demonstrates, that a significant performance increase can be achieved. More over, I compare my approach with a traditional personalization method to confirm the robustness. Finally I evaluate the process in a real world experiment where participants wear a smart watch on a daily basis for a month.
\chapter{Zusammenfassung}
......
\chapter{Introduction}\label{chap:introduction}
Detecting and monitoring people's activities can be the basis for observing user behavior and well-being. Human Activity Recognition (HAR) is a growing research area in many fields like healthcare~\cite{Zhou2020Apr, Wang2019Dec}, elder care~\cite{Jalal2014Jul, Hong2008Dec}, fitness tracking~\cite{Nadeem2020Oct} or entertainment~\cite{Lara2012Nov}. Especially the technical improvements in wearable sensors like smart watches offer an integration in everyday life over a wide user base~\cite{Weiss2016Feb, Jobanputra2019Jan, Bulling2014Jan}.
Detecting and monitoring peoples activities can be the basis for observing user behavior and well-being. Human Activity Recognition (HAR) is a growing research area in many fields like healthcare~\cite{Zhou2020Apr, Wang2019Dec}, elder care~\cite{Jalal2014Jul, Hong2008Dec}, fitness tracking~\cite{Nadeem2020Oct} or entertainment~\cite{Lara2012Nov}. Especially the technical improvements in wearable sensors like smart watches offer an integration in everyday life over a wide user base~\cite{Weiss2016Feb, Jobanputra2019Jan, Bulling2014Jan}.
One of the application scenarios in healthcare can be the observation of various diseases such as Obsessive-Compulsive Disorder (OCD). For example the detection of hand washing activities can be used to derive the frequency or excessiveness which occurs in some people with OCD. More over it is possible to diagnose and even treat such diseases outside a clinical setting~\cite{Ferreri2019Dec, Briffault2018May}. If excessive hand washing is detected Just-in-Time Interventions can be presented to the user which offer an enormous potential for promoting health behavior change~\cite{10.1007/s12160-016-9830-8}.
One of the application scenarios in healthcare is the observation of various diseases such as Obsessive-Compulsive Disorder (OCD). For example the detection of hand washing activities can be used to derive the frequency or excessiveness which occurs in some people with OCD. More over it is possible to diagnose and even treat such diseases outside a clinical setting~\cite{Ferreri2019Dec, Briffault2018May}. If excessive hand washing is detected Just-in-Time Interventions can be presented to the user which offer an enormous potential for promoting health behavior change~\cite{10.1007/s12160-016-9830-8}.
State of the art Human Activiy Recognition methods are supervised deep neural networks derived from concepts like Convolutional Layers or LSTMs. These require lots of training data to archive good performance. Since movement patterns of each human are unique, the performance of activity detection can differ. So training data of a wide variety of humans would be necessary to generalize to new users. Therefore it has been shown that personalized models can achieve better accuracy against user-independent models ~\cite{Hossain2019Jul, Lin2020Mar}.
State of the art Human Activiy Recognition methods are supervised deep neural networks derived from concepts like Convolutional Layers or Long short-term memory (LSTM). These require lots of training data to achieve good performance. Since movement patterns of each human are unique, the performance of activity detection can differ. So training data of a wide variety of humans is necessary to generalize to new users. Therefore it has been shown that personalized models can achieve better accuracy against user-independent models ~\cite{Hossain2019Jul, Lin2020Mar}.
To personalize a model retraining on new unseen sensor data is necessary. Obtaining the ground truth labels is crucial for most deep learning techniques. However, the annotation process is time and cost-intensive. Typically training data is labeled in controlled environments by hand. In a real context scenario the user would have to take over the main part.
To personalize a model retraining on new unseen sensor data is necessary. Obtaining the ground truth labels is crucial for most deep learning techniques. However, the annotation process is time and cost-intensive. Typically, training data is labeled in controlled environments by hand. In a real context scenario the user would have to take over the main part.
Indeed this requires lots of user interaction and a decent expertise which would contradict the usability.
There has been different research in how to preprocess data to make it usable for training. It turned out that a good trade-off is semi-supervised-learning or active learning, where a general base model is used to label the data and in uncertain cases it relies on user interaction ~\cite{siirtola2019importance, Siirtola2019Nov}. Here a small part of labeled data is combined with a larger unlabeled part to improve the detection model. But still some sort of explicit user interaction is required for personalization. So there is a overhead in the usage of a HAR application.
The goal of my work was to personalize a detection model without increasing the user interaction. Information for labeling is drawn from indicators that arise during the use of the application. These can be derived by user feedback to triggered actions resulted from the predictions of the underlying recognition model. More over the personalization should be an additional and separated part, so no change of the model architecture is required.
The goal of my work is to personalize a detection model without increasing the user interaction. Information for labeling is drawn from indicators that arise during the use of the application. These can be derived by user feedback to triggered actions resulted from the predictions of the underlying recognition model. More over the personalization should be an additional and separated part, so no change of the model architecture is required.
At first, all new unseen sensor data is labeled by the same general model which is used for activity recognition. These model predictions are corrected to a certain extent by using pretrained filters. High confidence labels are considered for personalization. In addition, the previously obtained indicators are used to further refine the data to generate a valid training set. Therefore the process of manual labeling can be skipped and replaced by an automatic combination of available indications. With the newly collected and labeled training data the previous model can be fine tuned in a incremental learning approach ~\cite{Amrani2021Jan, Siirtola2019May, Sztyler2017Mar}. For neuronal networks it has been shown that transfer learning offers high performance with decent computation time ~\cite{Chen2020Apr}. All this leads to a personalized model which has improved performance in detecting specific gestures of an individual user.
At first, all new unseen sensor data is labeled by the same general model which is used for activity recognition. These model predictions are corrected to a certain extent by using pretrained filters. High confidence labels are considered for personalization. In addition, the previously obtained indicators are used to further refine the data to generate a valid training set. Therefore the process of manual labeling can be skipped and replaced by an automatic combination of available indications. With the newly collected and labeled training data the previous model can be fine tuned in a incremental learning approach ~\cite{Amrani2021Jan, Siirtola2019May, Sztyler2017Mar}. For neuronal networks it has been shown that transfer learning offers high performance with decent computation time ~\cite{Chen2020Apr}. In combination this leads to a personalized model which has improved performance in detecting specific gestures of an individual user.
I applied the described personalization process to a hand washing detection application which is used for observing the behavior of OCD patients. During the observation, the user answers requested evaluations if the application detects hand washing. For miss predictions the user has the opportunity to reject evaluations. Depending on how the user reacted to the evaluations, conclusions can be drawn about the correctness of the predictions, which leads to the required indicators.
I applied the described personalization process to a hand washing detection application which is used for observing the behavior of OCD patients. During the observation, the user answers requested evaluations if the application detects hand washing. For miss predictions the user has the opportunity to reject evaluations. Depending on how the user reacts to the evaluations, conclusions are drawn about the correctness of the predictions, which leads to the required indicators.
The contributions of my work are as follows:
......
This diff is collapsed.
This diff is collapsed.
......@@ -83,7 +83,7 @@ First we observe how the model performance evolves over the iteration steps. \fi
\input{figures/experiments/supervised_evolution_single}
\subsubsection{Comparison of filter configurations}
\subsubsection{Comparison of filter configurations}\label{sec:expEvolCompFilter}
In this step I compare the evaluation of the personalized model over the different filter configurations. Additionally I apply different number of epochs and split the regularization methods. The models are trained with 50, 100, and 150 epochs. \figref{fig:evolutionAll} shows their S scores. In (a) freezing the feature layers and for (b) in l2-sp penalty is used for regularization. The personalizations trained with freezed layers show all a similar increasing trend in performance. With more epochs they seem to achieve higher S values. Especially in the first iteration, all personalized models trained with 150 epochs already outperformed the general model. With l2-sp regularization the performance varies heavily. For each selection of epochs the personalization lead to different results. It could be possible, that a better performing model is trained, but no exact statement can be made.
\input{figures/experiments/supervised_evolution_all}
......
\begin{figure}[t]
\begin{centering}
\subfloat[manual hand washing]
{\includegraphics[width=0.28\textwidth]{figures/approach/base_application_screen_manual.png}}
\qquad
\subfloat[detected hand washing]
{\includegraphics[width=0.28\textwidth]{figures/approach/base_application_screen_yes_no.png}}
\qquad
\subfloat[evaluation]
{\includegraphics[width=0.28\textwidth]{figures/approach/base_application_screen_eval.png}}
\caption[Base application screen shots]{\textbf{Base application screen shots.} (a) shows the apllication in default state, where the user has the opportunity to trigger a hand wash event manually. (b) shows the notification, which appears, when the application has detected a hand wash activity. Here the user can confirm or decline. (b) shows one of the evaluation queries which the user has to answer for the OCD observation. These are shown, if the user triggered a manual hand wash event or confirmed a detected hand washing activity.}
\label{fig:baseApplicationScreen}
\end{centering}
\end{figure}
......@@ -21,12 +21,12 @@
% Change to your first examiner
% The ~ enables non sentence spacing after a period
\newcommand{\firstexaminer}{Prof.~Dr.~Bugs Bunny}
\newcommand{\firstexaminer}{Dr.~Philipp Scholl}
% Change to your second examiner, some undergraduate studies don't have a second examiner
% in this case just comment out the following line
\newcommand{\secondexaminer}{Prof.~Dr.~Wile E. Coyote}
\newcommand{\secondexaminer}{Prof.~Marco Zimmerling}
% Change to your adivers
\newcommand{\advisers}{Terence Hill, Bud Spencer}
\newcommand{\advisers}{Dr.~Philipp Scholl}
% include all packages and define commands in setup.tex
\input{setup}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment