Commit f5552682 authored by Alexander Henkel's avatar Alexander Henkel
Browse files

work on experiments

parent 5d22fdd2
\begin{figure}[htbp]
\begin{centering}
\makebox[\textwidth]{\includegraphics[width=0.8\paperwidth]{figures/approach/PersonalizationImplementation.pdf}}
\makebox[\textwidth]{\includegraphics[width=\textwidth]{figures/approach/PersonalizationImplementation.pdf}}
\caption[Personalization implementation]{\textbf{Personalization implementation} The personalization process as it is implemented on the server. Red highlighted entries depict new recordings which haven't been used before and the new personalization entry based on these. Green highlighted is the personalization model which performs best.}
\label{fig:personalizationImplementation}
\end{centering}
......
\begin{figure}[t]
\begin{centering}
\makebox[\textwidth]{\includegraphics[width=0.8\paperwidth]{figures/approach/personalization_pipeline.png}}
\makebox[\textwidth]{\includegraphics[width=\textwidth]{figures/approach/personalization_pipeline.png}}
\caption[Personalization interface]{\textbf{Personalization interface} Screenshot of the interface of the personalization implementation for the user 'Participant3'. Each box represents one personalization run. The test sets have been selected by hand in advance. First run does not include new training recordings for personalization, so just potential best running mean settings for the general model are computed. Upper left number is the ID of a run and the blue below gives the ID of the model on which the personalization depends. A green ID represents the best performing model which would transmitted to the user.}
\label{fig:personalizationPipeline}
\end{centering}
......
......@@ -3,7 +3,7 @@
\subfloat[evaluation]
{\includegraphics[width=\textwidth]{figures/experiments/supervised_pseudo_models_training_data_evaluation.png}}
\subfloat[training data]
\subfloat[confusion matrix]
{\includegraphics[width=\textwidth]{figures/experiments/supervised_pseudo_models_training_data_confusion.png}}
......
\begin{table}[ht]
\centering
% spacing in table
\ra{1.3}
\begin{tabular}{lrlrrrr}
\toprule
{} & epochs & weighting & B & s & h & S score \\
\midrule
0 & 50 & True & 20 & 10 & 0.000 & 0.845124 \\
1 & 50 & True & 20 & 10 & 0.001 & 0.844993 \\
2 & 50 & False & 20 & 5 & 0.000 & 0.844671 \\
3 & 50 & False & 50 & 5 & 0.001 & 0.844527 \\
4 & 50 & False & 20 & 10 & 0.000 & 0.844508 \\
5 & 50 & False & 20 & 10 & 0.001 & 0.844502 \\
6 & 50 & False & 20 & 5 & 0.001 & 0.844449 \\
7 & 50 & True & 30 & 5 & 0.000 & 0.844443 \\
8 & 50 & True & 50 & 5 & 0.000 & 0.844419 \\
\bottomrule
\end{tabular}
\caption[Active learning evaluation]{\textbf{Active learning evaluation} Best hyper parameter settings for active learning according the S score of resulting model.}
\label{tab:activeLearningEvaluation}
\end{table}
\begin{table}[ht]
\centering
% spacing in table
\ra{1.3}
\begin{tabular}{|l|ccc|}
%\toprule
\hline
{model} & estimated F1 & ground truth F1 & variation \\
%\midrule
\hline
general & 0.4785 & 0.4127 & 15\% \\
personalized & 0.5702 & 0.6205 & 8\% \\
\hline
%\bottomrule
\end{tabular}
\caption[Quality estimation evaluation]{\textbf{Quality estimation evaluation.} Comparison of estimated F1 and ground truth F1 score}
\label{tab:qualityEstimationEvaluation}
\end{table}
\begin{table}[ht]
\centering
% spacing in table
\ra{1.3}
\subfloat[Training data]
{\resizebox{\textwidth}{!}{%
\begin{tabular}{lrrrrrrr}
%\toprule
\thead{participant} & \thead{recordings} & \thead{samples} & \thead{hours} & \thead{manual\\indicator} & \thead{correct\\indicator} & \thead{false\\indicator} & \thead{neutral\\indicator} \\
\midrule
OCDetect\_02 & 13 & 19398547 & 107 & 7 & 81 & 65 & 112 \\
OCDetect\_03 & 7 & 19333998 & 107 & 36 & 77 & 12 & 27 \\
OCDetect\_04 & 15 & 30371163 & 168 & 38 & 56 & 0 & 39 \\
OCDetect\_05 & 27 & 45313733 & 251 & 105 & 243 & 17 & 949 \\
OCDetect\_07 & 11 & 13178338 & 73 & 11 & 23 & 4 & 7 \\
OCDetect\_09 & 11 & 39808808 & 221 & 42 & 43 & 7 & 77 \\
OCDetect\_10 & 10 & 8387805 & 46 & 1 & 8 & 0 & 119 \\
OCDetect\_11 & 16 & 40522845 & 225 & 45 & 56 & 38 & 26 \\
OCDetect\_12 & 13 & 8299920 & 46 & 72 & 73 & 0 & 5 \\
OCDetect\_13 & 15 & 33018908 & 183 & 21 & 42 & 14 & 39 \\
\bottomrule
\end{tabular}}}
\subfloat[Test data]
{\resizebox{\textwidth}{!}{%
\begin{tabular}{lrrrrrrr}
%\toprule
\thead{participant} & \thead{recordings} & \thead{samples} & \thead{hours} & \thead{manual\\indicator} & \thead{correct\\indicator} & \thead{false\\indicator} & \thead{neutral\\indicator} \\
\midrule
OCDetect\_02 & 6 & 10861527 & 60 & 9 & 52 & 41 & 71 \\
OCDetect\_03 & 8 & 21251292 & 118 & 42 & 87 & 11 & 65 \\
OCDetect\_04 & 5 & 10668781 & 59 & 18 & 33 & 0 & 22 \\
OCDetect\_05 & 10 & 19345852 & 107 & 52 & 117 & 6 & 538 \\
OCDetect\_07 & 4 & 4813866 & 26 & 4 & 15 & 7 & 9 \\
OCDetect\_09 & 4 & 13724780 & 76 & 10 & 20 & 14 & 92 \\
OCDetect\_10 & 2 & 2243083 & 12 & 1 & 9 & 0 & 193 \\
OCDetect\_11 & 5 & 15818377 & 87 & 25 & 35 & 14 & 20 \\
OCDetect\_12 & 5 & 6502526 & 36 & 76 & 76 & 0 & 1 \\
OCDetect\_13 & 6 & 16679159 & 92 & 11 & 30 & 15 & 37 \\
\bottomrule
\end{tabular}}}
\caption[Real world dataset]{\textbf{Real world dataset} Overview of used datasets for real world experiments. Recordings for test split have been selected by hand. They have been chosen because they cover a wider variety of user feedback.}
\label{tab:realWorldDataset}
\end{table}
\begin{table}[ht]
\centering
% spacing in table
\ra{1.3}
\begin{adjustbox}{angle=90}
\resizebox{0.9\textheight}{!}{%
\begin{tabular}{llllrrrrrrrr}
\toprule
\thead{participant} & \thead{filter\\ configuration} & \thead{base\\ on\\ best} & \thead{l2-sp} & \thead{\rotatebox[origin=c]{-90}{iterations}} & \thead{false\\ diff\\ relative} & \thead{correct\\ diff\\ relative} & \thead{F1} & \thead{base\\ false\\ diff\\ relative} & \thead{base\\ correct\\ diff\\ relative} & \thead{base\\F1} & \thead{F1\\diff}\\
\midrule
OCDetect\_02 & all\_cnn\_convlstm3\_hard & True & True & 2 & -0.3600 & 0.1636 & 0.5246 & -0.2057 & 0.1455 & 0.4667 & 0.0579 \\
OCDetect\_03 & all\_cnn\_convlstm2\_hard & True & True & 1 & -0.3880 & 0.0000 & 0.4800 & -0.2240 & 0.1212 & 0.4728 & 0.0072 \\
OCDetect\_04 & all\_cnn\_convlstm3\_hard & True & True & 1 & -0.3889 & 0.1176 & 0.5507 & -0.1111 & 0.1176 & 0.5135 & 0.0372 \\
OCDetect\_05 & all\_cnn\_convlstm3\_hard & True & True & 2 & -0.1336 & 0.2111 & 0.3664 & -0.1270 & 0.1556 & 0.3514 & 0.0150 \\
OCDetect\_07 & all\_cnn\_convlstm3\_hard & True & False & 2 & -0.6429 & 0.0769 & 0.8000 & -0.5714 & 0.0000 & 0.7429 & 0.0571 \\
OCDetect\_09 & all\_cnn\_convlstm3\_hard & True & False & 0 & -0.5273 & 0.1000 & 0.4000 & -0.5273 & 0.1000 & 0.4000 & 0.0000 \\
OCDetect\_10 & all\_cnn\_convlstm2\_hard & True & True & 2 & -0.8209 & 0.1429 & 0.3265 & -0.6418 & 0.1429 & 0.2192 & 0.1074 \\
OCDetect\_11 & all\_cnn\_convlstm2\_hard & True & True & 1 & -0.1333 & 0.2857 & 0.3600 & -0.4000 & 0.1429 & 0.3556 & 0.0044 \\
OCDetect\_12 & all\_cnn\_convlstm3\_hard & True & False & 2 & 0.0625 & 0.4516 & 0.6618 & -0.3125 & 0.1613 & 0.5950 & 0.0667 \\
OCDetect\_13 & all\_null\_convlstm3 & True & False & 0 & -0.4600 & 0.0000 & 0.4471 & -0.4600 & 0.0000 & 0.4471 & 0.0000 \\
\bottomrule
\end{tabular}}
\end{adjustbox}
\caption[Real world evaluation]{\textbf{Real world evaluation} Summary of quality estimation over multiple participants. The personalization with highest F1 score per participant is shown.}
\label{tab:realWorldEvaluation}
\end{table}
......@@ -2,22 +2,20 @@
\centering
% spacing in table
\ra{1.3}
\begin{tabular}{|l|rrr|rrr|}
%\toprule
\resizebox{\textwidth}{!}{%
\begin{tabular}{|l|rrrrrr|rrrrrr|}
\hline
{} & \multicolumn{3}{c|}{train split} & \multicolumn{3}{c|}{test split} \\
{} & recordings & null labels & hw labels & recordings & null labels & hw labels \\
%\midrule
{} & \multicolumn{6}{l|}{train split} & \multicolumn{6}{l|}{test split} \\
{} & \thead{\#} & \thead{null\\labels} & \thead{hw\\ labels} & \thead{manual\\ indicators} & \thead{correct\\ indicators} & \thead{false\\indicators} & \thead{\#} & \thead{null\\ labels} & \thead{hw\\ labels} & \thead{manual\\indicators} & \thead{correct\\ indicators} & \thead{false\\ indicators} \\
\hline
synthetic\_01 & 4 & 61712 & 642 & 2 & 28368 & 376 \\
synthetic\_02 & 4 & 61819 & 631 & 2 & 28120 & 382 \\
synthetic\_10 & 3 & 37585 & 322 & 1 & 8976 & 138 \\
synthetic\_01 & 4 & 642 & 61712 & 3 & 13 & 84 & 2 & 376 & 28368 & 1 & 7 & 81 \\
synthetic\_02 & 4 & 631 & 61819 & 7 & 8 & 76 & 2 & 382 & 28120 & 2 & 7 & 29 \\
synthetic\_10 & 3 & 322 & 37585 & 2 & 6 & 69 & 1 & 138 & 8976 & 0 & 3 & 16 \\
recorded\_01 & 5 & 209 & 40577 & 3 & 12 & 26 & 4 & 169 & 74003 & 11 & 7 & 21 \\
recorded\_02 & 5 & 254 & 68898 & 2 & 14 & 12 & 3 & 281 & 66794 & 6 & 9 & 8 \\
\hline
recorded\_01 & 5 & 40577 & 209 & 4 & 74003 & 169 \\
recorded\_02 & 5 & 68898 & 254 & 3 & 66794 & 281 \\
\hline
%\bottomrule
\end{tabular}
\end{tabular}}
......
......@@ -85,6 +85,8 @@
\usepackage{dsfont}
\usepackage{svg}
\usepackage{pdfpages}
\usepackage{makecell}
\usepackage{adjustbox}
%------------------------------------------------------------------------------
% (re)new commands / settings
......@@ -172,4 +174,10 @@
\newcommand{\draft}[1]{\textbf{\textcolor{NavyBlue}{(DRAFT: #1)}}\refstepcounter{drafts}\label{draft \thedrafts}}
% microtype with lmodern, see https://tex.stackexchange.com/questions/75305/microtype-warning-with-lmodern-package-and-koma-script
%\DeclareMicrotypeAlias{lmss}{cmr}
\ No newline at end of file
%\DeclareMicrotypeAlias{lmss}{cmr}
%-------------------- table commands ---------------------
\renewcommand\theadalign{bc}
\renewcommand\theadfont{\bfseries}
\renewcommand\theadgape{\Gape[4pt]}
\renewcommand\cellgape{\Gape[4pt]}
This diff is collapsed.
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment