Commit 4fec348f authored by Alexander Henkel's avatar Alexander Henkel
Browse files

work on pseudo filters

parent 9cc4aa18
...@@ -42,7 +42,11 @@ In the next step I want to observe if smoothing could have a negative effect if ...@@ -42,7 +42,11 @@ In the next step I want to observe if smoothing could have a negative effect if
\section{Evaluation of different Pseudo label generations} \section{Evaluation of different Pseudo label generations}
In this section, I describe the evaluation of different pseudo labeling approaches using the filters introduced in \secref{sec:approachFilterConfigurations}. For each filter configuration, the base model is used to predict the labels of the training sets and create pseudo labels. After that the filter is applied to the pseudo labels. To determine the quality of the pseudo labels, they are evaluated against the ground truth values using soft versions of the metrics $Sensitivity^{soft}$, $Specificity^{soft}$, $F_1^{soft}$, $S^{soft}$. The general model is then trained by the refined pseudo labels. All resulted models are evaluated by their test sets and the mean over all is computed. In this section, I describe the evaluation of different pseudo labeling approaches using the filters introduced in \secref{sec:approachFilterConfigurations}. For each filter configuration, the base model is used to predict the labels of the training sets and create pseudo labels. After that the filter is applied to the pseudo labels. To determine the quality of the pseudo labels, they are evaluated against the ground truth values using soft versions of the metrics $Sensitivity^{soft}$, $Specificity^{soft}$, $F_1^{soft}$, $S^{soft}$. The general model is then trained by the refined pseudo labels. All resulted models are evaluated by their test sets and the mean over all is computed. \figref{fig:pseudoModelsEvaluation} shows a bar plot over the metrics for all filter configuration. I concentrate to the values of S score. The configurations \texttt{all}, \texttt{high\_conf}, \texttt{scope}, \texttt{all\_corrected\_null}, \texttt{scope\_corrected\_null}, \texttt{all\_corrected\_null\_hwgt}, \texttt{scope\_corrected\_null\_hwgt} lead to a lower performance than the base model. Insights gives \figref{fig:pseudoModelsTrainingData} (b). All of the generated training data contain false positive labels, i.e. \textit{null} samples which are labeled as hand washing, or there are just a few true negative labels. For all \texttt{all\_null\_*} configurations nearly all not hand washing samples are correctly labeled as \textit{null}. More over, the training data consists of slightly more true positive labels than the others. The resulted models reach almost the same performance as the supervised trained model.
\input{figures/experiments/supervised_pseudo_models}
\input{figures/experiments/supervised_pseudo_models_training_data}
\begin{itemize} \begin{itemize}
\item Setup of filters \item Setup of filters
\item Comparison \item Comparison
......
\begin{figure}[t]
\begin{centering}
\subfloat
{\includegraphics[width=\textwidth]{figures/experiments/supervised_pseudo_models_sen_spec.png}}
\subfloat
{\includegraphics[width=\textwidth]{figures/experiments/supervised_pseudo_models_f1_s.png}}
\caption[Pseudo models evaluation]{\textbf{Pseudo models evaluation} Measured metrics on models which have been trained on different filter configurations of training data labels. In plot of S score, the evaluation of respective training data is included. The scaled evaluations part is scaled to the ratio of training data labels to ground truth labels.}
\label{fig:pseudoModelsEvaluation}
\end{centering}
\end{figure}
\begin{figure}[t]
\begin{centering}
\subfloat[evaluation]
{\includegraphics[width=\textwidth]{figures/experiments/supervised_pseudo_models_training_data_evaluation.png}}
\subfloat[training data]
{\includegraphics[width=\textwidth]{figures/experiments/supervised_pseudo_models_training_data_confusion.png}}
\caption[Pseudo models training data]{\textbf{Pseudo models training data} Analysis of training data which was used to train previous models. Brighter parts in (a) and values in (b) are scaled to the respective ratio of training data labels to ground truth labels.}
\label{fig:pseudoModelsTrainingData}
\end{centering}
\end{figure}
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
%% Cell type:code id:9e1924eb tags: %% Cell type:code id:9e1924eb tags:
   
``` python ``` python
%load_ext autoreload %load_ext autoreload
%autoreload 2 %autoreload 2
   
%matplotlib notebook %matplotlib notebook
``` ```
   
%% Cell type:code id:32100899 tags: %% Cell type:code id:32100899 tags:
   
``` python ``` python
import sys import sys
import os import os
import numpy as np import numpy as np
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from IPython.display import display, Markdown from IPython.display import display, Markdown
import copy import copy
import torch import torch
import pandas as pd import pandas as pd
``` ```
   
%% Cell type:code id:162a8662 tags: %% Cell type:code id:162a8662 tags:
   
``` python ``` python
module_path = os.path.abspath(os.path.join('..')) module_path = os.path.abspath(os.path.join('..'))
os.chdir(module_path) os.chdir(module_path)
if module_path not in sys.path: if module_path not in sys.path:
sys.path.append(module_path) sys.path.append(module_path)
``` ```
   
%% Cell type:code id:8e909dd4 tags: %% Cell type:code id:8e909dd4 tags:
   
``` python ``` python
from tools.load_data_sets import * from personalization_tools.load_data_sets import *
from tools.helpers import * from personalization_tools.helpers import *
from tools.learner_pipeline import LearnerPipeline, Evaluation from personalization_tools.learner_pipeline import LearnerPipeline, Evaluation
from tools.dataset_builder import * from personalization_tools.dataset_builder import *
from tools.model_evaluation import ModelEvaluation from personalization_tools.model_evaluation import ModelEvaluation
from tools.sensor_recorder_data_reader import SensorRecorderDataReader from personalization_tools.sensor_recorder_data_reader import SensorRecorderDataReader
from tools.dataset_manager import DatasetManager from personalization_tools.dataset_manager import DatasetManager
from tools.dataset import SyntheticDataset from personalization_tools.dataset import SyntheticDataset
from tools.models import HandWashingDeepConvLSTMA from personalization_tools.models import HandWashingDeepConvLSTMA
from tools.metrics import sensitivity, specificity, S1_score from personalization_tools.metrics import sensitivity, specificity, S1_score
from sklearn.metrics import f1_score from sklearn.metrics import f1_score
from tools.pseudo_labels import PseudoLabelGenerator from personalization_tools.pseudo_labels import PseudoLabelGenerator
from tools.pseudo_label_evaluation import calc_evaluations_average, generate_pseudo_label_evaluations, get_avg_pseudo_evaluation_of_collection from personalization_tools.pseudo_label_evaluation import calc_evaluations_average, generate_pseudo_label_evaluations, get_avg_pseudo_evaluation_of_collection
from tools.pseudo_model_settings import pseudo_model_settings, pseudo_label_generators_to_int from personalization_tools.pseudo_model_settings import pseudo_model_settings, pseudo_label_generators_to_int
from tools.globals import Indicators from personalization_tools.globals import Indicators
``` ```
   
%% Cell type:code id:3ac7a322 tags: %% Cell type:code id:3ac7a322 tags:
   
``` python ``` python
dataset_db = './data/synthetic_dataset_db_2' dataset_db = './data/synthetic_dataset_db_2'
base_model = './data/HandWashingDeepConvLSTMA_trunc_11.pt' base_model = './data/HandWashingDeepConvLSTMA_trunc_11.pt'
#collection_name = 'base_synthetic_01' #collection_name = 'base_synthetic_01'
dataset_db = './data/recorded_dataset_db_2' dataset_db = './data/recorded_dataset_db'
base_model = './data/DeepConvLSTMA_statedict.pt' base_model = './data/DeepConvLSTMA_statedict.pt'
collection_name = '01_training' collection_name = '01_training'
``` ```
   
%% Cell type:code id:713a099c tags: %% Cell type:code id:713a099c tags:
   
``` python ``` python
dataset_manager = DatasetManager(dataset_db) dataset_manager = DatasetManager(dataset_db)
print(dataset_manager.show_categories()) print(dataset_manager.show_categories())
``` ```
   
%% Output %% Output
   
['01_all', '01_test', '01_training'] ['01_training', '01_test', '01_all', '02_training', '02_test', '02_all']
   
%% Cell type:code id:276d5c35 tags: %% Cell type:code id:276d5c35 tags:
   
``` python ``` python
collection = dataset_manager.filter_by_category(collection_name) collection = dataset_manager.filter_by_category(collection_name)
print(collection.keys()) print(collection.keys())
``` ```
   
%% Output %% Output
   
dict_keys(['cf0f18c5-a4e8-489c-b38b-af8cd0067000', '43de4036-a1e1-4aab-a3ce-671bcef3df0c', '0d07466e-797e-4663-9779-f16bc2b9ce86', '7a069df0-6a1f-41a9-b5b4-6c576651ccf7', '6370f5e9-81ea-45dc-a126-d55566b64f1f', '402b4d56-4b67-446c-a929-63ecb0ec58c7', '92d09936-6ba6-4183-a5ec-6bb8b7ac701f', '0be9f1e5-99b4-4b57-82f2-670420a7278c', '4c82aa1d-ed11-4d46-bde4-31f5b003ceab', '0c53f7b4-7aaa-4022-b3fb-acf7f523c30d', '57c3e2b9-ed9e-4140-9d13-ef31a6911b42', '65d3d0e6-389a-48f2-b02e-fcd3eb058e0a', '959a3761-3af5-4957-8d04-177ba5cfef75', 'f9f676bd-b997-46c6-a654-8a9a3c0ba583', 'a3187994-6574-4344-bcbb-001e90a00ff9', '9485d9c0-e3eb-4d31-8ae3-c843bca1ab7c']) dict_keys(['34a0c627-45a9-4b96-9900-497907da94b1', 'b7584e6d-e21d-45cd-b1f7-3bac7b4c2612', 'e49a7273-65aa-40fa-bbb6-ef6a795f8f2b', '9045e352-bb8f-4dfa-956d-405e12987125', 'af233ab2-28b1-4b7f-8a02-6cd8f9b61311'])
   
%% Cell type:code id:8dafeb13 tags: %% Cell type:code id:8dafeb13 tags:
   
``` python ``` python
predicitons = generate_predictions(collection, base_model) predicitons = generate_predictions(collection, base_model)
``` ```
   
%% Output %% Output
   
run: cf0f18c5-a4e8-489c-b38b-af8cd0067000...0.7893915176391602 seconds run prediction on [cuda]: 34a0c627-45a9-4b96-9900-497907da94b1...0.6643788814544678 seconds
run: 43de4036-a1e1-4aab-a3ce-671bcef3df0c...1.0819857120513916 seconds run prediction on [cuda]: b7584e6d-e21d-45cd-b1f7-3bac7b4c2612...0.028804779052734375 seconds
run: 0d07466e-797e-4663-9779-f16bc2b9ce86...0.684497594833374 seconds run prediction on [cuda]: e49a7273-65aa-40fa-bbb6-ef6a795f8f2b...0.19591903686523438 seconds
run: 7a069df0-6a1f-41a9-b5b4-6c576651ccf7...1.3040337562561035 seconds run prediction on [cuda]: 9045e352-bb8f-4dfa-956d-405e12987125...0.4017653465270996 seconds
run: 6370f5e9-81ea-45dc-a126-d55566b64f1f...1.108980417251587 seconds run prediction on [cuda]: af233ab2-28b1-4b7f-8a02-6cd8f9b61311...0.7093448638916016 seconds
run: 402b4d56-4b67-446c-a929-63ecb0ec58c7...1.282562255859375 seconds
run: 92d09936-6ba6-4183-a5ec-6bb8b7ac701f...0.8270609378814697 seconds %% Cell type:code id:f1cbf188 tags:
run: 0be9f1e5-99b4-4b57-82f2-670420a7278c...0.7181124687194824 seconds
run: 4c82aa1d-ed11-4d46-bde4-31f5b003ceab...1.3844504356384277 seconds ``` python
run: 0c53f7b4-7aaa-4022-b3fb-acf7f523c30d...0.9225075244903564 seconds for dataset in collection.values():
run: 57c3e2b9-ed9e-4140-9d13-ef31a6911b42...0.43587183952331543 seconds #dataset.plot_windows()
run: 65d3d0e6-389a-48f2-b02e-fcd3eb058e0a...1.3762381076812744 seconds print('noise', np.count_nonzero(dataset.indicators[1][:, 1] == Indicators.NOISE), 'hw', np.count_nonzero(dataset.indicators[1][:, 1] == Indicators.HAND_WASH))
run: 959a3761-3af5-4957-8d04-177ba5cfef75...1.064845085144043 seconds ```
run: f9f676bd-b997-46c6-a654-8a9a3c0ba583...0.8436729907989502 seconds
run: a3187994-6574-4344-bcbb-001e90a00ff9...0.32437992095947266 seconds %% Output
run: 9485d9c0-e3eb-4d31-8ae3-c843bca1ab7c...1.1401820182800293 seconds
noise 12 hw 5
noise 0 hw 2
noise 1 hw 2
noise 1 hw 1
noise 12 hw 5
%% Cell type:code id:6f293240 tags:
``` python
for dataset in collection.values():
dataset.randomize_evaluation(0.8, 0.5)
```
%% Cell type:code id:647d9ced tags:
``` python
for dataset in collection.values():
#dataset.plot_windows()
print('noise', np.count_nonzero(dataset.indicators[1][:, 1] == Indicators.NOISE), 'hw', np.count_nonzero(dataset.indicators[1][:, 1] == Indicators.HAND_WASH))
```
%% Output
noise 10 hw 3
noise 0 hw 1
noise 1 hw 1
noise 1 hw 1
noise 10 hw 3
   
%% Cell type:code id:5643b323 tags: %% Cell type:code id:5643b323 tags:
   
``` python ``` python
#dataset_manager.update_dataset_collection(collection.values()) #dataset_manager.update_dataset_collection(collection.values())
#dataset_manager.__db_update__() #dataset_manager.__db_update__()
``` ```
   
%% Cell type:code id:c1d63a1c tags: %% Cell type:code id:c1d63a1c tags:
   
``` python ``` python
for dataset in collection.values(): for dataset in collection.values():
print(dataset.name) print(dataset.name)
if isinstance(dataset, SyntheticDataset): if isinstance(dataset, SyntheticDataset):
print('do') print('do')
#dataset.indicators[0] -= 30 #dataset.indicators[0] -= 30
#dataset.indicators[1][:, 0] -= 30 #dataset.indicators[1][:, 0] -= 30
dataset.indicators_synthetic = None dataset.indicators_synthetic = None
dataset.generate_synthetic_evaluation(predicitons[dataset.name], evaluation_reliability_no=1, evaluation_reliability_yes=1) dataset.generate_synthetic_evaluation(predicitons[dataset.name], evaluation_reliability_no=1, evaluation_reliability_yes=1)
# dataset.indicators_synthetic[0] = np.zeros((0)) # dataset.indicators_synthetic[0] = np.zeros((0))
# dataset.indicators[0] -= 60 # dataset.indicators[0] -= 60
# dataset.indicators[1][:, 0] -= 60 # dataset.indicators[1][:, 0] -= 60
dataset.generate_feedback_areas(prediction=predicitons[dataset.name]) dataset.generate_feedback_areas(prediction=predicitons[dataset.name])
# dataset_manager.__db_update__() # dataset_manager.__db_update__()
``` ```
   
%% Output %% Output
   
11_generated_3 11_generated_3
do do
11_generated_4 11_generated_4
do do
11_generated_5 11_generated_5
do do
11_generated_6 11_generated_6
do do
11_generated_7 11_generated_7
do do
11_generated_8 11_generated_8
do do
11_generated_9 11_generated_9
do do
11_generated_10 11_generated_10
do do
11_generated_11 11_generated_11
do do
11_generated_12 11_generated_12
do do
11_generated_13 11_generated_13
do do
11_generated_14 11_generated_14
do do
11_generated_15 11_generated_15
do do
11_generated_16 11_generated_16
do do
   
--------------------------------------------------------------------------- ---------------------------------------------------------------------------
ValueError Traceback (most recent call last) ValueError Traceback (most recent call last)
<ipython-input-12-9ee5cfafcc83> in <module> <ipython-input-12-9ee5cfafcc83> in <module>
6 #dataset.indicators[1][:, 0] -= 30 6 #dataset.indicators[1][:, 0] -= 30
7 dataset.indicators_synthetic = None 7 dataset.indicators_synthetic = None
----> 8 dataset.generate_synthetic_evaluation(predicitons[dataset.name], evaluation_reliability_no=1, evaluation_reliability_yes=1) ----> 8 dataset.generate_synthetic_evaluation(predicitons[dataset.name], evaluation_reliability_no=1, evaluation_reliability_yes=1)
9 # dataset.indicators_synthetic[0] = np.zeros((0)) 9 # dataset.indicators_synthetic[0] = np.zeros((0))
10 # dataset.indicators[0] -= 60 10 # dataset.indicators[0] -= 60
~/gitrepos/uni/handwashing_personalizer/src/tools/dataset.py in generate_synthetic_evaluation(self, y_predicted, kernel_size, kernel_threshold, evaluation_reliability_no, evaluation_reliability_yes, consider_actions, clear_manual, clear_just_covered) ~/gitrepos/uni/handwashing_personalizer/src/tools/dataset.py in generate_synthetic_evaluation(self, y_predicted, kernel_size, kernel_threshold, evaluation_reliability_no, evaluation_reliability_yes, consider_actions, clear_manual, clear_just_covered)
318 if len(manual_indicators) > 0: 318 if len(manual_indicators) > 0:
319 # print(evaluation_indices, manual_indicators) 319 # print(evaluation_indices, manual_indicators)
--> 320 evaluation_indices = np.append(evaluation_indices, np.asarray([(i, Indicators.HAND_WASH) for i in manual_indicators]), axis=0) --> 320 evaluation_indices = np.append(evaluation_indices, np.asarray([(i, Indicators.HAND_WASH) for i in manual_indicators]), axis=0)
321 evaluation_indices = evaluation_indices[evaluation_indices[:, 0].argsort()] 321 evaluation_indices = evaluation_indices[evaluation_indices[:, 0].argsort()]
322 self.indicators_synthetic[0] = np.asarray(manual_indicators) 322 self.indicators_synthetic[0] = np.asarray(manual_indicators)
<__array_function__ internals> in append(*args, **kwargs) <__array_function__ internals> in append(*args, **kwargs)
~/.local/lib/python3.8/site-packages/numpy/lib/function_base.py in append(arr, values, axis) ~/.local/lib/python3.8/site-packages/numpy/lib/function_base.py in append(arr, values, axis)
4815 values = ravel(values) 4815 values = ravel(values)
4816 axis = arr.ndim-1 4816 axis = arr.ndim-1
-> 4817 return concatenate((arr, values), axis=axis) -> 4817 return concatenate((arr, values), axis=axis)
4818 4818
4819 4819
<__array_function__ internals> in concatenate(*args, **kwargs) <__array_function__ internals> in concatenate(*args, **kwargs)
ValueError: all the input arrays must have same number of dimensions, but the array at index 0 has 1 dimension(s) and the array at index 1 has 2 dimension(s) ValueError: all the input arrays must have same number of dimensions, but the array at index 0 has 1 dimension(s) and the array at index 1 has 2 dimension(s)
   
%% Cell type:code id:816fbb1d tags: %% Cell type:code id:816fbb1d tags:
   
``` python ``` python
requests = 0 requests = 0
   
observed_collection = list(collection.values()) observed_collection = list(collection.values())
for dataset in observed_collection: for dataset in observed_collection:
evaluations = dataset.get_indicators()[1] evaluations = dataset.get_indicators()[1]
for evaluation in evaluations: for evaluation in evaluations:
if evaluation[1] != Indicators.NEUTRAL: if evaluation[1] != Indicators.NEUTRAL:
requests += 1 requests += 1
print('Num requests:', requests) print('Num requests:', requests)
print('requests per dataset:', requests/len(observed_collection)) print('requests per dataset:', requests/len(observed_collection))
``` ```
   
%% Output %% Output
   
Num requests: 28 Num requests: 28
requests per dataset: 5.6 requests per dataset: 5.6
   
%% Cell type:code id:55f54903 tags: %% Cell type:code id:55f54903 tags:
   
``` python ``` python
dataset_name = list(collection.keys())[0] dataset_name = list(collection.keys())[0]
#print(dataset_name) #print(dataset_name)
dataset = collection[dataset_name] dataset = collection[dataset_name]
#print(dataset.indicators) #print(dataset.indicators)
#print(dataset.x_win.shape, dataset.y_data.shape) #print(dataset.x_win.shape, dataset.y_data.shape)
#print(dataset.indicators) #print(dataset.indicators)
ax = dataset.plot_windows() ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax) dataset.plot_feedback_areas(ax)
print(dataset.indicators_synthetic) print(dataset.indicators_synthetic)
``` ```
   
%% Output %% Output
   
   
   
[array([7310]), array([[ 905, 0], [array([7310]), array([[ 905, 0],
[ 1143, -1], [ 1143, -1],
[ 1164, -1], [ 1164, -1],
[ 1185, -1], [ 1185, -1],
[ 1206, -1], [ 1206, -1],
[ 1227, 0], [ 1227, 0],
[ 1555, 0], [ 1555, 0],
[ 3786, 0], [ 3786, 0],
[ 4201, -1], [ 4201, -1],
[ 4222, 1], [ 4222, 1],
[ 7310, 1], [ 7310, 1],
[ 9517, 0], [ 9517, 0],
[ 9542, -1], [ 9542, -1],
[ 9563, -1], [ 9563, -1],
[ 9584, -1], [ 9584, -1],
[ 9605, -1], [ 9605, -1],
[ 9626, 0], [ 9626, 0],
[ 9670, -1], [ 9670, -1],
[ 9691, 0], [ 9691, 0],
[ 9811, 0], [ 9811, 0],
[ 9972, 0], [ 9972, 0],
[10219, 0], [10219, 0],
[10245, 0], [10245, 0],
[10551, -1], [10551, -1],
[10573, -1], [10573, -1],
[10594, -1], [10594, -1],
[10618, -1], [10618, -1],
[10639, -1], [10639, -1],
[10662, -1], [10662, -1],
[10683, 0], [10683, 0],
[10865, -1], [10865, -1],
[10886, 0], [10886, 0],
[11102, -1], [11102, -1],
[11123, 0], [11123, 0],
[11175, 1], [11175, 1],
[11435, 0], [11435, 0],
[11913, 0], [11913, 0],
[11942, -1], [11942, -1],
[11963, -1], [11963, -1],
[11984, -1], [11984, -1],
[12005, -1], [12005, -1],
[12026, -1], [12026, -1],
[12047, -1], [12047, -1],
[12068, 0], [12068, 0],
[12169, -1], [12169, -1],
[12190, 0], [12190, 0],
[12234, -1], [12234, -1],
[12255, -1], [12255, -1],
[12276, -1], [12276, -1],
[12297, -1], [12297, -1],
[12318, -1], [12318, -1],
[12339, 0], [12339, 0],
[13149, 0], [13149, 0],
[13243, 0], [13243, 0],
[13597, 0], [13597, 0],
[13637, 0], [13637, 0],
[13726, 0], [13726, 0],
[13912, -1], [13912, -1],
[13933, 1], [13933, 1],
[15866, -1], [15866, -1],
[15887, 0], [15887, 0],
[15942, 0], [15942, 0],
[15992, -1], [15992, -1],
[16013, -1], [16013, -1],
[16034, 0], [16034, 0],
[16121, 0], [16121, 0],
[16158, -1], [16158, -1],
[16179, -1], [16179, -1],
[16201, -1], [16201, -1],
[16222, -1], [16222, -1],
[16243, 0], [16243, 0],
[16296, 0], [16296, 0],
[16343, 0], [16343, 0],
[16389, -1], [16389, -1],
[16410, -1], [16410, -1],
[16431, -1], [16431, -1],
[16452, -1], [16452, -1],
[16473, -1], [16473, -1],
[16494, -1], [16494, -1],
[16515, 0]])] [16515, 0]])]
   
%% Cell type:code id:1da7daa6 tags: %% Cell type:code id:1da7daa6 tags:
   
``` python ``` python
relevant_pseudo_models = ['all', 'scope', 'all_corrected', 'scope_corrected', 'synthetic', 'allnoise_correctedhwgt_scope', 'allnoise_correctedscore', 'allnoise_correctedscore_flatten', 'allnoise_correctbydeepconvfilter', 'allnoise_correctbyfcndaefilter', 'allnoise_correctbyfcndaefiltertest'] relevant_pseudo_models = ['all', 'scope', 'all_corrected', 'scope_corrected', 'synthetic', 'allnoise_correctedhwgt_scope', 'allnoise_correctedscore', 'allnoise_correctedscore_flatten', 'allnoise_correctbydeepconvfilter', 'allnoise_correctbyfcndaefilter', 'allnoise_correctbyfcndaefiltertest']
relevant_pseudo_models = ['allnoise_correctedscore', 'allnoise_correctbydeepconvfilter', 'allnoise_correctbyfcndaefilter', 'allnoise_correctbyfcndaefiltertest', 'allnoise_correctbyconvlstmfilter', 'allnoise_correctbyconvlstm2filter', 'allnoise_correctbyconvlstm3filter', 'alldeepconv_correctbyconvlstm3filter', 'alldeepconv_correctbyconvlstm3filter4', 'alldeepconv_correctbyconvlstm3filter5','alldeepconv_correctbyconvlstm3filter6'] relevant_pseudo_models = ['allnoise_correctedscore', 'allnoise_correctbydeepconvfilter', 'allnoise_correctbyfcndaefilter', 'allnoise_correctbyfcndaefiltertest', 'allnoise_correctbyconvlstmfilter', 'allnoise_correctbyconvlstm2filter', 'allnoise_correctbyconvlstm3filter', 'alldeepconv_correctbyconvlstm3filter', 'alldeepconv_correctbyconvlstm3filter4', 'alldeepconv_correctbyconvlstm3filter5','alldeepconv_correctbyconvlstm3filter6']
``` ```
   
%% Cell type:code id:087a3c33 tags: %% Cell type:code id:087a3c33 tags:
   
``` python ``` python
def plot_pseudo_labels(dataset_name, observed_models): def plot_pseudo_labels(dataset_name, observed_models):
evaluations = [] evaluations = []
fig, axes = plt.subplots(len(observed_models), figsize=(8,26)) fig, axes = plt.subplots(len(observed_models), figsize=(8,26))
for i, model_setting in enumerate(observed_models): for i, model_setting in enumerate(observed_models):
print(model_setting) print(model_setting)
collection[dataset_name].apply_pseudo_label_generators(pseudo_model_settings[model_setting]) collection[dataset_name].apply_pseudo_label_generators(pseudo_model_settings[model_setting])
evals = collection[dataset_name].evaluate_pseudo_labels(model_setting) evals = collection[dataset_name].evaluate_pseudo_labels(model_setting)
ax = collection[dataset_name].plot_feedback_areas(ax=axes[i]) ax = collection[dataset_name].plot_feedback_areas(ax=axes[i])
collection[dataset_name].plot_pseudo_labels(ax=ax) collection[dataset_name].plot_pseudo_labels(ax=ax)
evaluations.append(evals) evaluations.append(evals)
ax.set_title(model_setting) ax.set_title(model_setting)
   
fig.tight_layout() fig.tight_layout()
fig.show() fig.show()
return evaluations return evaluations
   
def generate_pseudo_labels(dataset_name, observed_models): def generate_pseudo_labels(dataset_name, observed_models):
evaluations = [] evaluations = []
for i, model_setting in enumerate(observed_models): for i, model_setting in enumerate(observed_models):
collection[dataset_name].apply_pseudo_label_generators(pseudo_model_settings[model_setting]) collection[dataset_name].apply_pseudo_label_generators(pseudo_model_settings[model_setting])
evals = collection[dataset_name].evaluate_pseudo_labels(model_setting) evals = collection[dataset_name].evaluate_pseudo_labels(model_setting)
evaluations.append(evals) evaluations.append(evals)
   
   
return evaluations return evaluations
``` ```
   
%% Cell type:code id:d66ef0c4 tags: %% Cell type:code id:d66ef0c4 tags:
   
``` python ``` python
evaluations = plot_pseudo_labels('01_generated_3', relevant_pseudo_models) evaluations = plot_pseudo_labels('01_generated_3', relevant_pseudo_models)
``` ```
   
%% Cell type:code id:d63549e1 tags: %% Cell type:code id:d63549e1 tags:
   
``` python ``` python
gen_evaluations = dict() gen_evaluations = dict()
for dataset in list(collection.values()): for dataset in list(collection.values()):
gen_evaluations[dataset.name] = generate_pseudo_label_evaluations(collection[dataset.name], relevant_pseudo_models) gen_evaluations[dataset.name] = generate_pseudo_label_evaluations(collection[dataset.name], relevant_pseudo_models)
   
averages = calc_evaluations_average(list(gen_evaluations.values())) averages = calc_evaluations_average(list(gen_evaluations.values()))
``` ```
   
%% Cell type:code id:181e9e74 tags: %% Cell type:code id:181e9e74 tags:
   
``` python ``` python
series = [] series = []
for evaluation in evaluations: for evaluation in evaluations:
series.append(evaluation[0].get_values()) series.append(evaluation[0].get_values())
series.append(evaluation[1].get_values()) series.append(evaluation[1].get_values())
series.append(evaluation[2].get_values()) series.append(evaluation[2].get_values())
#series = (eval_base.get_values(), eval_pseudo.get_values(), eval_base_scoped.get_values(), eval_pseudo_scoped.get_values(), eval_base_scoped_extneut.get_values(), eval_pseudo_scoped_extneut.get_values()) #series = (eval_base.get_values(), eval_pseudo.get_values(), eval_base_scoped.get_values(), eval_pseudo_scoped.get_values(), eval_base_scoped_extneut.get_values(), eval_pseudo_scoped_extneut.get_values())
df = pd.DataFrame(series, columns=['labels', 'specificity', 'sensitivity', 'f1', 'S1']) df = pd.DataFrame(series, columns=['labels', 'specificity', 'sensitivity', 'f1', 'S1'])
df df
``` ```
   
%% Cell type:code id:7fde8f11 tags: %% Cell type:code id:7fde8f11 tags:
   
``` python ``` python
dataframes = [] dataframes = []
for dataset_evaluation in evaluations.values(): for dataset_evaluation in evaluations.values():
series = [] series = []
for evaluation in dataset_evaluation: for evaluation in dataset_evaluation:
series.append(evaluation[0].get_values()) series.append(evaluation[0].get_values())
series.append(evaluation[1].get_values()) series.append(evaluation[1].get_values())
#series = (eval_base.get_values(), eval_pseudo.get_values(), eval_base_scoped.get_values(), eval_pseudo_scoped.get_values(), eval_base_scoped_extneut.get_values(), eval_pseudo_scoped_extneut.get_values()) #series = (eval_base.get_values(), eval_pseudo.get_values(), eval_base_scoped.get_values(), eval_pseudo_scoped.get_values(), eval_base_scoped_extneut.get_values(), eval_pseudo_scoped_extneut.get_values())
df = pd.DataFrame(series, columns=['labels', 'specificity', 'sensitivity', 'f1', 'S1']) df = pd.DataFrame(series, columns=['labels', 'specificity', 'sensitivity', 'f1', 'S1'])
dataframes.append(df) dataframes.append(df)
``` ```
   
%% Cell type:code id:02fc7bf3 tags: %% Cell type:code id:02fc7bf3 tags:
   
``` python ``` python
   
dataframes[0] dataframes[0]
``` ```
   
%% Cell type:code id:f78bd2c7 tags: %% Cell type:code id:f78bd2c7 tags:
   
``` python ``` python
def plot_pseudo_label_stats_single(evaluations, target_models, target_value): def plot_pseudo_label_stats_single(evaluations, target_models, target_value):
fig, ax = plt.subplots(figsize=(8, 6)) fig, ax = plt.subplots(figsize=(8, 6))
ax.set_title(target_value) ax.set_title(target_value)
y_vals_hard = [] y_vals_hard = []
y_vals_soft = [] y_vals_soft = []
for i in range(len(target_models)): for i in range(len(target_models)):
y_vals_hard.append(getattr(evaluations[i][1], target_value)) y_vals_hard.append(getattr(evaluations[i][1], target_value))
y_vals_soft.append(getattr(evaluations[i][2], target_value)) y_vals_soft.append(getattr(evaluations[i][2], target_value))
ax.plot(target_models, y_vals_hard, label='hard') ax.plot(target_models, y_vals_hard, label='hard')
ax.plot(target_models, y_vals_soft, label='soft') ax.plot(target_models, y_vals_soft, label='soft')
ax.tick_params(labelrotation=45) ax.tick_params(labelrotation=45)
fig.legend() fig.legend()
ax.grid() ax.grid()
fig.show() fig.show()
   
def plot_pseudo_label_stats(evaluations, target_models, target_value): def plot_pseudo_label_stats(evaluations, target_models, target_value):
fig, ax = plt.subplots() fig, ax = plt.subplots()
ax.set_title(target_value) ax.set_title(target_value)
for dataset, evals in evaluations.items(): for dataset, evals in evaluations.items():
y_vals_hard = [] y_vals_hard = []
y_vals_soft = [] y_vals_soft = []
for i in range(len(target_models)): for i in range(len(target_models)):
y_vals_hard.append(getattr(evals[i][1], target_value)) y_vals_hard.append(getattr(evals[i][1], target_value))
y_vals_soft.append(getattr(evals[i][2], target_value)) y_vals_soft.append(getattr(evals[i][2], target_value))
#ax.plot(target_models, y_vals_hard, label='hard') #ax.plot(target_models, y_vals_hard, label='hard')
ax.plot(target_models, y_vals_soft, label='soft') ax.plot(target_models, y_vals_soft, label='soft')
ax.tick_params(labelrotation=45) ax.tick_params(labelrotation=45)
fig.legend() fig.legend()
fig.show() fig.show()
   
   
def plot_pseudo_label_averages(evaluations, target_models, target_value, ax=None): def plot_pseudo_label_averages(evaluations, target_models, target_value, ax=None):
if ax is None: if ax is None:
fig, axis = plt.subplots() fig, axis = plt.subplots()
else: else:
axis = ax axis = ax
axis.set_title(target_value) axis.set_title(target_value)
y_vals_soft = [] y_vals_soft = []
for model_evals in evaluations: for model_evals in evaluations:
y_vals_soft.append(model_evals[2][target_value]) y_vals_soft.append(model_evals[2][target_value])
   
#ax.plot(target_models, y_vals_hard, label='hard') #ax.plot(target_models, y_vals_hard, label='hard')
axis.plot(target_models, y_vals_soft, label='soft') axis.plot(target_models, y_vals_soft, label='soft')
axis.tick_params(labelrotation=45) axis.tick_params(labelrotation=45)
axis.grid() axis.grid()
if ax is None: if ax is None:
fig.legend() fig.legend()
fig.show() fig.show()
return axis return axis
   
``` ```
   
%% Cell type:code id:857e4701 tags: %% Cell type:code id:857e4701 tags:
   
``` python ``` python
gen_evaluations = dict() gen_evaluations = dict()
for dataset in list(collection.values()): for dataset in list(collection.values()):
gen_evaluations[dataset.name] = generate_pseudo_label_evaluations(collection[dataset.name], relevant_pseudo_models) gen_evaluations[dataset.name] = generate_pseudo_label_evaluations(collection[dataset.name], relevant_pseudo_models)
   
averages = calc_evaluations_average(list(gen_evaluations.values())) averages = calc_evaluations_average(list(gen_evaluations.values()))
   
fig, ax = plt.subplots(4, figsize=(8,20)) fig, ax = plt.subplots(4, figsize=(8,20))
   
plot_pseudo_label_averages(averages, relevant_pseudo_models, 'specificity', ax[0]) plot_pseudo_label_averages(averages, relevant_pseudo_models, 'specificity', ax[0])
plot_pseudo_label_averages(averages, relevant_pseudo_models, 'sensitivity', ax[1]) plot_pseudo_label_averages(averages, relevant_pseudo_models, 'sensitivity', ax[1])
plot_pseudo_label_averages(averages, relevant_pseudo_models, 'f1', ax[2]) plot_pseudo_label_averages(averages, relevant_pseudo_models, 'f1', ax[2])
plot_pseudo_label_averages(averages, relevant_pseudo_models, 's1', ax[3]) plot_pseudo_label_averages(averages, relevant_pseudo_models, 's1', ax[3])
plt.subplots_adjust(hspace=0.7) plt.subplots_adjust(hspace=0.7)
fig.legend() fig.legend()
fig.show() fig.show()
``` ```
   
%% Output %% Output
   
   
   
%% Cell type:code id:df94b630 tags: %% Cell type:code id:df94b630 tags:
   
``` python ``` python
plot_pseudo_label_stats(gen_evaluations, relevant_pseudo_models, 'specificity') plot_pseudo_label_stats(gen_evaluations, relevant_pseudo_models, 'specificity')
plot_pseudo_label_stats(gen_evaluations, relevant_pseudo_models, 'sensitivity') plot_pseudo_label_stats(gen_evaluations, relevant_pseudo_models, 'sensitivity')
plot_pseudo_label_stats(gen_evaluations, relevant_pseudo_models, 'f1') plot_pseudo_label_stats(gen_evaluations, relevant_pseudo_models, 'f1')
plot_pseudo_label_stats(gen_evaluations, relevant_pseudo_models, 's1') plot_pseudo_label_stats(gen_evaluations, relevant_pseudo_models, 's1')
``` ```
   
%% Cell type:code id:de35210b tags: %% Cell type:code id:de35210b tags:
   
``` python ``` python
``` ```
   
%% Cell type:code id:b1d443c6 tags: %% Cell type:code id:b1d443c6 tags:
   
``` python ``` python
dataset_name = '01_generated_4' dataset_name = '01_generated_4'
# dataset_name = '34a0c627-45a9-4b96-9900-497907da94b1' # dataset_name = '34a0c627-45a9-4b96-9900-497907da94b1'
dataset = collection[dataset_name] dataset = collection[dataset_name]
# dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1'] # dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1']
dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctedscore']) dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctedscore'])
ax = dataset.plot_windows() ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax) dataset.plot_feedback_areas(ax)
dataset.plot_pseudo_labels(ax) dataset.plot_pseudo_labels(ax)
dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax=ax) dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax=ax)
   
dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbydeepconvfilter']) dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbydeepconvfilter'])
ax = dataset.plot_windows() ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax) dataset.plot_feedback_areas(ax)
dataset.plot_pseudo_labels(ax) dataset.plot_pseudo_labels(ax)
dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax=ax) dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax=ax)
   
   
plot_pseudo_label_stats_single(gen_evaluations[dataset_name], relevant_pseudo_models, 'specificity') plot_pseudo_label_stats_single(gen_evaluations[dataset_name], relevant_pseudo_models, 'specificity')
plt.subplots_adjust(bottom=0.3) plt.subplots_adjust(bottom=0.3)
plot_pseudo_label_stats_single(gen_evaluations[dataset_name], relevant_pseudo_models, 'sensitivity') plot_pseudo_label_stats_single(gen_evaluations[dataset_name], relevant_pseudo_models, 'sensitivity')
plt.subplots_adjust(bottom=0.3) plt.subplots_adjust(bottom=0.3)
plot_pseudo_label_stats_single(gen_evaluations[dataset_name], relevant_pseudo_models, 'f1') plot_pseudo_label_stats_single(gen_evaluations[dataset_name], relevant_pseudo_models, 'f1')
plt.subplots_adjust(bottom=0.3) plt.subplots_adjust(bottom=0.3)
plot_pseudo_label_stats_single(gen_evaluations[dataset_name], relevant_pseudo_models, 's1') plot_pseudo_label_stats_single(gen_evaluations[dataset_name], relevant_pseudo_models, 's1')
plt.subplots_adjust(bottom=0.3) plt.subplots_adjust(bottom=0.3)
``` ```
   
%% Cell type:code id:39e6a390 tags: %% Cell type:code id:39e6a390 tags:
   
``` python ``` python
def sensitivity_soft(y_true, y_pred): def sensitivity_soft(y_true, y_pred):
# r: reference -> y_true # r: reference -> y_true
# p: pred -> y_pred # p: pred -> y_pred
# Sum_n: sum over all samples # Sum_n: sum over all samples
diff = np.absolute(y_pred - y_true) diff = np.absolute(y_pred - y_true)
frac = y_true / np.sum(y_true, axis=0) frac = y_true / np.sum(y_true, axis=0)
frac_sum = np.sum(frac*diff, axis=0) frac_sum = np.sum(frac*diff, axis=0)
return 1 - frac_sum return 1 - frac_sum
   
def specificity_soft(y_true, y_pred): def specificity_soft(y_true, y_pred):
diff = np.absolute(y_pred - y_true) diff = np.absolute(y_pred - y_true)
frac = (1-y_true) / np.sum(1-y_true, axis=0) frac = (1-y_true) / np.sum(1-y_true, axis=0)
frac_sum = np.sum(frac*diff, axis=0) frac_sum = np.sum(frac*diff, axis=0)
return 1 - frac_sum return 1 - frac_sum
   
def positive_predicted_value_soft(y_true, y_pred): def positive_predicted_value_soft(y_true, y_pred):
diff = np.absolute(y_pred - y_true) diff = np.absolute(y_pred - y_true)
frac = y_pred / np.sum(y_pred, axis=0) frac = y_pred / np.sum(y_pred, axis=0)
frac_sum = np.sum(frac*diff, axis=0) frac_sum = np.sum(frac*diff, axis=0)
return 1 - frac_sum return 1 - frac_sum
   
def S1_score_soft(y_true, y_pred): def S1_score_soft(y_true, y_pred):
sens = sensitivity_soft(y_true, y_pred) sens = sensitivity_soft(y_true, y_pred)
spec = specificity_soft(y_true, y_pred) spec = specificity_soft(y_true, y_pred)
return 2 * sens * spec / (sens + spec) return 2 * sens * spec / (sens + spec)
   
def f1_score_soft(y_true, y_pred): def f1_score_soft(y_true, y_pred):
# recall = sensitivity # recall = sensitivity
# precision = positive predictive value (PPV) # precision = positive predictive value (PPV)
recall = sensitivity_soft(y_true, y_pred) recall = sensitivity_soft(y_true, y_pred)
precision = positive_predicted_value_soft(y_true, y_pred) precision = positive_predicted_value_soft(y_true, y_pred)
return 2 * (precision * recall) / (precision + recall) return 2 * (precision * recall) / (precision + recall)
``` ```
   
%% Cell type:code id:7f4e7091 tags: %% Cell type:code id:7f4e7091 tags:
   
``` python ``` python
test_arr_true = np.array(((1,0), (1,0), (1,0), (0,1), (0,1), (1,0), (1,0), (0,1), (1,0), (1,0))) test_arr_true = np.array(((1,0), (1,0), (1,0), (0,1), (0,1), (1,0), (1,0), (0,1), (1,0), (1,0)))
test_arr_pred = np.array(((1,0), (0,1), (1,0), (0,1), (1,0), (1,0), (0.4,0.6), (0,1), (1,0), (1,0))) test_arr_pred = np.array(((1,0), (0,1), (1,0), (0,1), (1,0), (1,0), (0.4,0.6), (0,1), (1,0), (1,0)))
print(sensitivity_soft(test_arr_true, test_arr_pred)) print(sensitivity_soft(test_arr_true, test_arr_pred))
print(specificity_soft(test_arr_true, test_arr_pred)) print(specificity_soft(test_arr_true, test_arr_pred))
print(S1_score_soft(test_arr_true, test_arr_pred)) print(S1_score_soft(test_arr_true, test_arr_pred))
print(f1_score_soft(test_arr_true, test_arr_pred)) print(f1_score_soft(test_arr_true, test_arr_pred))
``` ```
   
%% Cell type:code id:b6b3275d tags: %% Cell type:code id:b6b3275d tags:
   
``` python ``` python
print(sensitivity(np.argmax(test_arr_true,axis=1), np.argmax(test_arr_pred,axis=1))) print(sensitivity(np.argmax(test_arr_true,axis=1), np.argmax(test_arr_pred,axis=1)))
print(specificity(np.argmax(test_arr_true,axis=1), np.argmax(test_arr_pred,axis=1))) print(specificity(np.argmax(test_arr_true,axis=1), np.argmax(test_arr_pred,axis=1)))
print(S1_score(np.argmax(test_arr_true,axis=1), np.argmax(test_arr_pred,axis=1))) print(S1_score(np.argmax(test_arr_true,axis=1), np.argmax(test_arr_pred,axis=1)))
print(f1_score(np.argmax(test_arr_true,axis=1), np.argmax(test_arr_pred,axis=1))) print(f1_score(np.argmax(test_arr_true,axis=1), np.argmax(test_arr_pred,axis=1)))
``` ```
   
%% Cell type:code id:2c39e41f tags: %% Cell type:code id:2c39e41f tags:
   
``` python ``` python
dataset_name = '01_generated_3' dataset_name = '01_generated_3'
#dataset_name = '34a0c627-45a9-4b96-9900-497907da94b1' #dataset_name = '34a0c627-45a9-4b96-9900-497907da94b1'
dataset = collection[dataset_name] dataset = collection[dataset_name]
# dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1'] # dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1']
dataset.pseudo_labels.set_all_noise() dataset.pseudo_labels.set_all_noise()
dataset.pseudo_labels.correct_hw_by_neepconv_filter() dataset.pseudo_labels.correct_hw_by_neepconv_filter()
dataset.pseudo_labels.correct_by_feedback_noise() dataset.pseudo_labels.correct_by_feedback_noise()
ax = dataset.plot_windows() ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax) dataset.plot_feedback_areas(ax)
dataset.plot_pseudo_labels(ax) dataset.plot_pseudo_labels(ax)
dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax) dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax)
evaluations = dataset.pseudo_labels.evaluate(dataset.y_win) evaluations = dataset.pseudo_labels.evaluate(dataset.y_win)
print(evaluations[2].get_values()) print(evaluations[2].get_values())
``` ```
   
%% Cell type:code id:9a77ddb9 tags: %% Cell type:code id:9a77ddb9 tags:
   
``` python ``` python
evaluations[2].get_values() evaluations[2].get_values()
``` ```
   
%% Cell type:code id:6438cbdc tags: %% Cell type:code id:6438cbdc tags:
   
``` python ``` python
train_collection = dataset_manager.filter_by_category('base_synthetic_01_training') train_collection = dataset_manager.filter_by_category('base_synthetic_01_training')
averages = get_avg_pseudo_evaluation_of_collection(train_collection.values()) averages = get_avg_pseudo_evaluation_of_collection(train_collection.values())
``` ```
   
%% Cell type:code id:2afc55f3 tags: %% Cell type:code id:2afc55f3 tags:
   
``` python ``` python
averages averages
``` ```
   
%% Cell type:code id:287caf19 tags: %% Cell type:code id:287caf19 tags:
   
``` python ``` python
for dataset in train_collection.values(): for dataset in train_collection.values():
dataset.pseudo_labels.reset() dataset.pseudo_labels.reset()
dataset.pseudo_labels.set_all_noise() dataset.pseudo_labels.set_all_noise()
dataset.pseudo_labels.correct_hw_by_neepconv_filter(model_name='handwash_deepconv_filter_2.pt') dataset.pseudo_labels.correct_hw_by_neepconv_filter(model_name='handwash_deepconv_filter_2.pt')
dataset.pseudo_labels.correct_by_feedback_noise() dataset.pseudo_labels.correct_by_feedback_noise()
averages = get_avg_pseudo_evaluation_of_collection(train_collection.values()) averages = get_avg_pseudo_evaluation_of_collection(train_collection.values())
print(averages) print(averages)
``` ```
   
%% Cell type:code id:1a7e34db tags: %% Cell type:code id:1a7e34db tags:
   
``` python ``` python
for dataset in train_collection.values(): for dataset in train_collection.values():
dataset.pseudo_labels.reset() dataset.pseudo_labels.reset()
dataset.pseudo_labels.set_all_noise() dataset.pseudo_labels.set_all_noise()
dataset.pseudo_labels.correct_hw_by_neepconv_filter(model_name='handwash_filter.pt') dataset.pseudo_labels.correct_hw_by_neepconv_filter(model_name='handwash_filter.pt')
dataset.pseudo_labels.correct_by_feedback_noise() dataset.pseudo_labels.correct_by_feedback_noise()
averages = get_avg_pseudo_evaluation_of_collection(train_collection.values()) averages = get_avg_pseudo_evaluation_of_collection(train_collection.values())
print(averages) print(averages)
``` ```
   
%% Cell type:code id:cb856e33 tags: %% Cell type:code id:cb856e33 tags:
   
``` python ``` python
dataset_name = '01_generated_4' dataset_name = '01_generated_4'
#dataset_name = '34a0c627-45a9-4b96-9900-497907da94b1' #dataset_name = '34a0c627-45a9-4b96-9900-497907da94b1'
dataset = collection[dataset_name] dataset = collection[dataset_name]
# dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1'] # dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1']
dataset.pseudo_labels.set_all_noise() dataset.pseudo_labels.set_all_noise()
dataset.pseudo_labels.correct_hw_by_fcndae_filter_test('handwash_filter_fcndae_1.pt') dataset.pseudo_labels.correct_hw_by_fcndae_filter_test('handwash_filter_fcndae_1.pt')
dataset.pseudo_labels.correct_by_feedback_noise() dataset.pseudo_labels.correct_by_feedback_noise()
ax = dataset.plot_windows() ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax) dataset.plot_feedback_areas(ax)
dataset.plot_pseudo_labels(ax) dataset.plot_pseudo_labels(ax)
dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax) dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax)
evaluations = dataset.pseudo_labels.evaluate(dataset.y_win) evaluations = dataset.pseudo_labels.evaluate(dataset.y_win)
print(evaluations[2].get_values()) print(evaluations[2].get_values())
``` ```
   
%% Cell type:code id:29394ae9 tags: %% Cell type:code id:29394ae9 tags:
   
``` python ``` python
('_pseudo', 0.9989988151094844, 0.8930696899131422, 0.8850275892320095, 0.9430689847276482) ('_pseudo', 0.9989988151094844, 0.8930696899131422, 0.8850275892320095, 0.9430689847276482)
('_pseudo', 0.9987164602423361, 0.96314140192924, 0.9318188089206001, 0.9806063835663211) ('_pseudo', 0.9987164602423361, 0.96314140192924, 0.9318188089206001, 0.9806063835663211)
('_pseudo', 0.9986266540771351, 0.9630133999932197, 0.9281106736362976, 0.980496750635969) ('_pseudo', 0.9986266540771351, 0.9630133999932197, 0.9281106736362976, 0.980496750635969)
``` ```
   
%% Cell type:code id:8c411a8b tags: %% Cell type:code id:8c411a8b tags:
   
``` python ``` python
dataset_name = '01_generated_2' dataset_name = '01_generated_2'
# dataset_name = '743a5b7d-5dd3-4c9e-9a0d-9f64e1a1bb44' # dataset_name = '743a5b7d-5dd3-4c9e-9a0d-9f64e1a1bb44'
dataset_name = list(collection.keys())[3] dataset_name = list(collection.keys())[3]
dataset = collection[dataset_name] dataset = collection[dataset_name]
# dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1'] # dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1']
dataset.pseudo_labels.set_all_noise() dataset.pseudo_labels.set_all_noise()
dataset.pseudo_labels.correct_hw_by_convlstmae3_filter() dataset.pseudo_labels.correct_hw_by_convlstmae3_filter()
dataset.pseudo_labels.correct_by_feedback_noise() dataset.pseudo_labels.correct_by_feedback_noise()
ax = dataset.plot_windows() ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax) dataset.plot_feedback_areas(ax)
dataset.plot_pseudo_labels(ax) dataset.plot_pseudo_labels(ax)
dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax) dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax)
evaluations = dataset.pseudo_labels.evaluate(dataset.y_win) evaluations = dataset.pseudo_labels.evaluate(dataset.y_win)
print(evaluations[2].get_values()) print(evaluations[2].get_values())
``` ```
   
%% Cell type:code id:4ad85dd4 tags: %% Cell type:code id:4ad85dd4 tags:
   
``` python ``` python
dataset_name = '01_generated_2' dataset_name = '01_generated_2'
# dataset_name = '743a5b7d-5dd3-4c9e-9a0d-9f64e1a1bb44' # dataset_name = '743a5b7d-5dd3-4c9e-9a0d-9f64e1a1bb44'
dataset_name = list(collection.keys())[4] dataset_name = list(collection.keys())[4]
dataset = collection[dataset_name] dataset = collection[dataset_name]
# dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1'] # dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1']
dataset.pseudo_labels.reset() dataset.pseudo_labels.reset()
#dataset.pseudo_labels.scope[:] = False #dataset.pseudo_labels.scope[:] = False
#dataset.pseudo_labels.set_scope_to_feedback_windows() #dataset.pseudo_labels.set_scope_to_feedback_windows()
#dataset.pseudo_labels.correct_all_by_neepconv_filter() #dataset.pseudo_labels.correct_all_by_neepconv_filter()
#dataset.pseudo_labels.set_all_noise() #dataset.pseudo_labels.set_all_noise()
#dataset.pseudo_labels.include_high_confidence_areas() #dataset.pseudo_labels.include_high_confidence_areas()
# dataset.pseudo_labels.correct_hw_by_convlstmae3_filter() # dataset.pseudo_labels.correct_hw_by_convlstmae3_filter()
#dataset.pseudo_labels.correct_hw_by_convlstmae3_filter() #dataset.pseudo_labels.correct_hw_by_convlstmae3_filter()
#dataset.pseudo_labels.correct_by_feedback_noise() #dataset.pseudo_labels.correct_by_feedback_noise()
#dataset.pseudo_labels.exclude_neuts_from_scope() #dataset.pseudo_labels.exclude_neuts_from_scope()
dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbyfcndaefiltertest']) dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbyfcndaefiltertest'])
#dataset.pseudo_labels.exclude_uncovered_noise() #dataset.pseudo_labels.exclude_uncovered_noise()
#dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbyconvlstm3filter']) #dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbyconvlstm3filter'])
ax = dataset.plot_windows() ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax) dataset.plot_feedback_areas(ax)
dataset.plot_pseudo_labels(ax) dataset.plot_pseudo_labels(ax)
dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax) dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax)
#dataset.plot_prediction(predicitons[dataset_name], ax) #dataset.plot_prediction(predicitons[dataset_name], ax)
evaluations = dataset.pseudo_labels.evaluate(dataset.y_win) evaluations = dataset.pseudo_labels.evaluate(dataset.y_win)
print(evaluations[2].get_values()) print(evaluations[2].get_values())
``` ```
   
%% Output %% Output
   
   
   
('_pseudo', 0.9992361561097568, 0.7534647974211516, 0.7882134641560624, 0.8591189117828607) ('_pseudo', 0.9992361561097568, 0.7534647974211516, 0.7882134641560624, 0.8591189117828607)
   
%% Cell type:code id:7971e435 tags: %% Cell type:code id:7971e435 tags:
   
``` python ``` python
   
dataset_name = list(collection.keys())[8] dataset_name = list(collection.keys())[8]
dataset = collection[dataset_name] dataset = collection[dataset_name]
dataset.pseudo_labels.reset() dataset.pseudo_labels.reset()
dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbydeepconvfilter']) dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbydeepconvfilter'])
ax = dataset.plot_windows() ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax) dataset.plot_feedback_areas(ax)
dataset.plot_pseudo_labels(ax) dataset.plot_pseudo_labels(ax)
#dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax) #dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax)
evaluations = dataset.pseudo_labels.evaluate(dataset.y_win) evaluations = dataset.pseudo_labels.evaluate(dataset.y_win)
print(evaluations[2].get_values()) print(evaluations[2].get_values())
``` ```
   
%% Output %% Output
   
(60, 20, 2) [8005, 8085] (60, 20, 2) [8005, 8085]
(60, 20, 2) [10039, 10119] (60, 20, 2) [10039, 10119]
(60, 20, 2) [11291, 11371] (60, 20, 2) [11291, 11371]
(60, 20, 2) [11296, 11376] (60, 20, 2) [11296, 11376]
(60, 20, 2) [15927, 16007] (60, 20, 2) [15927, 16007]
(60, 20, 2) [18816, 18896] (60, 20, 2) [18816, 18896]
(60, 20, 2) [23213, 23293] (60, 20, 2) [23213, 23293]
(60, 20, 2) [23226, 23306] (60, 20, 2) [23226, 23306]
(60, 20, 2) [24728, 24808] (60, 20, 2) [24728, 24808]
(60, 20, 2) [24744, 24824] (60, 20, 2) [24744, 24824]
(60, 20, 2) [24758, 24838] (60, 20, 2) [24758, 24838]
(60, 20, 2) [24771, 24851] (60, 20, 2) [24771, 24851]
(60, 20, 2) [26192, 26272] (60, 20, 2) [26192, 26272]
(60, 20, 2) [27302, 27382] (60, 20, 2) [27302, 27382]
(60, 20, 2) [27309, 27389] (60, 20, 2) [27309, 27389]
(60, 20, 2) [31745, 31825] (60, 20, 2) [31745, 31825]
(0,) [3315, 3322] (0,) [3315, 3322]
   
--------------------------------------------------------------------------- ---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last) RuntimeError Traceback (most recent call last)
<ipython-input-31-1858473e0362> in <module> <ipython-input-31-1858473e0362> in <module>
2 dataset = collection[dataset_name] 2 dataset = collection[dataset_name]
3 dataset.pseudo_labels.reset() 3 dataset.pseudo_labels.reset()
----> 4 dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbydeepconvfilter']) ----> 4 dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbydeepconvfilter'])
5 ax = dataset.plot_windows() 5 ax = dataset.plot_windows()
6 dataset.plot_feedback_areas(ax) 6 dataset.plot_feedback_areas(ax)
~/gitrepos/uni/handwashing_personalizer/src/tools/dataset.py in apply_pseudo_label_generators(self, generators) ~/gitrepos/uni/handwashing_personalizer/src/tools/dataset.py in apply_pseudo_label_generators(self, generators)
196 196
197 def apply_pseudo_label_generators(self, generators: List[PseudoLabelGenerator]): 197 def apply_pseudo_label_generators(self, generators: List[PseudoLabelGenerator]):
--> 198 self.pseudo_labels.apply_generators(generators, self.y_win) --> 198 self.pseudo_labels.apply_generators(generators, self.y_win)
199 199
200 200
~/gitrepos/uni/handwashing_personalizer/src/tools/pseudo_labels.py in apply_generators(self, generators, ground_truth, do_reset) ~/gitrepos/uni/handwashing_personalizer/src/tools/pseudo_labels.py in apply_generators(self, generators, ground_truth, do_reset)
453 self.correct_by_feedback_noise() 453 self.correct_by_feedback_noise()
454 if generator == PseudoLabelGenerator.CorrectByDeepConvFilter: 454 if generator == PseudoLabelGenerator.CorrectByDeepConvFilter:
--> 455 self.correct_hw_by_neepconv_filter() --> 455 self.correct_hw_by_neepconv_filter()
456 if generator == PseudoLabelGenerator.CorrectByFCNDAEFilter: 456 if generator == PseudoLabelGenerator.CorrectByFCNDAEFilter:
457 self.correct_hw_by_fcndae_filter() 457 self.correct_hw_by_fcndae_filter()
~/gitrepos/uni/handwashing_personalizer/src/tools/pseudo_labels.py in correct_hw_by_neepconv_filter(self, model_name) ~/gitrepos/uni/handwashing_personalizer/src/tools/pseudo_labels.py in correct_hw_by_neepconv_filter(self, model_name)
318 with torch.no_grad(): 318 with torch.no_grad():
319 x_pred = torch.Tensor(x_data) 319 x_pred = torch.Tensor(x_data)
--> 320 pred = model(x_pred.to("cpu")).detach().cpu().numpy() --> 320 pred = model(x_pred.to("cpu")).detach().cpu().numpy()
321 del x_pred 321 del x_pred
322 322
~/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs) ~/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks 1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks): 1101 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102 return forward_call(*input, **kwargs) -> 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used 1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], [] 1104 full_backward_hooks, non_full_backward_hooks = [], []
~/anaconda3/lib/python3.8/site-packages/torch/nn/modules/container.py in forward(self, input) ~/anaconda3/lib/python3.8/site-packages/torch/nn/modules/container.py in forward(self, input)
139 def forward(self, input): 139 def forward(self, input):
140 for module in self: 140 for module in self:
--> 141 input = module(input) --> 141 input = module(input)
142 return input 142 return input
143 143
~/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs) ~/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks 1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks): 1101 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102 return forward_call(*input, **kwargs) -> 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used 1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], [] 1104 full_backward_hooks, non_full_backward_hooks = [], []
~/gitrepos/uni/handwashing_personalizer/src/tools/models.py in forward(self, x) ~/gitrepos/uni/handwashing_personalizer/src/tools/models.py in forward(self, x)
276 x = layer(x).relu() 276 x = layer(x).relu()
277 #print('conv:',x.shape) 277 #print('conv:',x.shape)
--> 278 x = x.view(x.size(0), -1) --> 278 x = x.view(x.size(0), -1)
279 #print('view:',x.shape) 279 #print('view:',x.shape)
280 output = self.output(x) 280 output = self.output(x)
RuntimeError: cannot reshape tensor of 0 elements into shape [0, -1] because the unspecified dimension size -1 can be any value and is ambiguous RuntimeError: cannot reshape tensor of 0 elements into shape [0, -1] because the unspecified dimension size -1 can be any value and is ambiguous
   
%% Cell type:code id:4a9fcac7 tags: %% Cell type:code id:4a9fcac7 tags:
   
``` python ``` python
('_pseudo', 0.992690131515155, 0.9200791781446349, 0.8786648353672806, 0.9550064566010888) ('_pseudo', 0.992690131515155, 0.9200791781446349, 0.8786648353672806, 0.9550064566010888)
('_pseudo', 0.9979044655960941, 0.5068569098133595, 0.55256521407621, 0.6722591129551805) ('_pseudo', 0.9979044655960941, 0.5068569098133595, 0.55256521407621, 0.6722591129551805)
('_pseudo', 0.9870815879400368, 0.4673015515306359, 0.32977055949516393, 0.6343098252632132) ('_pseudo', 0.9870815879400368, 0.4673015515306359, 0.32977055949516393, 0.6343098252632132)
('_pseudo', 0.9995293911532238, 0.9398119487533612, 0.9327355369807832, 0.9687512410592143) ('_pseudo', 0.9995293911532238, 0.9398119487533612, 0.9327355369807832, 0.9687512410592143)
``` ```
   
%% Cell type:code id:8103d666 tags: %% Cell type:code id:8103d666 tags:
   
``` python ``` python
dataset_name = list(collection.keys())[1] dataset_name = list(collection.keys())[1]
dataset = collection[dataset_name] dataset = collection[dataset_name]
print(dataset.indicators) print(dataset.indicators)
dataset.indicators[1][1, 1] = 1 dataset.indicators[1][1, 1] = 1
``` ```
   
%% Cell type:code id:c34adbc5 tags: %% Cell type:code id:c34adbc5 tags:
   
``` python ``` python
dataset_manager.__db_update__() dataset_manager.__db_update__()
``` ```
   
%% Cell type:code id:9d6edc9c tags: %% Cell type:code id:9d6edc9c tags:
   
``` python ``` python
print(dataset.x_data.shape, dataset.x_win.shape) print(dataset.x_data.shape, dataset.x_win.shape)
``` ```
   
%% Output %% Output
   
(2393112, 6) (31907, 150, 6) (2393112, 6) (31907, 150, 6)
   
%% Cell type:code id:29ee464f tags: %% Cell type:code id:29ee464f tags:
   
``` python ``` python
predicitons[dataset_name].shape predicitons[dataset_name].shape
``` ```
   
%% Output %% Output
   
(31907, 2) (31907, 2)
   
%% Cell type:code id:aa4f8a2c tags: %% Cell type:code id:aa4f8a2c tags:
   
``` python ``` python
``` ```
...@@ -223,6 +223,48 @@ class Dataset: ...@@ -223,6 +223,48 @@ class Dataset:
self.pseudo_labels.apply_generators(generators, self.y_win) self.pseudo_labels.apply_generators(generators, self.y_win)
def randomize_evaluation(self, evaluation_reliability_no=1, evaluation_reliability_yes=1):
feedback = self.indicators[1].copy()
hw_indicators_hw = feedback[feedback[:, 1] == Indicators.HAND_WASH]
hw_indicators_neut = feedback[feedback[:, 1] == Indicators.NEUTRAL]
hw_indicators_noise = feedback[feedback[:, 1] == Indicators.NOISE]
#print('HW:', hw_indicators_hw.shape, 'NOISE:', hw_indicators_noise.shape)
consistent_indicators_hw_indexes = np.random.choice(np.arange(hw_indicators_hw.shape[0]),
size=int(np.ceil(hw_indicators_hw.shape[0]*evaluation_reliability_yes)),
replace=False)
consistent_indicators_noise_indexes = np.random.choice(np.arange(hw_indicators_noise.shape[0]),
size=int(np.ceil(hw_indicators_noise.shape[0] * evaluation_reliability_no)),
replace=False)
#print('take hw', hw_indicators_hw[consistent_indicators_hw_indexes], 'take noise:', hw_indicators_noise[consistent_indicators_noise_indexes])
exclude_hw_indexes = [index for index in np.arange(hw_indicators_hw.shape[0]) if index not in consistent_indicators_hw_indexes]
#print('exclude hw:', exclude_hw_indexes)
exclude_noise_indexes = [index for index in np.arange(hw_indicators_noise.shape[0]) if index not in consistent_indicators_noise_indexes]
#print('exclude noise:', exclude_noise_indexes)
# hw_indicators_neut = np.concatenate((hw_indicators_neut, hw_indicators_hw[np.where(consistent_indicators_hw not in hw_indicators_hw)]))
hw_indicators_neut = np.concatenate((hw_indicators_neut, hw_indicators_hw[exclude_hw_indexes], hw_indicators_noise[exclude_noise_indexes]))