Commit 4fec348f authored by Alexander Henkel's avatar Alexander Henkel
Browse files

work on pseudo filters

parent 9cc4aa18
......@@ -42,7 +42,11 @@ In the next step I want to observe if smoothing could have a negative effect if
\section{Evaluation of different Pseudo label generations}
In this section, I describe the evaluation of different pseudo labeling approaches using the filters introduced in \secref{sec:approachFilterConfigurations}. For each filter configuration, the base model is used to predict the labels of the training sets and create pseudo labels. After that the filter is applied to the pseudo labels. To determine the quality of the pseudo labels, they are evaluated against the ground truth values using soft versions of the metrics $Sensitivity^{soft}$, $Specificity^{soft}$, $F_1^{soft}$, $S^{soft}$. The general model is then trained by the refined pseudo labels. All resulted models are evaluated by their test sets and the mean over all is computed.
In this section, I describe the evaluation of different pseudo labeling approaches using the filters introduced in \secref{sec:approachFilterConfigurations}. For each filter configuration, the base model is used to predict the labels of the training sets and create pseudo labels. After that the filter is applied to the pseudo labels. To determine the quality of the pseudo labels, they are evaluated against the ground truth values using soft versions of the metrics $Sensitivity^{soft}$, $Specificity^{soft}$, $F_1^{soft}$, $S^{soft}$. The general model is then trained by the refined pseudo labels. All resulted models are evaluated by their test sets and the mean over all is computed. \figref{fig:pseudoModelsEvaluation} shows a bar plot over the metrics for all filter configuration. I concentrate to the values of S score. The configurations \texttt{all}, \texttt{high\_conf}, \texttt{scope}, \texttt{all\_corrected\_null}, \texttt{scope\_corrected\_null}, \texttt{all\_corrected\_null\_hwgt}, \texttt{scope\_corrected\_null\_hwgt} lead to a lower performance than the base model. Insights gives \figref{fig:pseudoModelsTrainingData} (b). All of the generated training data contain false positive labels, i.e. \textit{null} samples which are labeled as hand washing, or there are just a few true negative labels. For all \texttt{all\_null\_*} configurations nearly all not hand washing samples are correctly labeled as \textit{null}. More over, the training data consists of slightly more true positive labels than the others. The resulted models reach almost the same performance as the supervised trained model.
\input{figures/experiments/supervised_pseudo_models}
\input{figures/experiments/supervised_pseudo_models_training_data}
\begin{itemize}
\item Setup of filters
\item Comparison
......
\begin{figure}[t]
\begin{centering}
\subfloat
{\includegraphics[width=\textwidth]{figures/experiments/supervised_pseudo_models_sen_spec.png}}
\subfloat
{\includegraphics[width=\textwidth]{figures/experiments/supervised_pseudo_models_f1_s.png}}
\caption[Pseudo models evaluation]{\textbf{Pseudo models evaluation} Measured metrics on models which have been trained on different filter configurations of training data labels. In plot of S score, the evaluation of respective training data is included. The scaled evaluations part is scaled to the ratio of training data labels to ground truth labels.}
\label{fig:pseudoModelsEvaluation}
\end{centering}
\end{figure}
\begin{figure}[t]
\begin{centering}
\subfloat[evaluation]
{\includegraphics[width=\textwidth]{figures/experiments/supervised_pseudo_models_training_data_evaluation.png}}
\subfloat[training data]
{\includegraphics[width=\textwidth]{figures/experiments/supervised_pseudo_models_training_data_confusion.png}}
\caption[Pseudo models training data]{\textbf{Pseudo models training data} Analysis of training data which was used to train previous models. Brighter parts in (a) and values in (b) are scaled to the respective ratio of training data labels to ground truth labels.}
\label{fig:pseudoModelsTrainingData}
\end{centering}
\end{figure}
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
%% Cell type:code id:9e1924eb tags:
 
``` python
%load_ext autoreload
%autoreload 2
 
%matplotlib notebook
```
 
%% Cell type:code id:32100899 tags:
 
``` python
import sys
import os
import numpy as np
import matplotlib.pyplot as plt
from IPython.display import display, Markdown
import copy
import torch
import pandas as pd
```
 
%% Cell type:code id:162a8662 tags:
 
``` python
module_path = os.path.abspath(os.path.join('..'))
os.chdir(module_path)
if module_path not in sys.path:
sys.path.append(module_path)
```
 
%% Cell type:code id:8e909dd4 tags:
 
``` python
from tools.load_data_sets import *
from tools.helpers import *
from tools.learner_pipeline import LearnerPipeline, Evaluation
from tools.dataset_builder import *
from tools.model_evaluation import ModelEvaluation
from tools.sensor_recorder_data_reader import SensorRecorderDataReader
from tools.dataset_manager import DatasetManager
from tools.dataset import SyntheticDataset
from tools.models import HandWashingDeepConvLSTMA
from tools.metrics import sensitivity, specificity, S1_score
from personalization_tools.load_data_sets import *
from personalization_tools.helpers import *
from personalization_tools.learner_pipeline import LearnerPipeline, Evaluation
from personalization_tools.dataset_builder import *
from personalization_tools.model_evaluation import ModelEvaluation
from personalization_tools.sensor_recorder_data_reader import SensorRecorderDataReader
from personalization_tools.dataset_manager import DatasetManager
from personalization_tools.dataset import SyntheticDataset
from personalization_tools.models import HandWashingDeepConvLSTMA
from personalization_tools.metrics import sensitivity, specificity, S1_score
from sklearn.metrics import f1_score
from tools.pseudo_labels import PseudoLabelGenerator
from tools.pseudo_label_evaluation import calc_evaluations_average, generate_pseudo_label_evaluations, get_avg_pseudo_evaluation_of_collection
from tools.pseudo_model_settings import pseudo_model_settings, pseudo_label_generators_to_int
from tools.globals import Indicators
from personalization_tools.pseudo_labels import PseudoLabelGenerator
from personalization_tools.pseudo_label_evaluation import calc_evaluations_average, generate_pseudo_label_evaluations, get_avg_pseudo_evaluation_of_collection
from personalization_tools.pseudo_model_settings import pseudo_model_settings, pseudo_label_generators_to_int
from personalization_tools.globals import Indicators
```
 
%% Cell type:code id:3ac7a322 tags:
 
``` python
dataset_db = './data/synthetic_dataset_db_2'
base_model = './data/HandWashingDeepConvLSTMA_trunc_11.pt'
#collection_name = 'base_synthetic_01'
dataset_db = './data/recorded_dataset_db_2'
dataset_db = './data/recorded_dataset_db'
base_model = './data/DeepConvLSTMA_statedict.pt'
collection_name = '01_training'
```
 
%% Cell type:code id:713a099c tags:
 
``` python
dataset_manager = DatasetManager(dataset_db)
print(dataset_manager.show_categories())
```
 
%% Output
 
['01_all', '01_test', '01_training']
['01_training', '01_test', '01_all', '02_training', '02_test', '02_all']
 
%% Cell type:code id:276d5c35 tags:
 
``` python
collection = dataset_manager.filter_by_category(collection_name)
print(collection.keys())
```
 
%% Output
 
dict_keys(['cf0f18c5-a4e8-489c-b38b-af8cd0067000', '43de4036-a1e1-4aab-a3ce-671bcef3df0c', '0d07466e-797e-4663-9779-f16bc2b9ce86', '7a069df0-6a1f-41a9-b5b4-6c576651ccf7', '6370f5e9-81ea-45dc-a126-d55566b64f1f', '402b4d56-4b67-446c-a929-63ecb0ec58c7', '92d09936-6ba6-4183-a5ec-6bb8b7ac701f', '0be9f1e5-99b4-4b57-82f2-670420a7278c', '4c82aa1d-ed11-4d46-bde4-31f5b003ceab', '0c53f7b4-7aaa-4022-b3fb-acf7f523c30d', '57c3e2b9-ed9e-4140-9d13-ef31a6911b42', '65d3d0e6-389a-48f2-b02e-fcd3eb058e0a', '959a3761-3af5-4957-8d04-177ba5cfef75', 'f9f676bd-b997-46c6-a654-8a9a3c0ba583', 'a3187994-6574-4344-bcbb-001e90a00ff9', '9485d9c0-e3eb-4d31-8ae3-c843bca1ab7c'])
dict_keys(['34a0c627-45a9-4b96-9900-497907da94b1', 'b7584e6d-e21d-45cd-b1f7-3bac7b4c2612', 'e49a7273-65aa-40fa-bbb6-ef6a795f8f2b', '9045e352-bb8f-4dfa-956d-405e12987125', 'af233ab2-28b1-4b7f-8a02-6cd8f9b61311'])
 
%% Cell type:code id:8dafeb13 tags:
 
``` python
predicitons = generate_predictions(collection, base_model)
```
 
%% Output
 
run: cf0f18c5-a4e8-489c-b38b-af8cd0067000...0.7893915176391602 seconds
run: 43de4036-a1e1-4aab-a3ce-671bcef3df0c...1.0819857120513916 seconds
run: 0d07466e-797e-4663-9779-f16bc2b9ce86...0.684497594833374 seconds
run: 7a069df0-6a1f-41a9-b5b4-6c576651ccf7...1.3040337562561035 seconds
run: 6370f5e9-81ea-45dc-a126-d55566b64f1f...1.108980417251587 seconds
run: 402b4d56-4b67-446c-a929-63ecb0ec58c7...1.282562255859375 seconds
run: 92d09936-6ba6-4183-a5ec-6bb8b7ac701f...0.8270609378814697 seconds
run: 0be9f1e5-99b4-4b57-82f2-670420a7278c...0.7181124687194824 seconds
run: 4c82aa1d-ed11-4d46-bde4-31f5b003ceab...1.3844504356384277 seconds
run: 0c53f7b4-7aaa-4022-b3fb-acf7f523c30d...0.9225075244903564 seconds
run: 57c3e2b9-ed9e-4140-9d13-ef31a6911b42...0.43587183952331543 seconds
run: 65d3d0e6-389a-48f2-b02e-fcd3eb058e0a...1.3762381076812744 seconds
run: 959a3761-3af5-4957-8d04-177ba5cfef75...1.064845085144043 seconds
run: f9f676bd-b997-46c6-a654-8a9a3c0ba583...0.8436729907989502 seconds
run: a3187994-6574-4344-bcbb-001e90a00ff9...0.32437992095947266 seconds
run: 9485d9c0-e3eb-4d31-8ae3-c843bca1ab7c...1.1401820182800293 seconds
run prediction on [cuda]: 34a0c627-45a9-4b96-9900-497907da94b1...0.6643788814544678 seconds
run prediction on [cuda]: b7584e6d-e21d-45cd-b1f7-3bac7b4c2612...0.028804779052734375 seconds
run prediction on [cuda]: e49a7273-65aa-40fa-bbb6-ef6a795f8f2b...0.19591903686523438 seconds
run prediction on [cuda]: 9045e352-bb8f-4dfa-956d-405e12987125...0.4017653465270996 seconds
run prediction on [cuda]: af233ab2-28b1-4b7f-8a02-6cd8f9b61311...0.7093448638916016 seconds
%% Cell type:code id:f1cbf188 tags:
``` python
for dataset in collection.values():
#dataset.plot_windows()
print('noise', np.count_nonzero(dataset.indicators[1][:, 1] == Indicators.NOISE), 'hw', np.count_nonzero(dataset.indicators[1][:, 1] == Indicators.HAND_WASH))
```
%% Output
noise 12 hw 5
noise 0 hw 2
noise 1 hw 2
noise 1 hw 1
noise 12 hw 5
%% Cell type:code id:6f293240 tags:
``` python
for dataset in collection.values():
dataset.randomize_evaluation(0.8, 0.5)
```
%% Cell type:code id:647d9ced tags:
``` python
for dataset in collection.values():
#dataset.plot_windows()
print('noise', np.count_nonzero(dataset.indicators[1][:, 1] == Indicators.NOISE), 'hw', np.count_nonzero(dataset.indicators[1][:, 1] == Indicators.HAND_WASH))
```
%% Output
noise 10 hw 3
noise 0 hw 1
noise 1 hw 1
noise 1 hw 1
noise 10 hw 3
 
%% Cell type:code id:5643b323 tags:
 
``` python
#dataset_manager.update_dataset_collection(collection.values())
#dataset_manager.__db_update__()
```
 
%% Cell type:code id:c1d63a1c tags:
 
``` python
for dataset in collection.values():
print(dataset.name)
if isinstance(dataset, SyntheticDataset):
print('do')
#dataset.indicators[0] -= 30
#dataset.indicators[1][:, 0] -= 30
dataset.indicators_synthetic = None
dataset.generate_synthetic_evaluation(predicitons[dataset.name], evaluation_reliability_no=1, evaluation_reliability_yes=1)
# dataset.indicators_synthetic[0] = np.zeros((0))
# dataset.indicators[0] -= 60
# dataset.indicators[1][:, 0] -= 60
dataset.generate_feedback_areas(prediction=predicitons[dataset.name])
# dataset_manager.__db_update__()
```
 
%% Output
 
11_generated_3
do
11_generated_4
do
11_generated_5
do
11_generated_6
do
11_generated_7
do
11_generated_8
do
11_generated_9
do
11_generated_10
do
11_generated_11
do
11_generated_12
do
11_generated_13
do
11_generated_14
do
11_generated_15
do
11_generated_16
do
 
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-12-9ee5cfafcc83> in <module>
6 #dataset.indicators[1][:, 0] -= 30
7 dataset.indicators_synthetic = None
----> 8 dataset.generate_synthetic_evaluation(predicitons[dataset.name], evaluation_reliability_no=1, evaluation_reliability_yes=1)
9 # dataset.indicators_synthetic[0] = np.zeros((0))
10 # dataset.indicators[0] -= 60
~/gitrepos/uni/handwashing_personalizer/src/tools/dataset.py in generate_synthetic_evaluation(self, y_predicted, kernel_size, kernel_threshold, evaluation_reliability_no, evaluation_reliability_yes, consider_actions, clear_manual, clear_just_covered)
318 if len(manual_indicators) > 0:
319 # print(evaluation_indices, manual_indicators)
--> 320 evaluation_indices = np.append(evaluation_indices, np.asarray([(i, Indicators.HAND_WASH) for i in manual_indicators]), axis=0)
321 evaluation_indices = evaluation_indices[evaluation_indices[:, 0].argsort()]
322 self.indicators_synthetic[0] = np.asarray(manual_indicators)
<__array_function__ internals> in append(*args, **kwargs)
~/.local/lib/python3.8/site-packages/numpy/lib/function_base.py in append(arr, values, axis)
4815 values = ravel(values)
4816 axis = arr.ndim-1
-> 4817 return concatenate((arr, values), axis=axis)
4818
4819
<__array_function__ internals> in concatenate(*args, **kwargs)
ValueError: all the input arrays must have same number of dimensions, but the array at index 0 has 1 dimension(s) and the array at index 1 has 2 dimension(s)
 
%% Cell type:code id:816fbb1d tags:
 
``` python
requests = 0
 
observed_collection = list(collection.values())
for dataset in observed_collection:
evaluations = dataset.get_indicators()[1]
for evaluation in evaluations:
if evaluation[1] != Indicators.NEUTRAL:
requests += 1
print('Num requests:', requests)
print('requests per dataset:', requests/len(observed_collection))
```
 
%% Output
 
Num requests: 28
requests per dataset: 5.6
 
%% Cell type:code id:55f54903 tags:
 
``` python
dataset_name = list(collection.keys())[0]
#print(dataset_name)
dataset = collection[dataset_name]
#print(dataset.indicators)
#print(dataset.x_win.shape, dataset.y_data.shape)
#print(dataset.indicators)
ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax)
print(dataset.indicators_synthetic)
```
 
%% Output
 
 
 
[array([7310]), array([[ 905, 0],
[ 1143, -1],
[ 1164, -1],
[ 1185, -1],
[ 1206, -1],
[ 1227, 0],
[ 1555, 0],
[ 3786, 0],
[ 4201, -1],
[ 4222, 1],
[ 7310, 1],
[ 9517, 0],
[ 9542, -1],
[ 9563, -1],
[ 9584, -1],
[ 9605, -1],
[ 9626, 0],
[ 9670, -1],
[ 9691, 0],
[ 9811, 0],
[ 9972, 0],
[10219, 0],
[10245, 0],
[10551, -1],
[10573, -1],
[10594, -1],
[10618, -1],
[10639, -1],
[10662, -1],
[10683, 0],
[10865, -1],
[10886, 0],
[11102, -1],
[11123, 0],
[11175, 1],
[11435, 0],
[11913, 0],
[11942, -1],
[11963, -1],
[11984, -1],
[12005, -1],
[12026, -1],
[12047, -1],
[12068, 0],
[12169, -1],
[12190, 0],
[12234, -1],
[12255, -1],
[12276, -1],
[12297, -1],
[12318, -1],
[12339, 0],
[13149, 0],
[13243, 0],
[13597, 0],
[13637, 0],
[13726, 0],
[13912, -1],
[13933, 1],
[15866, -1],
[15887, 0],
[15942, 0],
[15992, -1],
[16013, -1],
[16034, 0],
[16121, 0],
[16158, -1],
[16179, -1],
[16201, -1],
[16222, -1],
[16243, 0],
[16296, 0],
[16343, 0],
[16389, -1],
[16410, -1],
[16431, -1],
[16452, -1],
[16473, -1],
[16494, -1],
[16515, 0]])]
 
%% Cell type:code id:1da7daa6 tags:
 
``` python
relevant_pseudo_models = ['all', 'scope', 'all_corrected', 'scope_corrected', 'synthetic', 'allnoise_correctedhwgt_scope', 'allnoise_correctedscore', 'allnoise_correctedscore_flatten', 'allnoise_correctbydeepconvfilter', 'allnoise_correctbyfcndaefilter', 'allnoise_correctbyfcndaefiltertest']
relevant_pseudo_models = ['allnoise_correctedscore', 'allnoise_correctbydeepconvfilter', 'allnoise_correctbyfcndaefilter', 'allnoise_correctbyfcndaefiltertest', 'allnoise_correctbyconvlstmfilter', 'allnoise_correctbyconvlstm2filter', 'allnoise_correctbyconvlstm3filter', 'alldeepconv_correctbyconvlstm3filter', 'alldeepconv_correctbyconvlstm3filter4', 'alldeepconv_correctbyconvlstm3filter5','alldeepconv_correctbyconvlstm3filter6']
```
 
%% Cell type:code id:087a3c33 tags:
 
``` python
def plot_pseudo_labels(dataset_name, observed_models):
evaluations = []
fig, axes = plt.subplots(len(observed_models), figsize=(8,26))
for i, model_setting in enumerate(observed_models):
print(model_setting)
collection[dataset_name].apply_pseudo_label_generators(pseudo_model_settings[model_setting])
evals = collection[dataset_name].evaluate_pseudo_labels(model_setting)
ax = collection[dataset_name].plot_feedback_areas(ax=axes[i])
collection[dataset_name].plot_pseudo_labels(ax=ax)
evaluations.append(evals)
ax.set_title(model_setting)
 
fig.tight_layout()
fig.show()
return evaluations
 
def generate_pseudo_labels(dataset_name, observed_models):
evaluations = []
for i, model_setting in enumerate(observed_models):
collection[dataset_name].apply_pseudo_label_generators(pseudo_model_settings[model_setting])
evals = collection[dataset_name].evaluate_pseudo_labels(model_setting)
evaluations.append(evals)
 
 
return evaluations
```
 
%% Cell type:code id:d66ef0c4 tags:
 
``` python
evaluations = plot_pseudo_labels('01_generated_3', relevant_pseudo_models)
```
 
%% Cell type:code id:d63549e1 tags:
 
``` python
gen_evaluations = dict()
for dataset in list(collection.values()):
gen_evaluations[dataset.name] = generate_pseudo_label_evaluations(collection[dataset.name], relevant_pseudo_models)
 
averages = calc_evaluations_average(list(gen_evaluations.values()))
```
 
%% Cell type:code id:181e9e74 tags:
 
``` python
series = []
for evaluation in evaluations:
series.append(evaluation[0].get_values())
series.append(evaluation[1].get_values())
series.append(evaluation[2].get_values())
#series = (eval_base.get_values(), eval_pseudo.get_values(), eval_base_scoped.get_values(), eval_pseudo_scoped.get_values(), eval_base_scoped_extneut.get_values(), eval_pseudo_scoped_extneut.get_values())
df = pd.DataFrame(series, columns=['labels', 'specificity', 'sensitivity', 'f1', 'S1'])
df
```
 
%% Cell type:code id:7fde8f11 tags:
 
``` python
dataframes = []
for dataset_evaluation in evaluations.values():
series = []
for evaluation in dataset_evaluation:
series.append(evaluation[0].get_values())
series.append(evaluation[1].get_values())
#series = (eval_base.get_values(), eval_pseudo.get_values(), eval_base_scoped.get_values(), eval_pseudo_scoped.get_values(), eval_base_scoped_extneut.get_values(), eval_pseudo_scoped_extneut.get_values())
df = pd.DataFrame(series, columns=['labels', 'specificity', 'sensitivity', 'f1', 'S1'])
dataframes.append(df)
```
 
%% Cell type:code id:02fc7bf3 tags:
 
``` python
 
dataframes[0]
```
 
%% Cell type:code id:f78bd2c7 tags:
 
``` python
def plot_pseudo_label_stats_single(evaluations, target_models, target_value):
fig, ax = plt.subplots(figsize=(8, 6))
ax.set_title(target_value)
y_vals_hard = []
y_vals_soft = []
for i in range(len(target_models)):
y_vals_hard.append(getattr(evaluations[i][1], target_value))
y_vals_soft.append(getattr(evaluations[i][2], target_value))
ax.plot(target_models, y_vals_hard, label='hard')
ax.plot(target_models, y_vals_soft, label='soft')
ax.tick_params(labelrotation=45)
fig.legend()
ax.grid()
fig.show()
 
def plot_pseudo_label_stats(evaluations, target_models, target_value):
fig, ax = plt.subplots()
ax.set_title(target_value)
for dataset, evals in evaluations.items():
y_vals_hard = []
y_vals_soft = []
for i in range(len(target_models)):
y_vals_hard.append(getattr(evals[i][1], target_value))
y_vals_soft.append(getattr(evals[i][2], target_value))
#ax.plot(target_models, y_vals_hard, label='hard')
ax.plot(target_models, y_vals_soft, label='soft')
ax.tick_params(labelrotation=45)
fig.legend()
fig.show()
 
 
def plot_pseudo_label_averages(evaluations, target_models, target_value, ax=None):
if ax is None:
fig, axis = plt.subplots()
else:
axis = ax
axis.set_title(target_value)
y_vals_soft = []
for model_evals in evaluations:
y_vals_soft.append(model_evals[2][target_value])
 
#ax.plot(target_models, y_vals_hard, label='hard')
axis.plot(target_models, y_vals_soft, label='soft')
axis.tick_params(labelrotation=45)
axis.grid()
if ax is None:
fig.legend()
fig.show()
return axis
 
```
 
%% Cell type:code id:857e4701 tags:
 
``` python
gen_evaluations = dict()
for dataset in list(collection.values()):
gen_evaluations[dataset.name] = generate_pseudo_label_evaluations(collection[dataset.name], relevant_pseudo_models)
 
averages = calc_evaluations_average(list(gen_evaluations.values()))
 
fig, ax = plt.subplots(4, figsize=(8,20))
 
plot_pseudo_label_averages(averages, relevant_pseudo_models, 'specificity', ax[0])
plot_pseudo_label_averages(averages, relevant_pseudo_models, 'sensitivity', ax[1])
plot_pseudo_label_averages(averages, relevant_pseudo_models, 'f1', ax[2])
plot_pseudo_label_averages(averages, relevant_pseudo_models, 's1', ax[3])
plt.subplots_adjust(hspace=0.7)
fig.legend()
fig.show()
```
 
%% Output
 
 
 
%% Cell type:code id:df94b630 tags:
 
``` python
plot_pseudo_label_stats(gen_evaluations, relevant_pseudo_models, 'specificity')
plot_pseudo_label_stats(gen_evaluations, relevant_pseudo_models, 'sensitivity')
plot_pseudo_label_stats(gen_evaluations, relevant_pseudo_models, 'f1')
plot_pseudo_label_stats(gen_evaluations, relevant_pseudo_models, 's1')
```
 
%% Cell type:code id:de35210b tags:
 
``` python
```
 
%% Cell type:code id:b1d443c6 tags:
 
``` python
dataset_name = '01_generated_4'
# dataset_name = '34a0c627-45a9-4b96-9900-497907da94b1'
dataset = collection[dataset_name]
# dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1']
dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctedscore'])
ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax)
dataset.plot_pseudo_labels(ax)
dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax=ax)
 
dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbydeepconvfilter'])
ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax)
dataset.plot_pseudo_labels(ax)
dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax=ax)
 
 
plot_pseudo_label_stats_single(gen_evaluations[dataset_name], relevant_pseudo_models, 'specificity')
plt.subplots_adjust(bottom=0.3)
plot_pseudo_label_stats_single(gen_evaluations[dataset_name], relevant_pseudo_models, 'sensitivity')
plt.subplots_adjust(bottom=0.3)
plot_pseudo_label_stats_single(gen_evaluations[dataset_name], relevant_pseudo_models, 'f1')
plt.subplots_adjust(bottom=0.3)
plot_pseudo_label_stats_single(gen_evaluations[dataset_name], relevant_pseudo_models, 's1')
plt.subplots_adjust(bottom=0.3)
```
 
%% Cell type:code id:39e6a390 tags:
 
``` python
def sensitivity_soft(y_true, y_pred):
# r: reference -> y_true
# p: pred -> y_pred
# Sum_n: sum over all samples
diff = np.absolute(y_pred - y_true)
frac = y_true / np.sum(y_true, axis=0)
frac_sum = np.sum(frac*diff, axis=0)
return 1 - frac_sum
 
def specificity_soft(y_true, y_pred):
diff = np.absolute(y_pred - y_true)
frac = (1-y_true) / np.sum(1-y_true, axis=0)
frac_sum = np.sum(frac*diff, axis=0)
return 1 - frac_sum
 
def positive_predicted_value_soft(y_true, y_pred):
diff = np.absolute(y_pred - y_true)
frac = y_pred / np.sum(y_pred, axis=0)
frac_sum = np.sum(frac*diff, axis=0)
return 1 - frac_sum
 
def S1_score_soft(y_true, y_pred):
sens = sensitivity_soft(y_true, y_pred)
spec = specificity_soft(y_true, y_pred)
return 2 * sens * spec / (sens + spec)
 
def f1_score_soft(y_true, y_pred):
# recall = sensitivity
# precision = positive predictive value (PPV)
recall = sensitivity_soft(y_true, y_pred)
precision = positive_predicted_value_soft(y_true, y_pred)
return 2 * (precision * recall) / (precision + recall)
```
 
%% Cell type:code id:7f4e7091 tags:
 
``` python
test_arr_true = np.array(((1,0), (1,0), (1,0), (0,1), (0,1), (1,0), (1,0), (0,1), (1,0), (1,0)))
test_arr_pred = np.array(((1,0), (0,1), (1,0), (0,1), (1,0), (1,0), (0.4,0.6), (0,1), (1,0), (1,0)))
print(sensitivity_soft(test_arr_true, test_arr_pred))
print(specificity_soft(test_arr_true, test_arr_pred))
print(S1_score_soft(test_arr_true, test_arr_pred))
print(f1_score_soft(test_arr_true, test_arr_pred))
```
 
%% Cell type:code id:b6b3275d tags:
 
``` python
print(sensitivity(np.argmax(test_arr_true,axis=1), np.argmax(test_arr_pred,axis=1)))
print(specificity(np.argmax(test_arr_true,axis=1), np.argmax(test_arr_pred,axis=1)))
print(S1_score(np.argmax(test_arr_true,axis=1), np.argmax(test_arr_pred,axis=1)))
print(f1_score(np.argmax(test_arr_true,axis=1), np.argmax(test_arr_pred,axis=1)))
```
 
%% Cell type:code id:2c39e41f tags:
 
``` python
dataset_name = '01_generated_3'
#dataset_name = '34a0c627-45a9-4b96-9900-497907da94b1'
dataset = collection[dataset_name]
# dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1']
dataset.pseudo_labels.set_all_noise()
dataset.pseudo_labels.correct_hw_by_neepconv_filter()
dataset.pseudo_labels.correct_by_feedback_noise()
ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax)
dataset.plot_pseudo_labels(ax)
dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax)
evaluations = dataset.pseudo_labels.evaluate(dataset.y_win)
print(evaluations[2].get_values())
```
 
%% Cell type:code id:9a77ddb9 tags:
 
``` python
evaluations[2].get_values()
```
 
%% Cell type:code id:6438cbdc tags:
 
``` python
train_collection = dataset_manager.filter_by_category('base_synthetic_01_training')
averages = get_avg_pseudo_evaluation_of_collection(train_collection.values())
```
 
%% Cell type:code id:2afc55f3 tags:
 
``` python
averages
```
 
%% Cell type:code id:287caf19 tags:
 
``` python
for dataset in train_collection.values():
dataset.pseudo_labels.reset()
dataset.pseudo_labels.set_all_noise()
dataset.pseudo_labels.correct_hw_by_neepconv_filter(model_name='handwash_deepconv_filter_2.pt')
dataset.pseudo_labels.correct_by_feedback_noise()
averages = get_avg_pseudo_evaluation_of_collection(train_collection.values())
print(averages)
```
 
%% Cell type:code id:1a7e34db tags:
 
``` python
for dataset in train_collection.values():
dataset.pseudo_labels.reset()
dataset.pseudo_labels.set_all_noise()
dataset.pseudo_labels.correct_hw_by_neepconv_filter(model_name='handwash_filter.pt')
dataset.pseudo_labels.correct_by_feedback_noise()
averages = get_avg_pseudo_evaluation_of_collection(train_collection.values())
print(averages)
```
 
%% Cell type:code id:cb856e33 tags:
 
``` python
dataset_name = '01_generated_4'
#dataset_name = '34a0c627-45a9-4b96-9900-497907da94b1'
dataset = collection[dataset_name]
# dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1']
dataset.pseudo_labels.set_all_noise()
dataset.pseudo_labels.correct_hw_by_fcndae_filter_test('handwash_filter_fcndae_1.pt')
dataset.pseudo_labels.correct_by_feedback_noise()
ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax)
dataset.plot_pseudo_labels(ax)
dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax)
evaluations = dataset.pseudo_labels.evaluate(dataset.y_win)
print(evaluations[2].get_values())
```
 
%% Cell type:code id:29394ae9 tags:
 
``` python
('_pseudo', 0.9989988151094844, 0.8930696899131422, 0.8850275892320095, 0.9430689847276482)
('_pseudo', 0.9987164602423361, 0.96314140192924, 0.9318188089206001, 0.9806063835663211)
('_pseudo', 0.9986266540771351, 0.9630133999932197, 0.9281106736362976, 0.980496750635969)
```
 
%% Cell type:code id:8c411a8b tags:
 
``` python
dataset_name = '01_generated_2'
# dataset_name = '743a5b7d-5dd3-4c9e-9a0d-9f64e1a1bb44'
dataset_name = list(collection.keys())[3]
dataset = collection[dataset_name]
# dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1']
dataset.pseudo_labels.set_all_noise()
dataset.pseudo_labels.correct_hw_by_convlstmae3_filter()
dataset.pseudo_labels.correct_by_feedback_noise()
ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax)
dataset.plot_pseudo_labels(ax)
dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax)
evaluations = dataset.pseudo_labels.evaluate(dataset.y_win)
print(evaluations[2].get_values())
```
 
%% Cell type:code id:4ad85dd4 tags:
 
``` python
dataset_name = '01_generated_2'
# dataset_name = '743a5b7d-5dd3-4c9e-9a0d-9f64e1a1bb44'
dataset_name = list(collection.keys())[4]
dataset = collection[dataset_name]
# dataset = collection['34a0c627-45a9-4b96-9900-497907da94b1']
dataset.pseudo_labels.reset()
#dataset.pseudo_labels.scope[:] = False
#dataset.pseudo_labels.set_scope_to_feedback_windows()
#dataset.pseudo_labels.correct_all_by_neepconv_filter()
#dataset.pseudo_labels.set_all_noise()
#dataset.pseudo_labels.include_high_confidence_areas()
# dataset.pseudo_labels.correct_hw_by_convlstmae3_filter()
#dataset.pseudo_labels.correct_hw_by_convlstmae3_filter()
#dataset.pseudo_labels.correct_by_feedback_noise()
#dataset.pseudo_labels.exclude_neuts_from_scope()
dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbyfcndaefiltertest'])
#dataset.pseudo_labels.exclude_uncovered_noise()
#dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbyconvlstm3filter'])
ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax)
dataset.plot_pseudo_labels(ax)
dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax)
#dataset.plot_prediction(predicitons[dataset_name], ax)
evaluations = dataset.pseudo_labels.evaluate(dataset.y_win)
print(evaluations[2].get_values())
```
 
%% Output
 
 
 
('_pseudo', 0.9992361561097568, 0.7534647974211516, 0.7882134641560624, 0.8591189117828607)
 
%% Cell type:code id:7971e435 tags:
 
``` python
 
dataset_name = list(collection.keys())[8]
dataset = collection[dataset_name]
dataset.pseudo_labels.reset()
dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbydeepconvfilter'])
ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax)
dataset.plot_pseudo_labels(ax)
#dataset.plot_prediction_in_feedback_areas(predicitons[dataset_name], ax)
evaluations = dataset.pseudo_labels.evaluate(dataset.y_win)
print(evaluations[2].get_values())
```
 
%% Output
 
(60, 20, 2) [8005, 8085]
(60, 20, 2) [10039, 10119]
(60, 20, 2) [11291, 11371]
(60, 20, 2) [11296, 11376]
(60, 20, 2) [15927, 16007]
(60, 20, 2) [18816, 18896]
(60, 20, 2) [23213, 23293]
(60, 20, 2) [23226, 23306]
(60, 20, 2) [24728, 24808]
(60, 20, 2) [24744, 24824]
(60, 20, 2) [24758, 24838]
(60, 20, 2) [24771, 24851]
(60, 20, 2) [26192, 26272]
(60, 20, 2) [27302, 27382]
(60, 20, 2) [27309, 27389]
(60, 20, 2) [31745, 31825]
(0,) [3315, 3322]
 
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-31-1858473e0362> in <module>
2 dataset = collection[dataset_name]
3 dataset.pseudo_labels.reset()
----> 4 dataset.apply_pseudo_label_generators(pseudo_model_settings['allnoise_correctbydeepconvfilter'])
5 ax = dataset.plot_windows()
6 dataset.plot_feedback_areas(ax)
~/gitrepos/uni/handwashing_personalizer/src/tools/dataset.py in apply_pseudo_label_generators(self, generators)
196
197 def apply_pseudo_label_generators(self, generators: List[PseudoLabelGenerator]):
--> 198 self.pseudo_labels.apply_generators(generators, self.y_win)
199
200
~/gitrepos/uni/handwashing_personalizer/src/tools/pseudo_labels.py in apply_generators(self, generators, ground_truth, do_reset)
453 self.correct_by_feedback_noise()
454 if generator == PseudoLabelGenerator.CorrectByDeepConvFilter:
--> 455 self.correct_hw_by_neepconv_filter()
456 if generator == PseudoLabelGenerator.CorrectByFCNDAEFilter:
457 self.correct_hw_by_fcndae_filter()
~/gitrepos/uni/handwashing_personalizer/src/tools/pseudo_labels.py in correct_hw_by_neepconv_filter(self, model_name)
318 with torch.no_grad():
319 x_pred = torch.Tensor(x_data)
--> 320 pred = model(x_pred.to("cpu")).detach().cpu().numpy()
321 del x_pred
322
~/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], []
~/anaconda3/lib/python3.8/site-packages/torch/nn/modules/container.py in forward(self, input)
139 def forward(self, input):
140 for module in self:
--> 141 input = module(input)
142 return input
143
~/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], []
~/gitrepos/uni/handwashing_personalizer/src/tools/models.py in forward(self, x)
276 x = layer(x).relu()
277 #print('conv:',x.shape)
--> 278 x = x.view(x.size(0), -1)
279 #print('view:',x.shape)
280 output = self.output(x)
RuntimeError: cannot reshape tensor of 0 elements into shape [0, -1] because the unspecified dimension size -1 can be any value and is ambiguous
 
%% Cell type:code id:4a9fcac7 tags:
 
``` python
('_pseudo', 0.992690131515155, 0.9200791781446349, 0.8786648353672806, 0.9550064566010888)
('_pseudo', 0.9979044655960941, 0.5068569098133595, 0.55256521407621, 0.6722591129551805)
('_pseudo', 0.9870815879400368, 0.4673015515306359, 0.32977055949516393, 0.6343098252632132)
('_pseudo', 0.9995293911532238, 0.9398119487533612, 0.9327355369807832, 0.9687512410592143)
```
 
%% Cell type:code id:8103d666 tags:
 
``` python
dataset_name = list(collection.keys())[1]
dataset = collection[dataset_name]
print(dataset.indicators)
dataset.indicators[1][1, 1] = 1
```
 
%% Cell type:code id:c34adbc5 tags:
 
``` python
dataset_manager.__db_update__()
```
 
%% Cell type:code id:9d6edc9c tags:
 
``` python
print(dataset.x_data.shape, dataset.x_win.shape)
```
 
%% Output
 
(2393112, 6) (31907, 150, 6)
 
%% Cell type:code id:29ee464f tags:
 
``` python
predicitons[dataset_name].shape
```
 
%% Output
 
(31907, 2)
 
%% Cell type:code id:aa4f8a2c tags:
 
``` python
```
......@@ -223,6 +223,48 @@ class Dataset:
self.pseudo_labels.apply_generators(generators, self.y_win)
def randomize_evaluation(self, evaluation_reliability_no=1, evaluation_reliability_yes=1):
feedback = self.indicators[1].copy()
hw_indicators_hw = feedback[feedback[:, 1] == Indicators.HAND_WASH]
hw_indicators_neut = feedback[feedback[:, 1] == Indicators.NEUTRAL]
hw_indicators_noise = feedback[feedback[:, 1] == Indicators.NOISE]
#print('HW:', hw_indicators_hw.shape, 'NOISE:', hw_indicators_noise.shape)
consistent_indicators_hw_indexes = np.random.choice(np.arange(hw_indicators_hw.shape[0]),
size=int(np.ceil(hw_indicators_hw.shape[0]*evaluation_reliability_yes)),
replace=False)
consistent_indicators_noise_indexes = np.random.choice(np.arange(hw_indicators_noise.shape[0]),
size=int(np.ceil(hw_indicators_noise.shape[0] * evaluation_reliability_no)),
replace=False)
#print('take hw', hw_indicators_hw[consistent_indicators_hw_indexes], 'take noise:', hw_indicators_noise[consistent_indicators_noise_indexes])
exclude_hw_indexes = [index for index in np.arange(hw_indicators_hw.shape[0]) if index not in consistent_indicators_hw_indexes]
#print('exclude hw:', exclude_hw_indexes)
exclude_noise_indexes = [index for index in np.arange(hw_indicators_noise.shape[0]) if index not in consistent_indicators_noise_indexes]
#print('exclude noise:', exclude_noise_indexes)
# hw_indicators_neut = np.concatenate((hw_indicators_neut, hw_indicators_hw[np.where(consistent_indicators_hw not in hw_indicators_hw)]))
hw_indicators_neut = np.concatenate((hw_indicators_neut, hw_indicators_hw[exclude_hw_indexes], hw_indicators_noise[exclude_noise_indexes]))
hw_indicators_neut[:, 1] = Indicators.NEUTRAL
hw_indicators_hw = hw_indicators_hw[consistent_indicators_hw_indexes]
hw_indicators_noise = hw_indicators_noise[consistent_indicators_noise_indexes]
new_indicators = np.concatenate((hw_indicators_neut, hw_indicators_hw, hw_indicators_noise))
new_indicators = new_indicators[new_indicators[:, 0].argsort()]
#print(new_indicators)
manual_indicators = self.indicators[0]
new_manual_indicators = []
#print('manuals', manual_indicators)
for indicator in manual_indicators:
if indicator in hw_indicators_hw[:, 0]:
#print(indicator)
new_manual_indicators.append(indicator)
self.indicators = (np.asarray(new_manual_indicators), new_indicators)
class RecordedDataset(Dataset):
def __init__(self, participant: str, x, y, indicators, name='recorded'):
......
......@@ -79,6 +79,13 @@ def positive_predicted_value_soft(y_true, y_pred):
frac_sum = np.sum(frac*diff, axis=0)
return 1 - frac_sum
@catch_empty_y_true_soft
def negative_predicted_value_soft(y_true, y_pred):
diff = np.absolute(y_pred - y_true)
frac = (1-y_pred) / np.sum(1-y_pred, axis=0)
frac_sum = np.sum(frac*diff, axis=0)
return 1 - frac_sum
@catch_empty_y_true_soft
def S1_score_soft(y_true, y_pred):
......@@ -107,6 +114,29 @@ def mcc_score_soft(y_true, y_pred):
fn = sum((y_true != y_pred) & (y_pred == 0))
return (tp * tn - fp * fn) / (np.sqrt((tp+fp)*(tp+fn)*(tn+fp)*(tn+fn)))
@catch_empty_y_true
def calc_tp_ratio(y_true, y_pred):
# return np.count_nonzero(np.logical_and(y_true[:, 1] > 0.5, y_pred[:, 1] > 0.5))/np.count_nonzero(y_true[:, 1] > 0.5)
return np.count_nonzero(np.logical_and(y_true[:, 1] > 0.5, y_pred[:, 1] > 0.5))
@catch_empty_y_true
def calc_fp_ratio(y_true, y_pred):
# return np.count_nonzero(np.logical_and(y_true[:, 1] <= 0.5, y_pred[:, 1] > 0.5))/np.count_nonzero(y_true[:, 1] <= 0.5)
return np.count_nonzero(np.logical_and(y_true[:, 1] <= 0.5, y_pred[:, 1] > 0.5))
@catch_empty_y_true
def calc_fn(y_true, y_pred):
# return np.count_nonzero(np.logical_and(y_true[:, 1] > 0.5, y_pred[:, 1] <= 0.5))/np.count_nonzero(y_true[:, 1] > 0.5)
return np.count_nonzero(np.logical_and(y_true[:, 1] > 0.5, y_pred[:, 1] <= 0.5))
@catch_empty_y_true
def calc_tn(y_true, y_pred):
# return np.count_nonzero(np.logical_and(y_true[:, 1] <= 0.5, y_pred[:, 1] <= 0.5))/np.count_nonzero(y_true[:, 1] <= 0.5)
return np.count_nonzero(np.logical_and(y_true[:, 1] <= 0.5, y_pred[:, 1] <= 0.5))
def f1_score_multi(y_true, y_pred, by_class=False):
print('F1 score multi')
f1_res = []
......
......@@ -153,7 +153,7 @@ thesis_filters = ['all', 'high', 'noneut', 'all_corrected_noise', 'scope_correct
'allnoise_correctbydeepconvfilter', 'allnoise_correctbyfcndaefilter',
'allnoise_correctbyconvlstmfilter', 'allnoise_correctbyconvlstm2filter',
'allnoise_correctbyconvlstm3filter', 'alldeepconv_correctbyconvlstm3filter',
'alldeepconv_correctbyconvlstm3filter6', 'alldeepconv_correctbyconvlstm2filter6']
'alldeepconv_correctbyconvlstm2filter6', 'alldeepconv_correctbyconvlstm3filter6']
thesis_translations = {'all': 'all', 'high': 'high_conf', 'noneut': 'scope',
'all_corrected_noise': 'all_corrected_null',
......@@ -168,7 +168,8 @@ thesis_translations = {'all': 'all', 'high': 'high_conf', 'noneut': 'scope',
'allnoise_correctbyconvlstm2filter': 'all_null_convlstm2',
'allnoise_correctbyconvlstm3filter': 'all_null_convlstm3',
'alldeepconv_correctbyconvlstm3filter': 'all_cnn_convlstm3',
'alldeepconv_correctbyconvlstm3filter6': 'all_cnn_convlstm3_hard'}
'alldeepconv_correctbyconvlstm3filter6': 'all_cnn_convlstm3_hard',
'alldeepconv_correctbyconvlstm2filter6': 'all_cnn_convlstm2_hard'}