Commit 32297bd2 authored by Alexander Henkel's avatar Alexander Henkel
Browse files

bugfixing

parent 56b185ac
%% Cell type:code id: tags: %% Cell type:code id:347f47a4 tags:
``` python ``` python
%load_ext autoreload %load_ext autoreload
%autoreload 2 %autoreload 2
%matplotlib notebook %matplotlib notebook
``` ```
%% Cell type:code id: tags: %% Cell type:code id:1e458131 tags:
``` python ``` python
import sys import sys
import os import os
import numpy as np import numpy as np
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from IPython.display import display, Markdown from IPython.display import display, Markdown
import copy import copy
import dbm import dbm
``` ```
%% Cell type:code id: tags: %% Cell type:code id:55746bea tags:
``` python ``` python
module_path = os.path.abspath(os.path.join('..')) module_path = os.path.abspath(os.path.join('..'))
os.chdir(module_path) os.chdir(module_path)
if module_path not in sys.path: if module_path not in sys.path:
sys.path.append(module_path) sys.path.append(module_path)
``` ```
%% Cell type:code id: tags: %% Cell type:code id:86d3a948 tags:
``` python ``` python
from personalization_tools.load_data_sets import * from personalization_tools.load_data_sets import *
from personalization_tools.helpers import * from personalization_tools.helpers import *
from personalization_tools.learner_pipeline import LearnerPipeline, Evaluation from personalization_tools.learner_pipeline import LearnerPipeline, Evaluation
from personalization_tools.dataset_builder import * from personalization_tools.dataset_builder import *
from personalization_tools.model_evaluation import ModelEvaluation from personalization_tools.model_evaluation import ModelEvaluation
from personalization_tools.sensor_recorder_data_reader import SensorRecorderDataReader from personalization_tools.sensor_recorder_data_reader import SensorRecorderDataReader
from personalization_tools.dataset_manager import DatasetManager from personalization_tools.dataset_manager import DatasetManager
from personalization_tools.trainings_manager import TrainingsManager from personalization_tools.trainings_manager import TrainingsManager
from personalization_tools.pseudo_model_settings import pseudo_model_settings, pseudo_label_generators_to_int from personalization_tools.pseudo_model_settings import pseudo_model_settings, pseudo_label_generators_to_int
from personalization_tools.pseudo_label_evaluation import * from personalization_tools.pseudo_label_evaluation import *
from personalization_tools.evaluation_manager import EvaluationManager from personalization_tools.evaluation_manager import EvaluationManager
``` ```
%% Cell type:code id: tags: %% Cell type:code id:cf5ca0eb tags:
``` python ``` python
predictions_db = './data/cluster/participant_training/predictions_db_2' predictions_db = './data/cluster/participant_training/predictions_db_2'
configuration_file = './data/cluster/participant_training/config_2.yaml' configuration_file = './data/cluster/participant_training/config_2.yaml'
evaluation_manager = EvaluationManager(predictions_db) evaluation_manager = EvaluationManager(predictions_db)
``` ```
%% Cell type:code id: tags: %% Cell type:code id:6a8d5388 tags:
``` python ``` python
evaluation_manager.load_config(configuration_file) evaluation_manager.load_config(configuration_file)
``` ```
%% Cell type:code id: tags: %% Output
load config
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
<ipython-input-6-1a95d51ed4d9> in <module>
----> 1 evaluation_manager.load_config(configuration_file)
~/gitrepos/uni/handwashing_personalizer/src/personalization_tools/evaluation_manager.py in load_config(self, config_file_path)
105
106 self.personalized_models[collection_config['name']] = list()
--> 107 for model in self.training_managers[training_db].get_all_training_runs()[collection_config['training_run_name']]:
108 # print('add model', model, 'assign', test_collection)
109 # print('assign', train_collection)
KeyError: 'pseudo_collection_01_run1'
%% Cell type:code id:05dd3052 tags:
``` python ``` python
for dataset_manager in evaluation_manager.dataset_managers.values(): for dataset_manager in evaluation_manager.dataset_managers.values():
for dataset in dataset_manager.get_all_datasets().values(): for dataset in dataset_manager.get_all_datasets().values():
dataset.generate_windows() dataset.generate_windows()
``` ```
%% Cell type:code id: tags: %% Cell type:code id:d206ac9a tags:
``` python ``` python
evaluation_manager.do_predictions() evaluation_manager.do_predictions()
``` ```
%% Cell type:code id: tags: %% Cell type:code id:b2bced4c tags:
``` python ``` python
evaluation_manager.model_evaluation.get_evaluations(calc_averages=True, sort_by='f1') evaluation_manager.model_evaluation.get_evaluations(calc_averages=True, sort_by='f1')
``` ```
%% Cell type:code id: tags: %% Cell type:code id:9b59398e tags:
``` python ``` python
simulated_trigger_buffer = dict() simulated_trigger_buffer = dict()
#simulated_trigger_buffer = None #simulated_trigger_buffer = None
``` ```
%% Cell type:code id: tags: %% Cell type:code id:cb52bd2f tags:
``` python ``` python
# observed_models = ['pseudo_collection_01_run1_allnoise_correctedhwgt.pt', 'pseudo_collection_01_run1_allnoise_correctedscore.pt', 'pseudo_collection_01_run1_allnoise_correctbydeepconvfilter.pt', 'pseudo_collection_01_run1_allnoise_correctbyfcndaefilter.pt', 'pseudo_collection_01_run1_allnoise_correctbyconvlstmfilter.pt', 'pseudo_collection_01_run1_allnoise_correctbyconvlstm2filter.pt', 'pseudo_collection_01_run1_allnoise_correctbyconvlstm3filter.pt','pseudo_collection_01_run1_alldeepconv_correctbyconvlstm3filter2.pt', 'pseudo_collection_01_run1_alldeepconv_correctbyconvlstm3filter4.pt', 'pseudo_collection_01_run1_alldeepconv_correctbyconvlstm3filter6.pt'] # observed_models = ['pseudo_collection_01_run1_allnoise_correctedhwgt.pt', 'pseudo_collection_01_run1_allnoise_correctedscore.pt', 'pseudo_collection_01_run1_allnoise_correctbydeepconvfilter.pt', 'pseudo_collection_01_run1_allnoise_correctbyfcndaefilter.pt', 'pseudo_collection_01_run1_allnoise_correctbyconvlstmfilter.pt', 'pseudo_collection_01_run1_allnoise_correctbyconvlstm2filter.pt', 'pseudo_collection_01_run1_allnoise_correctbyconvlstm3filter.pt','pseudo_collection_01_run1_alldeepconv_correctbyconvlstm3filter2.pt', 'pseudo_collection_01_run1_alldeepconv_correctbyconvlstm3filter4.pt', 'pseudo_collection_01_run1_alldeepconv_correctbyconvlstm3filter6.pt']
observed_models = ['pseudo_collection_01_run1_allnoise_correctbydeepconvfilter.pt', 'pseudo_collection_01_run1_allnoise_correctbyfcndaefilter.pt', 'pseudo_collection_01_run1_allnoise_correctbyconvlstm3filter.pt', 'pseudo_collection_01_run1_alldeepconv_correctbyconvlstm3filter6.pt'] observed_models = ['pseudo_collection_01_run1_allnoise_correctbydeepconvfilter.pt', 'pseudo_collection_01_run1_allnoise_correctbyfcndaefilter.pt', 'pseudo_collection_01_run1_allnoise_correctbyconvlstm3filter.pt', 'pseudo_collection_01_run1_alldeepconv_correctbyconvlstm3filter6.pt']
``` ```
%% Cell type:code id: tags: %% Cell type:code id:7b84c451 tags:
``` python ``` python
def filter_possible_handwashes(dataset, prediction): def filter_possible_handwashes(dataset, prediction):
possible_spots = [] possible_spots = []
filtered_predictions = np.zeros(prediction.shape) filtered_predictions = np.zeros(prediction.shape)
filtered_predictions[:, 0] = 1.00001 filtered_predictions[:, 0] = 1.00001
for ts in range(75, dataset.x_data.shape[0], 75): for ts in range(75, dataset.x_data.shape[0], 75):
min_val = np.min(dataset.x_data[ts-75:ts], axis=0) min_val = np.min(dataset.x_data[ts-75:ts], axis=0)
max_val = np.max(dataset.x_data[ts-75:ts], axis=0) max_val = np.max(dataset.x_data[ts-75:ts], axis=0)
diff = np.absolute(max_val - min_val) diff = np.absolute(max_val - min_val)
if np.any(diff > 10): if np.any(diff > 10):
possible_spots.append(int(ts/75)) possible_spots.append(int(ts/75))
for possible_spot in possible_spots: for possible_spot in possible_spots:
for i in range(0, 5): for i in range(0, 5):
if possible_spot+ i < prediction.shape[0]: if possible_spot+ i < prediction.shape[0]:
filtered_predictions[possible_spot + i] = prediction[possible_spot + i] filtered_predictions[possible_spot + i] = prediction[possible_spot + i]
return filtered_predictions return filtered_predictions
def build_simulated_running_mean(dataset, model_name, kernel_size=20, kernel_threshold=0.59, prediction_buffer=None): def build_simulated_running_mean(dataset, model_name, kernel_size=20, kernel_threshold=0.59, prediction_buffer=None):
if prediction_buffer is not None and (dataset.name, model_name) in prediction_buffer: if prediction_buffer is not None and (dataset.name, model_name) in prediction_buffer:
prediction = prediction_buffer[(dataset.name, model_name)] prediction = prediction_buffer[(dataset.name, model_name)]
else: else:
prediction = generate_predictions([dataset,], model_name, verbose=False)[dataset.name] prediction = generate_predictions([dataset,], model_name, verbose=False)[dataset.name]
if prediction_buffer is not None: if prediction_buffer is not None:
prediction_buffer[(dataset.name, model_name)] = prediction prediction_buffer[(dataset.name, model_name)] = prediction
if isinstance(dataset, SyntheticDataset): if isinstance(dataset, SyntheticDataset):
dataset.generate_synthetic_evaluation(prediction, kernel_size=kernel_size, kernel_threshold=kernel_threshold) dataset.generate_synthetic_evaluation(prediction, kernel_size=kernel_size, kernel_threshold=kernel_threshold)
dataset.generate_feedback_areas(prediction=prediction) dataset.generate_feedback_areas(prediction=prediction)
prediction = filter_possible_handwashes(dataset, prediction) prediction = filter_possible_handwashes(dataset, prediction)
kernel = np.ones(kernel_size) / kernel_size kernel = np.ones(kernel_size) / kernel_size
mean = np.convolve(prediction[:, 1], kernel, mode='same') mean = np.convolve(prediction[:, 1], kernel, mode='same')
r_mean = np.empty_like(mean) r_mean = np.empty_like(mean)
r_mean[kernel_size:] = mean[:-kernel_size] r_mean[kernel_size:] = mean[:-kernel_size]
return r_mean, prediction return r_mean, prediction
def get_triggers_on_running_mean(dataset, r_mean, kernel_threshold=0.59): def get_triggers_on_running_mean(dataset, r_mean, kernel_threshold=0.59):
trigger_spots = [] trigger_spots = []
for high_mean_spot in np.where(r_mean*1 > kernel_threshold)[0]: for high_mean_spot in np.where(r_mean*1 > kernel_threshold)[0]:
if len(trigger_spots) == 0: if len(trigger_spots) == 0:
trigger_spots.append(high_mean_spot) trigger_spots.append(high_mean_spot)
else: else:
if high_mean_spot - trigger_spots[-1] > 15: if high_mean_spot - trigger_spots[-1] > 15:
trigger_spots.append(high_mean_spot) trigger_spots.append(high_mean_spot)
# print(len(trigger_spots), 'handwash triggers:', trigger_spots) # print(len(trigger_spots), 'handwash triggers:', trigger_spots)
correct_triggers = 0 correct_triggers = 0
detected_regions = [] detected_regions = []
for trigger_spot in trigger_spots[:]: for trigger_spot in trigger_spots[:]:
for region in dataset.feedback_areas.labeled_regions_hw: for region in dataset.feedback_areas.labeled_regions_hw:
if region[0] <= trigger_spot <= region[1]: if region[0] <= trigger_spot <= region[1]:
if region in detected_regions: if region in detected_regions:
trigger_spots.remove(trigger_spot) trigger_spots.remove(trigger_spot)
else: else:
correct_triggers += 1 correct_triggers += 1
detected_regions.append(region) detected_regions.append(region)
break break
return trigger_spots, correct_triggers return trigger_spots, correct_triggers
def simulate_dataset(dataset, model_name, kernel_size=20, kernel_threshold=0.59, do_plot=False, prediction_buffer=None): def simulate_dataset(dataset, model_name, kernel_size=20, kernel_threshold=0.59, do_plot=False, prediction_buffer=None):
if simulated_trigger_buffer is not None and (dataset.name, model_name, kernel_size, kernel_threshold) in simulated_trigger_buffer and not do_plot: if simulated_trigger_buffer is not None and (dataset.name, model_name, kernel_size, kernel_threshold) in simulated_trigger_buffer and not do_plot:
return simulated_trigger_buffer[(dataset.name, model_name, kernel_size, kernel_threshold)] return simulated_trigger_buffer[(dataset.name, model_name, kernel_size, kernel_threshold)]
r_mean, prediction = build_simulated_running_mean(dataset, model_name, kernel_size=kernel_size, kernel_threshold=kernel_threshold, prediction_buffer=prediction_buffer) r_mean, prediction = build_simulated_running_mean(dataset, model_name, kernel_size=kernel_size, kernel_threshold=kernel_threshold, prediction_buffer=prediction_buffer)
trigger_spots, correct_triggers = get_triggers_on_running_mean(dataset, r_mean, kernel_threshold=kernel_threshold) trigger_spots, correct_triggers = get_triggers_on_running_mean(dataset, r_mean, kernel_threshold=kernel_threshold)
if do_plot: if do_plot:
ax = dataset.plot_windows() ax = dataset.plot_windows()
dataset.plot_feedback_areas(ax=ax) dataset.plot_feedback_areas(ax=ax)
dataset.plot_prediction(prediction, ax=ax) dataset.plot_prediction(prediction, ax=ax)
ax.scatter(trigger_spots, [1]*len(trigger_spots)) ax.scatter(trigger_spots, [1]*len(trigger_spots))
if simulated_trigger_buffer is not None: if simulated_trigger_buffer is not None:
simulated_trigger_buffer[(dataset.name, model_name, kernel_size, kernel_threshold)] = (len(trigger_spots)-correct_triggers, correct_triggers) simulated_trigger_buffer[(dataset.name, model_name, kernel_size, kernel_threshold)] = (len(trigger_spots)-correct_triggers, correct_triggers)
return len(trigger_spots)-correct_triggers, correct_triggers return len(trigger_spots)-correct_triggers, correct_triggers
def compare_personalized(collection_config, personalized_model): def compare_personalized(collection_config, personalized_model):
if personalized_model == collection_config['base_model'] or personalized_model == collection_config['base_inc_model']: if personalized_model == collection_config['base_model'] or personalized_model == collection_config['base_inc_model']:
personalized_model = collection_config['model_prefix'] + personalized_model personalized_model = collection_config['model_prefix'] + personalized_model
else: else:
personalized_model = collection_config['models_directory'] + personalized_model personalized_model = collection_config['models_directory'] + personalized_model
all_datasets = evaluation_manager.test_collections[collection_config['name']][:] all_datasets = evaluation_manager.test_collections[collection_config['name']][:]
all_datasets += evaluation_manager.train_collections[collection_config['name']] all_datasets += evaluation_manager.train_collections[collection_config['name']]
base_triggers = dict() base_triggers = dict()
pers_triggers = dict() pers_triggers = dict()
for dataset in all_datasets[:]: for dataset in all_datasets[:]:
pers_triggers[dataset] = simulate_dataset(dataset, personalized_model) pers_triggers[dataset] = simulate_dataset(dataset, personalized_model)
base_triggers[dataset] = simulate_dataset(dataset, collection_config['base_model']) base_triggers[dataset] = simulate_dataset(dataset, collection_config['base_model'])
fig, ax = plt.subplots() fig, ax = plt.subplots()
base_triggers = np.array(list(base_triggers.values())) base_triggers = np.array(list(base_triggers.values()))
pers_triggers = np.array(list(pers_triggers.values())) pers_triggers = np.array(list(pers_triggers.values()))
ax.plot(np.arange(base_triggers.shape[0]), base_triggers[:, 0], c='red') ax.plot(np.arange(base_triggers.shape[0]), base_triggers[:, 0], c='red')
ax.plot(np.arange(pers_triggers.shape[0]), pers_triggers[:, 0], c='orange') ax.plot(np.arange(pers_triggers.shape[0]), pers_triggers[:, 0], c='orange')
ax.plot(np.arange(base_triggers.shape[0]), base_triggers[:, 1], c='green') ax.plot(np.arange(base_triggers.shape[0]), base_triggers[:, 1], c='green')
ax.plot(np.arange(pers_triggers.shape[0]), pers_triggers[:, 1], c='blue') ax.plot(np.arange(pers_triggers.shape[0]), pers_triggers[:, 1], c='blue')
fig.show() fig.show()
def calc_diff_between_perso_base(collection_config, personalized_model, use_percentage=False, kernel_threshold=0.59, kernel_size=20, include_training=False): def calc_diff_between_perso_base(collection_config, personalized_model, use_percentage=False, kernel_threshold=0.59, kernel_size=20, include_training=False):
if personalized_model == collection_config['base_model'] or personalized_model == collection_config['base_inc_model']: if personalized_model == collection_config['base_model'] or personalized_model == collection_config['base_inc_model']:
personalized_model = collection_config['model_prefix'] + personalized_model personalized_model = collection_config['model_prefix'] + personalized_model
else: else:
personalized_model = collection_config['models_directory'] + personalized_model personalized_model = collection_config['models_directory'] + personalized_model
# print(personalized_model) # print(personalized_model)
all_datasets = evaluation_manager.test_collections[collection_config['name']][:] all_datasets = evaluation_manager.test_collections[collection_config['name']][:]
if include_training: if include_training:
all_datasets += evaluation_manager.train_collections[collection_config['name']] all_datasets += evaluation_manager.train_collections[collection_config['name']]
base_triggers = dict() base_triggers = dict()
pers_triggers = dict() pers_triggers = dict()
for dataset in all_datasets[:]: for dataset in all_datasets[:]:
pers_triggers[dataset] = simulate_dataset(dataset, personalized_model, kernel_threshold=kernel_threshold, kernel_size=kernel_size) pers_triggers[dataset] = simulate_dataset(dataset, personalized_model, kernel_threshold=kernel_threshold, kernel_size=kernel_size)
base_triggers[dataset] = simulate_dataset(dataset, collection_config['model_prefix'] + collection_config['base_model']) base_triggers[dataset] = simulate_dataset(dataset, collection_config['model_prefix'] + collection_config['base_model'])
# print(base_triggers[dataset], 'vs.', pers_triggers[dataset]) # print(base_triggers[dataset], 'vs.', pers_triggers[dataset])
base_triggers = np.array(list(base_triggers.values()), dtype=float) base_triggers = np.array(list(base_triggers.values()), dtype=float)
pers_triggers = np.array(list(pers_triggers.values()), dtype=float) pers_triggers = np.array(list(pers_triggers.values()), dtype=float)
wrong_diff = pers_triggers[:, 0] - base_triggers[:, 0] wrong_diff = pers_triggers[:, 0] - base_triggers[:, 0]
correct_diff = pers_triggers[:, 1] - base_triggers[:, 1] correct_diff = pers_triggers[:, 1] - base_triggers[:, 1]
if use_percentage: if use_percentage:
wrong_diff[base_triggers[:, 0] != 0] = wrong_diff[base_triggers[:, 0] != 0] / base_triggers[base_triggers[:, 0] != 0, 0] wrong_diff[base_triggers[:, 0] != 0] = wrong_diff[base_triggers[:, 0] != 0] / base_triggers[base_triggers[:, 0] != 0, 0]
correct_diff[base_triggers[:, 1] != 0] = correct_diff[base_triggers[:, 1] != 0] / base_triggers[base_triggers[:, 1] != 0, 1] correct_diff[base_triggers[:, 1] != 0] = correct_diff[base_triggers[:, 1] != 0] / base_triggers[base_triggers[:, 1] != 0, 1]
wd_zero = wrong_diff[base_triggers[:, 0] == 0] wd_zero = wrong_diff[base_triggers[:, 0] == 0]
cd_zero = correct_diff[base_triggers[:, 1] == 0] cd_zero = correct_diff[base_triggers[:, 1] == 0]
wd_zero = np.minimum(np.ones(wd_zero.shape[0]), wd_zero) wd_zero = np.minimum(np.ones(wd_zero.shape[0]), wd_zero)
cd_zero = np.minimum(np.ones(cd_zero.shape[0]), cd_zero) cd_zero = np.minimum(np.ones(cd_zero.shape[0]), cd_zero)
return wrong_diff, correct_diff return wrong_diff, correct_diff
def plot_diff_between_perso_base(collection_config, use_percentage=False, kernel_threshold=0.59, kernel_size=20, include_training=False): def plot_diff_between_perso_base(collection_config, use_percentage=False, kernel_threshold=0.59, kernel_size=20, include_training=False):
diffs = dict() diffs = dict()
cycle_models = evaluation_manager.personalized_models[collection_config['name']] cycle_models = evaluation_manager.personalized_models[collection_config['name']]
if len(observed_models) > 0: if len(observed_models) > 0:
cycle_models = observed_models cycle_models = observed_models
for model in cycle_models + [collection_config['base_model'], collection_config['base_inc_model']]: for model in cycle_models + [collection_config['base_model'], collection_config['base_inc_model']]:
model = os.path.basename(model) model = os.path.basename(model)
diffs[model] = calc_diff_between_perso_base(collection_config, model, use_percentage=use_percentage, kernel_threshold=kernel_threshold, kernel_size=kernel_size, include_training=include_training) diffs[model] = calc_diff_between_perso_base(collection_config, model, use_percentage=use_percentage, kernel_threshold=kernel_threshold, kernel_size=kernel_size, include_training=include_training)
x_vals = np.array(remove_model_name_suffix(list(diffs.keys())[:-2]) + list(diffs.keys())[-2:]) x_vals = np.array(remove_model_name_suffix(list(diffs.keys())[:-2]) + list(diffs.keys())[-2:])
y_vals = np.array(list(diffs.values())) y_vals = np.array(list(diffs.values()))
print(y_vals.shape) print(y_vals.shape)
y_vals = np.sum(y_vals, axis=2)/y_vals.shape[2] y_vals = np.sum(y_vals, axis=2)/y_vals.shape[2]
fig, ax = plt.subplots(figsize=(9,7)) fig, ax = plt.subplots(figsize=(9,7))
fig.suptitle(collection_config['name']) fig.suptitle(collection_config['name'])
ax.set_ylabel('num difference in triggers') ax.set_ylabel('num difference in triggers')
if use_percentage: if use_percentage:
ax.set_ylabel('% trigger difference') ax.set_ylabel('% trigger difference')
y_vals *= 100 y_vals *= 100
ax.plot(x_vals, y_vals[:, 0], c='red', label='false') ax.plot(x_vals, y_vals[:, 0], c='red', label='false')
ax.plot(x_vals, y_vals[:, 1], c='green', label='correct') ax.plot(x_vals, y_vals[:, 1], c='green', label='correct')
ax.tick_params(axis='x', labelrotation = 75) ax.tick_params(axis='x', labelrotation = 75)
#plt.subplots_adjust(hspace=0.8) #plt.subplots_adjust(hspace=0.8)
plt.subplots_adjust(bottom=0.4, top=0.95) plt.subplots_adjust(bottom=0.4, top=0.95)
plt.grid() plt.grid()
fig.legend() fig.legend()
fig.show() fig.show()
``` ```
%% Cell type:code id: tags: