Commit bc1811fa authored by Alexander Henkel's avatar Alexander Henkel
Browse files

bugfixes adjustments

parent f5552682
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
This diff is collapsed.
%% Cell type:code id:75ea2362 tags:
``` python
%load_ext autoreload
%autoreload 2
%matplotlib notebook
```
%% Cell type:code id:47091ae3 tags:
``` python
import sys
import os
import numpy as np
import matplotlib.pyplot as plt
from IPython.display import display, Markdown
import copy
```
%% Cell type:code id:e569de8f tags:
``` python
module_path = os.path.abspath(os.path.join('..'))
os.chdir(module_path)
if module_path not in sys.path:
sys.path.append(module_path)
```
%% Cell type:code id:4dd256e9 tags:
``` python
from personalization_tools.load_data_sets import *
from personalization_tools.helpers import *
from personalization_tools.learner_pipeline import LearnerPipeline, Evaluation
from personalization_tools.dataset_builder import *
from personalization_tools.model_evaluation import ModelEvaluation
from personalization_tools.sensor_recorder_data_reader import SensorRecorderDataReader
from personalization_tools.dataset_manager import DatasetManager
from personalization_tools.trainings_manager import TrainingsManager
from personalization_tools.evaluation_manager import EvaluationManager
from personalization_tools.helpers import generate_predictions
from personalization_tools.quality_estimation import calc_best_running_mean_settings, calc_quality_difference, \
build_simulated_running_mean, get_triggers_on_running_mean
```
%% Cell type:code id:877fa734 tags:
``` python
evaluation_config_file = './data/cluster/randomized_models/evaluation_config.yaml'
```
%% Cell type:code id:712d7cdd tags:
``` python
evaluation_manager = EvaluationManager()
evaluation_manager.load_config(evaluation_config_file)
```
%% Output
load config
%% Cell type:code id:c22a1c18 tags:
``` python
def calc_quality_comparison(test_collection, new_model_path: str,
kernel_size, kernel_threshold, prediction_buffer):
triggers = []
for dataset in test_collection:
r_mean, prediction = build_simulated_running_mean(dataset, new_model_path, kernel_size=kernel_size,
kernel_threshold=kernel_threshold,
prediction_buffer=prediction_buffer)
trigger_spots, correct_triggers, _ = get_triggers_on_running_mean(dataset, r_mean,
kernel_threshold=kernel_threshold)
triggers.append((len(trigger_spots), correct_triggers))
sum_hand_washes = 0
sum_correct_hand_washes = 0
sum_false_hand_washes = 0
for dataset in test_collection:
sum_hand_washes += len(dataset.feedback_areas.labeled_regions_hw)
for trigger in triggers:
sum_correct_hand_washes += trigger[1]
sum_false_hand_washes += trigger[0] - trigger[1]
if sum_hand_washes == 0:
sensitivity = 0
else:
sensitivity = sum_correct_hand_washes / sum_hand_washes
if sum_false_hand_washes + sum_correct_hand_washes == 0:
precision = 0
else:
precision = sum_correct_hand_washes / (sum_false_hand_washes + sum_correct_hand_washes)
if precision + sensitivity == 0:
f1 = 0
else:
f1 = 2 * ((precision * sensitivity) / (precision + sensitivity))
specificity = 1 / (1+sum_false_hand_washes)
return sensitivity, precision, f1
def calc_quality_mean(include_collections):
f1_mean_base = []
f1_mean_inc = []
real_f1_mean_base = []
real_f1_mean_inc = []
for target_collection in include_collections:
collection_config = evaluation_manager.get_collection_config(target_collection)
test_sets = list(evaluation_manager.get_test_sets_of_collection(target_collection).keys())
base_model_path = os.path.join(evaluation_manager.baseconfig['model_prefix'], collection_config['base_model'])
inc_model_path = os.path.join(evaluation_manager.baseconfig['model_prefix'], collection_config['base_inc_model'])
best_model_settings = calc_best_running_mean_settings(evaluation_manager.get_test_sets_of_collection(target_collection).values(), base_model_path, base_model_path, 20, 0.59,
prediction_buffer=None, do_print=False)
mean_kernel_width = best_model_settings[0][1][0]
mean_threshold = best_model_settings[0][1][1]
_, _, f1_base = calc_quality_comparison(evaluation_manager.get_test_sets_of_collection(target_collection).values(),
base_model_path,
mean_kernel_width, mean_threshold, None)
best_model_settings = calc_best_running_mean_settings(evaluation_manager.get_test_sets_of_collection(target_collection).values(), base_model_path, inc_model_path, 20, 0.59,
prediction_buffer=None, do_print=False)
mean_kernel_width = best_model_settings[0][1][0]
mean_threshold = best_model_settings[0][1][1]
_, _, f1_inc = calc_quality_comparison(evaluation_manager.get_test_sets_of_collection(target_collection).values(),
inc_model_path,
mean_kernel_width, mean_threshold, None)
real_eval = evaluation_manager.model_evaluation.get_averages(include_models=[collection_config['base_model'], collection_config['base_inc_model']], include_datasets=test_sets)
f1_mean_base.append(f1_base)
f1_mean_inc.append(f1_inc)
real_f1_mean_base.append(real_eval[collection_config['base_model']]['f1'])
real_f1_mean_inc.append(real_eval[collection_config['base_inc_model']]['f1'])
estimated_diff = f1_inc - f1_base
real_diff = real_eval[collection_config['base_inc_model']]['f1'] - real_eval[collection_config['base_model']]['f1']
print(target_collection, estimated_diff, real_diff)
print(f1_inc, real_eval[collection_config['base_model']]['f1'])
f1_mean_base = np.mean(np.array(f1_mean_base))
f1_mean_inc = np.mean(np.array(f1_mean_inc))
real_f1_mean_base = np.mean(np.array(real_f1_mean_base))
real_f1_mean_inc = np.mean(np.array(real_f1_mean_inc))
estimated_diff = f1_mean_inc - f1_mean_base
real_diff = real_f1_mean_inc - real_f1_mean_base
return estimated_diff, real_diff, f1_mean_inc, real_f1_mean_inc, f1_mean_base, real_f1_mean_base
```
%% Cell type:code id:22d514f8 tags:
``` python
include_collections = ['synthetic_01', 'synthetic_02', 'synthetic_10', 'recorded_01', 'recorded_02']
calc_quality_mean(include_collections)
```
%% Output
synthetic_01 -0.1305361305361304 0.2078462281273526
0.4848484848484849 0.3481930548750389
synthetic_02 0.09157509157509164 0.15421255525130978
0.4761904761904762 0.47683592186260576
synthetic_10 0.21212121212121204 0.24373989845922944
0.6666666666666666 0.3799694358478078
recorded_01 0.06666666666666671 0.2685592478053844
0.4 0.4271913495089722
recorded_02 0.2188782489740082 0.164991695908836
0.8235294117647058 0.4310766435717919
(0.09174101776016957,
0.2078699251104224,
0.5702470078940667,
0.6205232062436657,
0.4785059901338971,
0.41265328113324334)
%% Cell type:code id:e6feefb2 tags:
``` python
```
This diff is collapsed.
...@@ -53,12 +53,13 @@ class Dataset: ...@@ -53,12 +53,13 @@ class Dataset:
self.y_win = hard_label_to_soft_label(self.y_win) self.y_win = hard_label_to_soft_label(self.y_win)
def generate_feedback_areas(self, model=None, prediction=None, add_softmax=True): def generate_feedback_areas(self, model=None, prediction=None, add_softmax=True, use_manuals=True):
self.feedback_areas = FeedbackAreas() self.feedback_areas = FeedbackAreas()
if model is not None: if model is not None:
self.feedback_areas.generate_areas_with_model(self.x_win, self.get_indicators(), model, add_softmax) self.feedback_areas.generate_areas_with_model(self.x_win, self.get_indicators(), model, add_softmax,
use_manuals=use_manuals)
elif prediction is not None: elif prediction is not None:
self.feedback_areas.generate_areas(prediction.copy(), self.get_indicators()) self.feedback_areas.generate_areas(prediction.copy(), self.get_indicators(), use_manuals=use_manuals)
self.pseudo_labels = PseudoLabels(self.feedback_areas) self.pseudo_labels = PseudoLabels(self.feedback_areas)
......
from typing import Dict, List, Union from typing import Dict, List, Union, Any
import yaml import yaml
import os import os
...@@ -164,6 +164,19 @@ class EvaluationManager: ...@@ -164,6 +164,19 @@ class EvaluationManager:
collections.append(collection_config['name']) collections.append(collection_config['name'])
return collections return collections
def filter_collections_by(self, filter: Dict[Any, Any]):
collections = []
for collection_set in self.config['collection_configs']:
for collection_config in self.config['collection_configs'][collection_set].values():
append_this = True
for filter_key, filter_value in filter.items():
if filter_key not in collection_config or filter_value != collection_config[filter_key]:
append_this = False
break
if append_this:
collections.append(collection_config['name'])
return collections
def check(self, short=True): def check(self, short=True):
print('Collections:', self.test_collections.keys()) print('Collections:', self.test_collections.keys())
print('Dataset managers: ', self.dataset_managers.keys()) print('Dataset managers: ', self.dataset_managers.keys())
......
...@@ -82,7 +82,7 @@ class FeedbackAreas: ...@@ -82,7 +82,7 @@ class FeedbackAreas:
self.labeled_regions_noise = [] self.labeled_regions_noise = []
self.labeled_regions_neut = [] self.labeled_regions_neut = []
def generate_areas(self, pred, indicators): def generate_areas(self, pred, indicators, use_manuals=True):
self.reset() self.reset()
self.pred = pred self.pred = pred
self.indicators = indicators self.indicators = indicators
...@@ -122,15 +122,18 @@ class FeedbackAreas: ...@@ -122,15 +122,18 @@ class FeedbackAreas:
# skip = True # skip = True
# break # break
if not skip: if not skip:
self.labeled_regions_hw.append( if use_manuals:
[start_indicator, indicator + indicator_span_manual_hw[1]]) self.labeled_regions_hw.append(
[start_indicator, indicator + indicator_span_manual_hw[1]])
else:
self.labeled_regions_neut.append([start_indicator, indicator + indicator_span_manual_hw[1]])
for region in self.labeled_regions_hw: for region in self.labeled_regions_hw:
for noise_region in self.labeled_regions_noise: for noise_region in self.labeled_regions_noise:
if region[0] < noise_region[1] < region[1]: if region[0] < noise_region[1] < region[1]:
region[0] = noise_region[1] region[0] = noise_region[1]
def generate_areas_with_model(self, x_data, indicators, model, add_softmax=True): def generate_areas_with_model(self, x_data, indicators, model, add_softmax=True, use_manuals=True):
self.reset() self.reset()
if add_softmax: if add_softmax:
model = nn.Sequential(model, nn.Softmax(dim=-1)) model = nn.Sequential(model, nn.Softmax(dim=-1))
...@@ -140,7 +143,7 @@ class FeedbackAreas: ...@@ -140,7 +143,7 @@ class FeedbackAreas:
pred: np.ndarray = model(x_data_t.to("cpu")).detach().cpu().numpy() pred: np.ndarray = model(x_data_t.to("cpu")).detach().cpu().numpy()
self.pred = pred self.pred = pred
self.generate_areas(pred, indicators) self.generate_areas(pred, indicators, use_manuals=use_manuals)
class PseudoLabels: class PseudoLabels:
......
...@@ -25,9 +25,9 @@ class SensorRecorderDataReader: ...@@ -25,9 +25,9 @@ class SensorRecorderDataReader:
def get_data_set(self, data_set_name, participant_name='user', transform_hand=False): def get_data_set(self, data_set_name, participant_name='user', transform_hand=False):
processor = DataProcessor(os.path.join(self.data_set_path, data_set_name), init_all=False) processor = DataProcessor(os.path.join(self.data_set_path, data_set_name), init_all=False, use_numpy_caching=self.use_numpy_caching)
processor.read_entry(RecordingEntry.ACCELERATION, use_numpy_caching=True) processor.read_entry(RecordingEntry.ACCELERATION, use_numpy_caching=self.use_numpy_caching)
processor.read_entry(RecordingEntry.GYROSCOPE, use_numpy_caching=True) processor.read_entry(RecordingEntry.GYROSCOPE, use_numpy_caching=self.use_numpy_caching)
processor.read_entry(RecordingEntry.MARKERS) processor.read_entry(RecordingEntry.MARKERS)
processor.read_entry(RecordingEntry.MANUALWHTS) processor.read_entry(RecordingEntry.MANUALWHTS)
processor.read_entry(RecordingEntry.EVALUATIONS) processor.read_entry(RecordingEntry.EVALUATIONS)
......
...@@ -26,7 +26,6 @@ arg_parser.add_argument('-s', '--skip', ...@@ -26,7 +26,6 @@ arg_parser.add_argument('-s', '--skip',
action='store_true', action='store_true',
help='skip already existing configurations') help='skip already existing configurations')
args = arg_parser.parse_args() args = arg_parser.parse_args()
config = dict() config = dict()
base_config = dict() base_config = dict()
...@@ -93,15 +92,19 @@ def calc_quality_comparison(test_collection: List[RecordedDataset], new_model_pa ...@@ -93,15 +92,19 @@ def calc_quality_comparison(test_collection: List[RecordedDataset], new_model_pa
sensitivity = 0 sensitivity = 0
else: else:
sensitivity = sum_correct_hand_washes / sum_hand_washes sensitivity = sum_correct_hand_washes / sum_hand_washes
if sum_false_hand_washes + sum_correct_hand_washes == 0: if sum_false_hand_washes + sum_correct_hand_washes == 0:
precision = 0 precision = 0
else: else:
precision = sum_correct_hand_washes / (sum_false_hand_washes + sum_correct_hand_washes) precision = sum_correct_hand_washes / (sum_false_hand_washes + sum_correct_hand_washes)