Commit 22a09c45 authored by Alexander Henkel's avatar Alexander Henkel
Browse files

bugfixes, adjustments

parent 2d275867
......@@ -95,8 +95,8 @@ def generate_queried_data(dataset, prediction, query_size, subset_size=7, use_we
# print('queried_subsets:', queried_subsets.shape)
queried_entropies = np.empty((0, 1), dtype=int)
for query_subset in queried_subsets:
queried_entropies = np.append(queried_entropies, np.arange(query_subset * subset_size,
query_subset * subset_size + subset_size + 1))
max_query_index = min(query_subset * subset_size + subset_size + 1, dataset.x_win.shape[0])
queried_entropies = np.append(queried_entropies, np.arange(query_subset * subset_size, max_query_index))
# print('queried_entropies:', queried_entropies)
if include_high_confidence_hw:
......@@ -108,7 +108,6 @@ def generate_queried_data(dataset, prediction, query_size, subset_size=7, use_we
high_conf_instance = hw_instances[hw_entropies[:int(hw_instances.shape[0] * high_confidence_hw_part)]]
# print('high_instances:', high_conf_instance.shape)
queried_entropies = np.append(queried_entropies, high_conf_instance)
training_dataset = ManualDataset(dataset.name)
training_dataset.x_win = dataset.x_win[queried_entropies]
training_dataset.y_win = dataset.y_win[queried_entropies]
......
......@@ -45,7 +45,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 4,
"id": "fd753a9f",
"metadata": {},
"outputs": [],
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
%% Cell type:code id:b9f62c58 tags:
``` python
%load_ext autoreload
%autoreload 2
%matplotlib notebook
```
%% Cell type:code id:621e3e7f tags:
``` python
import sys
import os
import numpy as np
import matplotlib.pyplot as plt
from IPython.display import display, Markdown
```
%% Cell type:code id:0dab0b49 tags:
``` python
module_path = os.path.abspath(os.path.join('..'))
os.chdir(module_path)
if module_path not in sys.path:
sys.path.append(module_path)
```
%% Cell type:code id:16e7ff01 tags:
``` python
from personalization_tools.load_data_sets import *
from personalization_tools.helpers import *
from personalization_tools.learner_pipeline import LearnerPipeline
from personalization_tools.dataset_builder import *
from personalization_tools.personalizer import Personalizer
from personalization_tools.dataset_manager import DatasetManager
from personalization_tools.trainings_manager import TrainingsManager
from personalization_tools.sensor_recorder_data_reader import SensorRecorderDataReader
from personalization_tools.pseudo_model_settings import pseudo_model_settings
```
%% Cell type:code id:6f7b32bb tags:
``` python
WINDOW_LENGTH = 150
WINDOW_SHIFT = 75
inc_model_name = 'recorded_inc_02.pt'
inc_model_name = 'recorded_inc_reg_02.pt'
training_run_name = '02_training'
base_model_name = './data/DeepConvLSTMA_statedict.pt'
```
%% Cell type:code id:3f93660a tags:
``` python
personalizer = Personalizer()
personalizer.initialize(base_model_name)
```
%% Cell type:code id:7e0d7f13 tags:
``` python
trainings_manager = TrainingsManager('./data/cluster/recorded_pseudo_training_db_2')
```
%% Cell type:code id:f0a6fd1b tags:
``` python
dataset_manager = DatasetManager('./data/recorded_dataset_db')
```
%% Cell type:code id:608252a0 tags:
``` python
record_reader = SensorRecorderDataReader('/home/alex/gitrepos/uni/MasterProject/Webserver/uploads/recordings')
```
%% Cell type:code id:7090d170 tags:
``` python
series = dataset_manager.filter_by_category(training_run_name).values()
```
%% Cell type:code id:cf454d2f tags:
``` python
series = record_reader.get_collection(['e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6', '2252f3d0-cd90-42b4-9b4e-4172ed5fa847', '88134a7c-00f9-4b82-b579-95ef47cb8a20', '9c1913f3-e44d-40a0-a89a-feb2d3692494'])
```
%% Output
Load: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6
Load: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847
Load: 88134a7c-00f9-4b82-b579-95ef47cb8a20
Load: 9c1913f3-e44d-40a0-a89a-feb2d3692494
%% Cell type:code id:ecbb38ae tags:
``` python
personalizer.plot_series(series)
```
%% Output
%% Cell type:code id:1f6f9f91 tags:
``` python
personalizer.incremental_learn_series_gt(series, save_model_as='./data/' + inc_model_name, epochs=100)
use_regularization = True
personalizer.incremental_learn_series_gt(series, save_model_as='./data/' + inc_model_name, epochs=100, use_regularization=use_regularization, freeze_feature_layers=not use_regularization)
#trainings_manager.database['training_runs'][training_run_name] = []
#trainings_manager.database['training_runs'][training_run_name].append(inc_model_name)
# trainings_manager.db_update()
```
%% Output
Load base model: ./data/DeepConvLSTMA_statedict.pt
Train on: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6
tensor([ 0.5024, 102.6791])
Starting Training cuda
use regularization
loss before training: 0.09575944393873215
F1-Score Val: 0.2857142857142857
spec Val 0.9838187702265372
F1-Score Val: 1.0
spec Val 1.0
sens Val 1.0
Train on: daf902bc-01a6-43c5-a04e-f784c8f13c49
tensor([ 0.5029, 87.6667])
Starting Training cuda
use regularization
loss before training: 0.05737384781241417
stopped early! 55 0.06454429057107043
F1-Score Val: 0.0
spec Val 0
sens Val 0
Train on: 28fd37cc-c752-45ec-8adb-f6c008f72754
tensor([ 0.5012, 204.2273])
Starting Training cuda
use regularization
loss before training: 0.08396054059267044
stopped early! 15 0.0017458487796283761
F1-Score Val: 0.0
spec Val 0
sens Val 0
Train on: 8af26275-c037-4291-ab79-dcb050a4686d
tensor([ 0.5015, 161.8913])
Starting Training cuda
use regularization
loss before training: 0.04854266345500946
stopped early! 15 0.00018242024779790573
F1-Score Val: 0.0
spec Val 0
sens Val 0
Train on: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847
tensor([ 0.5021, 121.3232])
Starting Training cuda
use regularization
/home/alex/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1492: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.
_warn_prf(
loss before training: 0.06260821223258972
F1-Score Val: 0.2857142857142857
spec Val 0.9888143176733781
sens Val 1.0
F1-Score Val: 0.0
spec Val 0
sens Val 0
/home/alex/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1492: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.
_warn_prf(
%% Cell type:code id:56893cf5 tags:
``` python
del trainings_manager.database['training_runs'][training_run_name]
trainings_manager.db_update()
```
%% Cell type:code id:d7464036 tags:
``` python
personalizer.compare_model_evaluation_of_recordings(inc_model_name)
display(Markdown(personalizer.render_evaluation()))
```
%% Output
| name | specificity | sensitivity | f1 |
|:---- | ----------- | ----------- | ---------- |
| data: 34a0c627-45a9-4b96-9900-497907da94b1 model: DeepConvLSTMA_statedict | 0.913863 | 0.866667 | 0.063363 |
| data: 34a0c627-45a9-4b96-9900-497907da94b1 model: recorded_inc_alex_tmp | 0.978447 | 1.000000 | 0.238727 |
| data: b7584e6d-e21d-45cd-b1f7-3bac7b4c2612 model: DeepConvLSTMA_statedict | 0.890805 | 0.933333 | 0.486957 |
| data: b7584e6d-e21d-45cd-b1f7-3bac7b4c2612 model: recorded_inc_alex_tmp | 0.936782 | 0.966667 | 0.630435 |
| data: d729fbf5-48ed-4851-a5ea-eb41d2b1f688 model: DeepConvLSTMA_statedict | 0.947441 | 0.578947 | 0.037898 |
| data: d729fbf5-48ed-4851-a5ea-eb41d2b1f688 model: recorded_inc_alex_tmp | 0.989736 | 0.868421 | 0.230769 |
%% Cell type:code id:608d6b70 tags:
``` python
ax = personalizer.learner_pipeline.plot_dataset()
personalizer.learner_pipeline.plot_prediction(axes=ax)
```
%% Output
[[9.8303926e-01 1.6960735e-02]
[6.2543350e-01 3.7456647e-01]
[9.9613547e-01 3.8645437e-03]
...
[9.9962878e-01 3.7124348e-04]
[9.9982846e-01 1.7155796e-04]
[9.9764913e-01 2.3508898e-03]]
<AxesSubplot:>
%% Cell type:code id:9193d12e tags:
``` python
predictions = generate_predictions(series, base_model_name)
```
%% Output
run prediction on [cuda]: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6...0.6196446418762207 seconds
run prediction on [cuda]: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847...0.8462226390838623 seconds
run prediction on [cuda]: 88134a7c-00f9-4b82-b579-95ef47cb8a20...1.0194389820098877 seconds
run prediction on [cuda]: 9c1913f3-e44d-40a0-a89a-feb2d3692494...0.030718564987182617 seconds
%% Cell type:code id:f7096db7 tags:
``` python
pseudo_filter = 'allnoise_correctedhwgt'
for dataset in series:
dataset.generate_feedback_areas(prediction=predictions[dataset.name])
dataset.apply_pseudo_label_generators(pseudo_model_settings[pseudo_filter])
```
%% Cell type:code id:d2d4574e tags:
``` python
inc_model_name = 'gt_test3.pt'
base_model_name = './data/DeepConvLSTMA_statedict.pt'
```
%% Cell type:code id:387d9e22 tags:
``` python
personalizer = Personalizer()
personalizer.initialize(base_model_name)
personalizer.incremental_learn_series_pseudo(series, save_model_as='./data/' + inc_model_name, epochs=100, use_regularization=False)
```
%% Output
Train on: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6
Starting Training cuda
loss before training: 0.09575944393873215
F1-Score Val: 0.5333333333333333
spec Val 0.9771241830065359
sens Val 1.0
Train on: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847
Starting Training cuda
loss before training: 0.07606717944145203
F1-Score Val: 0.15384615384615385
spec Val 0.9753914988814317
sens Val 1.0
Train on: 88134a7c-00f9-4b82-b579-95ef47cb8a20
Starting Training cuda
loss before training: 0.025048060342669487
stopped early! 50 0.05347004958285835
F1-Score Val: 0.4
spec Val 0.961038961038961
sens Val 1.0
Train on: 9c1913f3-e44d-40a0-a89a-feb2d3692494
Starting Training cuda
loss before training: 0.5482482314109802
F1-Score Val: 0.0
spec Val 0
sens Val 0
%% Cell type:code id:6981b0c1 tags:
``` python
```
......
%% Cell type:code id:01c24b43 tags:
``` python
%load_ext autoreload
%autoreload 2
%matplotlib notebook
```
%% Cell type:code id:c68fe089 tags:
``` python
import sys
import os
import numpy as np
import matplotlib.pyplot as plt
from IPython.display import display, Markdown
import copy
```
%% Cell type:code id:c7213856 tags:
``` python
module_path = os.path.abspath(os.path.join('..'))
os.chdir(module_path)
if module_path not in sys.path:
sys.path.append(module_path)
```
%% Cell type:code id:3a8bad41 tags:
``` python
from personalization_tools.load_data_sets import *
from personalization_tools.helpers import *
from personalization_tools.learner_pipeline import LearnerPipeline
from personalization_tools.dataset_builder import *
from personalization_tools.personalizer_synthetic import PersonalizerSynthetic
from personalization_tools.dataset_manager import DatasetManager
from personalization_tools.trainings_manager import TrainingsManager
from personalization_tools.model_evaluation import ModelEvaluation
from personalization_tools.pseudo_model_settings import pseudo_model_settings
```
%% Cell type:code id:d9aea435 tags:
``` python
WINDOW_LENGTH = 150
WINDOW_SHIFT = 75
inc_model_name = 'synthetic_inc_01.pt'
inc_model_name = 'synthetic_inc_reg_10.pt'
predictions_db = './data/cluster/synthetic_learn_predictions_db'
dataset_db = './data/synthetic_dataset_db'
training_db = './data/cluster/pseudo_training_db'
models_directory = './data/'
base_model = './data/HandWashingDeepConvLSTMA_trunc_01.pt'
base_model = './data/HandWashingDeepConvLSTMA_trunc_10.pt'
training_run_name = 'test_base_synthetic_01'
training_collection = '01_training'
training_collection = '10_training'
```
%% Cell type:code id:4ea39d4e tags:
``` python
personalizer = PersonalizerSynthetic()
personalizer.initialize(base_model)
# personalizer.load_data(['01', '02'])
```
%% Cell type:code id:b72e8659 tags:
``` python
dataset_manager = DatasetManager(dataset_db)
```
%% Cell type:code id:e571f62e tags:
``` python
trainings_manager = TrainingsManager(training_db)
```
%% Cell type:code id:7e93e3c7 tags:
``` python
series = dataset_manager.filter_by_category(training_collection).values()
```
%% Cell type:code id:1cb0a355 tags:
``` python
series = dataset_manager.filter_by_category(training_collection).values()
print(series)
```
%% Output
dict_values([01_generated_2, 01_generated_3, 01_generated_4, 01_generated_5])
dict_values([10_generated_1, 10_generated_2, 10_generated_3])
%% Cell type:code id:ebac8347 tags:
``` python
# series = personalizer.build_series_for_participant('01', 2, target_hand_wash_counts=2, target_recorded_hours = 1, reuse_data=False)
personalizer.process_series(series)
```
%% Output
---------------------------------------------------------------------------
KeyboardInterrupt Traceback (most recent call last)
<ipython-input-15-9ceed92af4b6> in <module>
1 # series = personalizer.build_series_for_participant('01', 2, target_hand_wash_counts=2, target_recorded_hours = 1, reuse_data=False)
----> 2 personalizer.process_series(series)
~/gitrepos/uni/handwashing_personalizer/src/tools/personalizer_synthetic.py in process_series(self, series)
13 for i, ser in enumerate(series):
14 self.learner_pipeline.load_dataset(ser)
---> 15 predicted = self.learner_pipeline.predict_dataset()
16 evaluation = self.learner_pipeline.evaluate_prediction()
17 evaluations.append(evaluation)
~/gitrepos/uni/handwashing_personalizer/src/tools/learner_pipeline.py in predict_dataset(self)
137
138 def predict_dataset(self) -> np.ndarray:
--> 139 return self.predict(self.data_set.x_win, self.data_set.name)
140
141 def plot_dataset(self, axes: plt.Axes = None):
~/gitrepos/uni/handwashing_personalizer/src/tools/learner_pipeline.py in predict(self, x_val, data_name)
129 x_pred = x_pred.to(device)
130 pred = model(x_pred)
--> 131 pred = pred.detach().cpu().numpy()
132 pred_complete[batch:batch + batch_size] = pred
133 del x_pred
KeyboardInterrupt:
%% Cell type:code id:bcc0fff7 tags:
``` python
dataset_manager.update_dataset_collection(series)
```
%% Cell type:code id:f36aed3f tags:
``` python
personalizer.plot_series_windows(series)
```
%% Output
%% Cell type:code id:1fc8c4bf tags:
``` python
personalizer.incremental_learn_series_gt(list(series), save_model_as=models_directory + inc_model_name, epochs=100)
use_regularization = True
personalizer.incremental_learn_series_gt(list(series), save_model_as=models_directory + inc_model_name, epochs=100, use_regularization=use_regularization, freeze_feature_layers=not use_regularization)
#trainings_manager.database['training_runs'][training_run_name] = []
#trainings_manager.database['training_runs'][training_run_name].append(inc_model_name)
#trainings_manager.db_update()
```
%% Output
Train on: 01_generated_2
tensor([ 0.5030, 84.1154])
Starting Training cuda
loss before training: 0.2243620604276657
F1-Score Val: 0.20512820512820512
spec Val 0.9117647058823529
sens Val 0.8
Train on: 01_generated_3
tensor([ 0.5052, 48.5152])
Load base model: ./data/HandWashingDeepConvLSTMA_trunc_10.pt
Train on: 10_generated_1
tensor([ 0.5049, 51.6770])
Starting Training cuda
use regularization
loss before training: 0.34959739446640015
F1-Score Val: 0.125
spec Val 0.88268156424581