Commit 22a09c45 authored by Alexander Henkel's avatar Alexander Henkel
Browse files

bugfixes, adjustments

parent 2d275867
...@@ -95,8 +95,8 @@ def generate_queried_data(dataset, prediction, query_size, subset_size=7, use_we ...@@ -95,8 +95,8 @@ def generate_queried_data(dataset, prediction, query_size, subset_size=7, use_we
# print('queried_subsets:', queried_subsets.shape) # print('queried_subsets:', queried_subsets.shape)
queried_entropies = np.empty((0, 1), dtype=int) queried_entropies = np.empty((0, 1), dtype=int)
for query_subset in queried_subsets: for query_subset in queried_subsets:
queried_entropies = np.append(queried_entropies, np.arange(query_subset * subset_size, max_query_index = min(query_subset * subset_size + subset_size + 1, dataset.x_win.shape[0])
query_subset * subset_size + subset_size + 1)) queried_entropies = np.append(queried_entropies, np.arange(query_subset * subset_size, max_query_index))
# print('queried_entropies:', queried_entropies) # print('queried_entropies:', queried_entropies)
if include_high_confidence_hw: if include_high_confidence_hw:
...@@ -108,7 +108,6 @@ def generate_queried_data(dataset, prediction, query_size, subset_size=7, use_we ...@@ -108,7 +108,6 @@ def generate_queried_data(dataset, prediction, query_size, subset_size=7, use_we
high_conf_instance = hw_instances[hw_entropies[:int(hw_instances.shape[0] * high_confidence_hw_part)]] high_conf_instance = hw_instances[hw_entropies[:int(hw_instances.shape[0] * high_confidence_hw_part)]]
# print('high_instances:', high_conf_instance.shape) # print('high_instances:', high_conf_instance.shape)
queried_entropies = np.append(queried_entropies, high_conf_instance) queried_entropies = np.append(queried_entropies, high_conf_instance)
training_dataset = ManualDataset(dataset.name) training_dataset = ManualDataset(dataset.name)
training_dataset.x_win = dataset.x_win[queried_entropies] training_dataset.x_win = dataset.x_win[queried_entropies]
training_dataset.y_win = dataset.y_win[queried_entropies] training_dataset.y_win = dataset.y_win[queried_entropies]
......
...@@ -45,7 +45,7 @@ ...@@ -45,7 +45,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": 4,
"id": "fd753a9f", "id": "fd753a9f",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
%% Cell type:code id:b9f62c58 tags: %% Cell type:code id:b9f62c58 tags:
``` python ``` python
%load_ext autoreload %load_ext autoreload
%autoreload 2 %autoreload 2
%matplotlib notebook %matplotlib notebook
``` ```
%% Cell type:code id:621e3e7f tags: %% Cell type:code id:621e3e7f tags:
``` python ``` python
import sys import sys
import os import os
import numpy as np import numpy as np
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from IPython.display import display, Markdown from IPython.display import display, Markdown
``` ```
%% Cell type:code id:0dab0b49 tags: %% Cell type:code id:0dab0b49 tags:
``` python ``` python
module_path = os.path.abspath(os.path.join('..')) module_path = os.path.abspath(os.path.join('..'))
os.chdir(module_path) os.chdir(module_path)
if module_path not in sys.path: if module_path not in sys.path:
sys.path.append(module_path) sys.path.append(module_path)
``` ```
%% Cell type:code id:16e7ff01 tags: %% Cell type:code id:16e7ff01 tags:
``` python ``` python
from personalization_tools.load_data_sets import * from personalization_tools.load_data_sets import *
from personalization_tools.helpers import * from personalization_tools.helpers import *
from personalization_tools.learner_pipeline import LearnerPipeline from personalization_tools.learner_pipeline import LearnerPipeline
from personalization_tools.dataset_builder import * from personalization_tools.dataset_builder import *
from personalization_tools.personalizer import Personalizer from personalization_tools.personalizer import Personalizer
from personalization_tools.dataset_manager import DatasetManager from personalization_tools.dataset_manager import DatasetManager
from personalization_tools.trainings_manager import TrainingsManager from personalization_tools.trainings_manager import TrainingsManager
from personalization_tools.sensor_recorder_data_reader import SensorRecorderDataReader from personalization_tools.sensor_recorder_data_reader import SensorRecorderDataReader
from personalization_tools.pseudo_model_settings import pseudo_model_settings from personalization_tools.pseudo_model_settings import pseudo_model_settings
``` ```
%% Cell type:code id:6f7b32bb tags: %% Cell type:code id:6f7b32bb tags:
``` python ``` python
WINDOW_LENGTH = 150 WINDOW_LENGTH = 150
WINDOW_SHIFT = 75 WINDOW_SHIFT = 75
inc_model_name = 'recorded_inc_02.pt' inc_model_name = 'recorded_inc_reg_02.pt'
training_run_name = '02_training' training_run_name = '02_training'
base_model_name = './data/DeepConvLSTMA_statedict.pt' base_model_name = './data/DeepConvLSTMA_statedict.pt'
``` ```
%% Cell type:code id:3f93660a tags: %% Cell type:code id:3f93660a tags:
``` python ``` python
personalizer = Personalizer() personalizer = Personalizer()
personalizer.initialize(base_model_name) personalizer.initialize(base_model_name)
``` ```
%% Cell type:code id:7e0d7f13 tags: %% Cell type:code id:7e0d7f13 tags:
``` python ``` python
trainings_manager = TrainingsManager('./data/cluster/recorded_pseudo_training_db_2') trainings_manager = TrainingsManager('./data/cluster/recorded_pseudo_training_db_2')
``` ```
%% Cell type:code id:f0a6fd1b tags: %% Cell type:code id:f0a6fd1b tags:
``` python ``` python
dataset_manager = DatasetManager('./data/recorded_dataset_db') dataset_manager = DatasetManager('./data/recorded_dataset_db')
``` ```
%% Cell type:code id:608252a0 tags: %% Cell type:code id:608252a0 tags:
``` python ``` python
record_reader = SensorRecorderDataReader('/home/alex/gitrepos/uni/MasterProject/Webserver/uploads/recordings') record_reader = SensorRecorderDataReader('/home/alex/gitrepos/uni/MasterProject/Webserver/uploads/recordings')
``` ```
%% Cell type:code id:7090d170 tags: %% Cell type:code id:7090d170 tags:
``` python ``` python
series = dataset_manager.filter_by_category(training_run_name).values() series = dataset_manager.filter_by_category(training_run_name).values()
``` ```
%% Cell type:code id:cf454d2f tags: %% Cell type:code id:cf454d2f tags:
``` python ``` python
series = record_reader.get_collection(['e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6', '2252f3d0-cd90-42b4-9b4e-4172ed5fa847', '88134a7c-00f9-4b82-b579-95ef47cb8a20', '9c1913f3-e44d-40a0-a89a-feb2d3692494']) series = record_reader.get_collection(['e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6', '2252f3d0-cd90-42b4-9b4e-4172ed5fa847', '88134a7c-00f9-4b82-b579-95ef47cb8a20', '9c1913f3-e44d-40a0-a89a-feb2d3692494'])
``` ```
%% Output %% Output
Load: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6 Load: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6
Load: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847 Load: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847
Load: 88134a7c-00f9-4b82-b579-95ef47cb8a20 Load: 88134a7c-00f9-4b82-b579-95ef47cb8a20
Load: 9c1913f3-e44d-40a0-a89a-feb2d3692494 Load: 9c1913f3-e44d-40a0-a89a-feb2d3692494
%% Cell type:code id:ecbb38ae tags: %% Cell type:code id:ecbb38ae tags:
``` python ``` python
personalizer.plot_series(series) personalizer.plot_series(series)
``` ```
%% Output %% Output
%% Cell type:code id:1f6f9f91 tags: %% Cell type:code id:1f6f9f91 tags:
``` python ``` python
personalizer.incremental_learn_series_gt(series, save_model_as='./data/' + inc_model_name, epochs=100) use_regularization = True
personalizer.incremental_learn_series_gt(series, save_model_as='./data/' + inc_model_name, epochs=100, use_regularization=use_regularization, freeze_feature_layers=not use_regularization)
#trainings_manager.database['training_runs'][training_run_name] = [] #trainings_manager.database['training_runs'][training_run_name] = []
#trainings_manager.database['training_runs'][training_run_name].append(inc_model_name) #trainings_manager.database['training_runs'][training_run_name].append(inc_model_name)
# trainings_manager.db_update() # trainings_manager.db_update()
``` ```
%% Output %% Output
Load base model: ./data/DeepConvLSTMA_statedict.pt Load base model: ./data/DeepConvLSTMA_statedict.pt
Train on: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6 Train on: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6
tensor([ 0.5024, 102.6791]) tensor([ 0.5024, 102.6791])
Starting Training cuda Starting Training cuda
use regularization
loss before training: 0.09575944393873215 F1-Score Val: 1.0
F1-Score Val: 0.2857142857142857 spec Val 1.0
spec Val 0.9838187702265372
sens Val 1.0 sens Val 1.0
Train on: daf902bc-01a6-43c5-a04e-f784c8f13c49 Train on: daf902bc-01a6-43c5-a04e-f784c8f13c49
tensor([ 0.5029, 87.6667]) tensor([ 0.5029, 87.6667])
Starting Training cuda Starting Training cuda
use regularization
loss before training: 0.05737384781241417
stopped early! 55 0.06454429057107043
F1-Score Val: 0.0 F1-Score Val: 0.0
spec Val 0 spec Val 0
sens Val 0 sens Val 0
Train on: 28fd37cc-c752-45ec-8adb-f6c008f72754 Train on: 28fd37cc-c752-45ec-8adb-f6c008f72754
tensor([ 0.5012, 204.2273]) tensor([ 0.5012, 204.2273])
Starting Training cuda Starting Training cuda
use regularization
loss before training: 0.08396054059267044 stopped early! 15 0.0017458487796283761
F1-Score Val: 0.0 F1-Score Val: 0.0
spec Val 0 spec Val 0
sens Val 0 sens Val 0
Train on: 8af26275-c037-4291-ab79-dcb050a4686d Train on: 8af26275-c037-4291-ab79-dcb050a4686d
tensor([ 0.5015, 161.8913]) tensor([ 0.5015, 161.8913])
Starting Training cuda Starting Training cuda
use regularization
loss before training: 0.04854266345500946 stopped early! 15 0.00018242024779790573
F1-Score Val: 0.0 F1-Score Val: 0.0
spec Val 0 spec Val 0
sens Val 0 sens Val 0
Train on: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847 Train on: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847
tensor([ 0.5021, 121.3232]) tensor([ 0.5021, 121.3232])
Starting Training cuda Starting Training cuda
use regularization
/home/alex/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1492: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.
_warn_prf(
loss before training: 0.06260821223258972
F1-Score Val: 0.2857142857142857 F1-Score Val: 0.0
spec Val 0.9888143176733781 spec Val 0
sens Val 1.0 sens Val 0
/home/alex/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1492: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.
_warn_prf(
%% Cell type:code id:56893cf5 tags: %% Cell type:code id:56893cf5 tags:
``` python ``` python
del trainings_manager.database['training_runs'][training_run_name] del trainings_manager.database['training_runs'][training_run_name]
trainings_manager.db_update() trainings_manager.db_update()
``` ```
%% Cell type:code id:d7464036 tags: %% Cell type:code id:d7464036 tags:
``` python ``` python
personalizer.compare_model_evaluation_of_recordings(inc_model_name) personalizer.compare_model_evaluation_of_recordings(inc_model_name)
display(Markdown(personalizer.render_evaluation())) display(Markdown(personalizer.render_evaluation()))
``` ```
%% Output %% Output
| name | specificity | sensitivity | f1 | | name | specificity | sensitivity | f1 |
|:---- | ----------- | ----------- | ---------- | |:---- | ----------- | ----------- | ---------- |
| data: 34a0c627-45a9-4b96-9900-497907da94b1 model: DeepConvLSTMA_statedict | 0.913863 | 0.866667 | 0.063363 | | data: 34a0c627-45a9-4b96-9900-497907da94b1 model: DeepConvLSTMA_statedict | 0.913863 | 0.866667 | 0.063363 |
| data: 34a0c627-45a9-4b96-9900-497907da94b1 model: recorded_inc_alex_tmp | 0.978447 | 1.000000 | 0.238727 | | data: 34a0c627-45a9-4b96-9900-497907da94b1 model: recorded_inc_alex_tmp | 0.978447 | 1.000000 | 0.238727 |
| data: b7584e6d-e21d-45cd-b1f7-3bac7b4c2612 model: DeepConvLSTMA_statedict | 0.890805 | 0.933333 | 0.486957 | | data: b7584e6d-e21d-45cd-b1f7-3bac7b4c2612 model: DeepConvLSTMA_statedict | 0.890805 | 0.933333 | 0.486957 |
| data: b7584e6d-e21d-45cd-b1f7-3bac7b4c2612 model: recorded_inc_alex_tmp | 0.936782 | 0.966667 | 0.630435 | | data: b7584e6d-e21d-45cd-b1f7-3bac7b4c2612 model: recorded_inc_alex_tmp | 0.936782 | 0.966667 | 0.630435 |
| data: d729fbf5-48ed-4851-a5ea-eb41d2b1f688 model: DeepConvLSTMA_statedict | 0.947441 | 0.578947 | 0.037898 | | data: d729fbf5-48ed-4851-a5ea-eb41d2b1f688 model: DeepConvLSTMA_statedict | 0.947441 | 0.578947 | 0.037898 |
| data: d729fbf5-48ed-4851-a5ea-eb41d2b1f688 model: recorded_inc_alex_tmp | 0.989736 | 0.868421 | 0.230769 | | data: d729fbf5-48ed-4851-a5ea-eb41d2b1f688 model: recorded_inc_alex_tmp | 0.989736 | 0.868421 | 0.230769 |
%% Cell type:code id:608d6b70 tags: %% Cell type:code id:608d6b70 tags:
``` python ``` python
ax = personalizer.learner_pipeline.plot_dataset() ax = personalizer.learner_pipeline.plot_dataset()
personalizer.learner_pipeline.plot_prediction(axes=ax) personalizer.learner_pipeline.plot_prediction(axes=ax)
``` ```
%% Output %% Output
[[9.8303926e-01 1.6960735e-02] [[9.8303926e-01 1.6960735e-02]
[6.2543350e-01 3.7456647e-01] [6.2543350e-01 3.7456647e-01]
[9.9613547e-01 3.8645437e-03] [9.9613547e-01 3.8645437e-03]
... ...
[9.9962878e-01 3.7124348e-04] [9.9962878e-01 3.7124348e-04]
[9.9982846e-01 1.7155796e-04] [9.9982846e-01 1.7155796e-04]
[9.9764913e-01 2.3508898e-03]] [9.9764913e-01 2.3508898e-03]]
<AxesSubplot:> <AxesSubplot:>
%% Cell type:code id:9193d12e tags: %% Cell type:code id:9193d12e tags:
``` python ``` python
predictions = generate_predictions(series, base_model_name) predictions = generate_predictions(series, base_model_name)
``` ```
%% Output %% Output
run prediction on [cuda]: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6...0.6196446418762207 seconds run prediction on [cuda]: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6...0.6196446418762207 seconds
run prediction on [cuda]: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847...0.8462226390838623 seconds run prediction on [cuda]: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847...0.8462226390838623 seconds
run prediction on [cuda]: 88134a7c-00f9-4b82-b579-95ef47cb8a20...1.0194389820098877 seconds run prediction on [cuda]: 88134a7c-00f9-4b82-b579-95ef47cb8a20...1.0194389820098877 seconds
run prediction on [cuda]: 9c1913f3-e44d-40a0-a89a-feb2d3692494...0.030718564987182617 seconds run prediction on [cuda]: 9c1913f3-e44d-40a0-a89a-feb2d3692494...0.030718564987182617 seconds
%% Cell type:code id:f7096db7 tags: %% Cell type:code id:f7096db7 tags:
``` python ``` python
pseudo_filter = 'allnoise_correctedhwgt' pseudo_filter = 'allnoise_correctedhwgt'
for dataset in series: for dataset in series:
dataset.generate_feedback_areas(prediction=predictions[dataset.name]) dataset.generate_feedback_areas(prediction=predictions[dataset.name])
dataset.apply_pseudo_label_generators(pseudo_model_settings[pseudo_filter]) dataset.apply_pseudo_label_generators(pseudo_model_settings[pseudo_filter])
``` ```
%% Cell type:code id:d2d4574e tags: %% Cell type:code id:d2d4574e tags:
``` python ``` python
inc_model_name = 'gt_test3.pt' inc_model_name = 'gt_test3.pt'
base_model_name = './data/DeepConvLSTMA_statedict.pt' base_model_name = './data/DeepConvLSTMA_statedict.pt'
``` ```
%% Cell type:code id:387d9e22 tags: %% Cell type:code id:387d9e22 tags:
``` python ``` python
personalizer = Personalizer() personalizer = Personalizer()
personalizer.initialize(base_model_name) personalizer.initialize(base_model_name)
personalizer.incremental_learn_series_pseudo(series, save_model_as='./data/' + inc_model_name, epochs=100, use_regularization=False) personalizer.incremental_learn_series_pseudo(series, save_model_as='./data/' + inc_model_name, epochs=100, use_regularization=False)
``` ```
%% Output %% Output
Train on: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6 Train on: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6
Starting Training cuda Starting Training cuda
loss before training: 0.09575944393873215 loss before training: 0.09575944393873215
F1-Score Val: 0.5333333333333333 F1-Score Val: 0.5333333333333333
spec Val 0.9771241830065359 spec Val 0.9771241830065359
sens Val 1.0 sens Val 1.0
Train on: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847 Train on: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847
Starting Training cuda Starting Training cuda
loss before training: 0.07606717944145203 loss before training: 0.07606717944145203
F1-Score Val: 0.15384615384615385 F1-Score Val: 0.15384615384615385
spec Val 0.9753914988814317 spec Val 0.9753914988814317
sens Val 1.0 sens Val 1.0
Train on: 88134a7c-00f9-4b82-b579-95ef47cb8a20 Train on: 88134a7c-00f9-4b82-b579-95ef47cb8a20
Starting Training cuda Starting Training cuda
loss before training: 0.025048060342669487 loss before training: 0.025048060342669487
stopped early! 50 0.05347004958285835 stopped early! 50 0.05347004958285835
F1-Score Val: 0.4 F1-Score Val: 0.4
spec Val 0.961038961038961 spec Val 0.961038961038961
sens Val 1.0 sens Val 1.0
Train on: 9c1913f3-e44d-40a0-a89a-feb2d3692494 Train on: 9c1913f3-e44d-40a0-a89a-feb2d3692494
Starting Training cuda Starting Training cuda
loss before training: 0.5482482314109802 loss before training: 0.5482482314109802
F1-Score Val: 0.0 F1-Score Val: 0.0
spec Val 0 spec Val 0
sens Val 0 sens Val 0
%% Cell type:code id:6981b0c1 tags: %% Cell type:code id:6981b0c1 tags:
``` python ``` python
``` ```
......