bugfixes, adjustments

22a09c45 · Alexander Henkel · 2d275867 · 22a09c45 · 22a09c45 · 22a09c45
Commit 22a09c45 authored Jun 09, 2022 by Alexander Henkel
--- a/src/active_training_grid_search.py
+++ b/src/active_training_grid_search.py
@@ -95,8 +95,8 @@ def generate_queried_data(dataset, prediction, query_size, subset_size=7, use_we
    # print('queried_subsets:', queried_subsets.shape)
    queried_entropies = np.empty((0, 1), dtype=int)
    for query_subset in queried_subsets:
-        queried_entropies = np.append(queried_entropies, np.arange(query_subset * subset_size,
-                                                                   query_subset * subset_size + subset_size + 1))
+        max_query_index = min(query_subset * subset_size + subset_size + 1, dataset.x_win.shape[0])
+        queried_entropies = np.append(queried_entropies, np.arange(query_subset * subset_size, max_query_index))
    # print('queried_entropies:', queried_entropies)

    if include_high_confidence_hw:
@@ -108,7 +108,6 @@ def generate_queried_data(dataset, prediction, query_size, subset_size=7, use_we
        high_conf_instance = hw_instances[hw_entropies[:int(hw_instances.shape[0] * high_confidence_hw_part)]]
        # print('high_instances:', high_conf_instance.shape)
        queried_entropies = np.append(queried_entropies, high_conf_instance)
-
    training_dataset = ManualDataset(dataset.name)
    training_dataset.x_win = dataset.x_win[queried_entropies]
    training_dataset.y_win = dataset.y_win[queried_entropies]

--- a/src/notebooks/ActiveLearning.ipynb
+++ b/src/notebooks/ActiveLearning.ipynb
@@ -45,7 +45,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 4,
   "id": "fd753a9f",
   "metadata": {},
   "outputs": [],
--- a/src/notebooks/ActiveLearningEvaluation.ipynb
+++ b/src/notebooks/ActiveLearningEvaluation.ipynb
--- a/src/notebooks/DatasetManager.ipynb
+++ b/src/notebooks/DatasetManager.ipynb
--- a/src/notebooks/PersonalizationEvolutionEvaluation.ipynb
+++ b/src/notebooks/PersonalizationEvolutionEvaluation.ipynb
--- a/src/notebooks/PseudoEvaluation.ipynb
+++ b/src/notebooks/PseudoEvaluation.ipynb
--- a/src/notebooks/PseudoLabeling.ipynb
+++ b/src/notebooks/PseudoLabeling.ipynb
--- a/src/notebooks/RecordedLearning.ipynb
+++ b/src/notebooks/RecordedLearning.ipynb
@@ -67,14 +67,14 @@
   "source": [
    "WINDOW_LENGTH = 150\n",
    "WINDOW_SHIFT = 75\n",
-    "inc_model_name = 'recorded_inc_02.pt'\n",
+    "inc_model_name = 'recorded_inc_reg_02.pt'\n",
    "training_run_name = '02_training'\n",
    "base_model_name = './data/DeepConvLSTMA_statedict.pt'"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 12,
   "id": "3f93660a",
   "metadata": {},
   "outputs": [],
@@ -95,7 +95,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 11,
   "id": "f0a6fd1b",
   "metadata": {},
   "outputs": [],
@@ -1138,13 +1138,14 @@
      "Load base model: ./data/DeepConvLSTMA_statedict.pt\n",
      "Train on: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6\n",
      "tensor([  0.5024, 102.6791])\n",
-      "Starting Training cuda\n"
+      "Starting Training cuda\n",
+      "use regularization\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "4f507dad01f441128b58e8beddb68b47",
+       "model_id": "933588d83bf2476da5dcf7c05c8a5de9",
       "version_major": 2,
       "version_minor": 0
      },
@@ -1159,20 +1160,20 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "loss before training: 0.09575944393873215\n",
-      "F1-Score Val:\t0.2857142857142857\n",
-      "spec Val 0.9838187702265372\n",
+      "F1-Score Val:\t1.0\n",
+      "spec Val 1.0\n",
      "sens Val 1.0\n",
      " \n",
      "Train on: daf902bc-01a6-43c5-a04e-f784c8f13c49\n",
      "tensor([ 0.5029, 87.6667])\n",
-      "Starting Training cuda\n"
+      "Starting Training cuda\n",
+      "use regularization\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "b3a0ae8e9184435eb165dd660ecf79f9",
+       "model_id": "74ce92ebb7344a81b66fd7629b823d86",
       "version_major": 2,
       "version_minor": 0
      },
@@ -1187,21 +1188,20 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "loss before training: 0.05737384781241417\n",
-      "stopped early! 55 0.06454429057107043\n",
      "F1-Score Val:\t0.0\n",
      "spec Val 0\n",
      "sens Val 0\n",
      " \n",
      "Train on: 28fd37cc-c752-45ec-8adb-f6c008f72754\n",
      "tensor([  0.5012, 204.2273])\n",
-      "Starting Training cuda\n"
+      "Starting Training cuda\n",
+      "use regularization\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "e661f1cab1034fdc80e208fbd69b47ba",
+       "model_id": "d4f82d5349764403b74777b14fc42dee",
       "version_major": 2,
       "version_minor": 0
      },
@@ -1216,20 +1216,21 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "loss before training: 0.08396054059267044\n",
+      "stopped early! 15 0.0017458487796283761\n",
      "F1-Score Val:\t0.0\n",
      "spec Val 0\n",
      "sens Val 0\n",
      " \n",
      "Train on: 8af26275-c037-4291-ab79-dcb050a4686d\n",
      "tensor([  0.5015, 161.8913])\n",
-      "Starting Training cuda\n"
+      "Starting Training cuda\n",
+      "use regularization\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "34b6093d8b9349548b669fda494414b8",
+       "model_id": "7f6e5a15cd6346cba9e38b67e6d7955b",
       "version_major": 2,
       "version_minor": 0
      },
@@ -1244,20 +1245,29 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "loss before training: 0.04854266345500946\n",
+      "stopped early! 15 0.00018242024779790573\n",
      "F1-Score Val:\t0.0\n",
      "spec Val 0\n",
      "sens Val 0\n",
      " \n",
      "Train on: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847\n",
      "tensor([  0.5021, 121.3232])\n",
-      "Starting Training cuda\n"
+      "Starting Training cuda\n",
+      "use regularization\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/alex/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1492: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.\n",
+      "  _warn_prf(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "2a41456fb0364c33be7206ccc6152fe9",
+       "model_id": "6f8fca3e4ea945f8ab87df8081342fcd",
       "version_major": 2,
       "version_minor": 0
      },
@@ -1272,16 +1282,25 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "loss before training: 0.06260821223258972\n",
-      "F1-Score Val:\t0.2857142857142857\n",
-      "spec Val 0.9888143176733781\n",
-      "sens Val 1.0\n",
+      "F1-Score Val:\t0.0\n",
+      "spec Val 0\n",
+      "sens Val 0\n",
      " \n"
     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/alex/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1492: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.\n",
+      "  _warn_prf(\n"
+     ]
    }
   ],
   "source": [
-    "personalizer.incremental_learn_series_gt(series, save_model_as='./data/' + inc_model_name, epochs=100)\n",
+    "use_regularization = True\n",
+    "personalizer.incremental_learn_series_gt(series, save_model_as='./data/' + inc_model_name, epochs=100, use_regularization=use_regularization, freeze_feature_layers=not use_regularization)\n",
+    "\n",
    "#trainings_manager.database['training_runs'][training_run_name] = []\n",
    "#trainings_manager.database['training_runs'][training_run_name].append(inc_model_name)\n",
    "# trainings_manager.db_update()"

 %% Cell type:code id:b9f62c58 tags:

 ``` python
 %load_ext autoreload
 %autoreload 2

 %matplotlib notebook
 ```

 %% Cell type:code id:621e3e7f tags:

 ``` python
 import sys
 import os
 import numpy as np
 import matplotlib.pyplot as plt
 from IPython.display import display, Markdown
 ```

 %% Cell type:code id:0dab0b49 tags:

 ``` python
 module_path = os.path.abspath(os.path.join('..'))
 os.chdir(module_path)
 if module_path not in sys.path:
    sys.path.append(module_path)
 ```

 %% Cell type:code id:16e7ff01 tags:

 ``` python
 from personalization_tools.load_data_sets import *
 from personalization_tools.helpers import *
 from personalization_tools.learner_pipeline import LearnerPipeline
 from personalization_tools.dataset_builder import *
 from personalization_tools.personalizer import Personalizer
 from personalization_tools.dataset_manager import DatasetManager
 from personalization_tools.trainings_manager import TrainingsManager
 from personalization_tools.sensor_recorder_data_reader import SensorRecorderDataReader
 from personalization_tools.pseudo_model_settings import pseudo_model_settings
 ```

 %% Cell type:code id:6f7b32bb tags:

 ``` python
 WINDOW_LENGTH = 150
 WINDOW_SHIFT = 75
-inc_model_name = 'recorded_inc_02.pt'
+inc_model_name = 'recorded_inc_reg_02.pt'
 training_run_name = '02_training'
 base_model_name = './data/DeepConvLSTMA_statedict.pt'
 ```

 %% Cell type:code id:3f93660a tags:

 ``` python
 personalizer = Personalizer()
 personalizer.initialize(base_model_name)
 ```

 %% Cell type:code id:7e0d7f13 tags:

 ``` python
 trainings_manager = TrainingsManager('./data/cluster/recorded_pseudo_training_db_2')
 ```

 %% Cell type:code id:f0a6fd1b tags:

 ``` python
 dataset_manager = DatasetManager('./data/recorded_dataset_db')
 ```

 %% Cell type:code id:608252a0 tags:

 ``` python
 record_reader = SensorRecorderDataReader('/home/alex/gitrepos/uni/MasterProject/Webserver/uploads/recordings')
 ```

 %% Cell type:code id:7090d170 tags:

 ``` python
 series = dataset_manager.filter_by_category(training_run_name).values()
 ```

 %% Cell type:code id:cf454d2f tags:

 ``` python
 series = record_reader.get_collection(['e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6', '2252f3d0-cd90-42b4-9b4e-4172ed5fa847', '88134a7c-00f9-4b82-b579-95ef47cb8a20', '9c1913f3-e44d-40a0-a89a-feb2d3692494'])
 ```

 %% Output

    Load: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6
    Load: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847
    Load: 88134a7c-00f9-4b82-b579-95ef47cb8a20
    Load: 9c1913f3-e44d-40a0-a89a-feb2d3692494

 %% Cell type:code id:ecbb38ae tags:

 ``` python
 personalizer.plot_series(series)
 ```

 %% Output



 %% Cell type:code id:1f6f9f91 tags:

 ``` python
-personalizer.incremental_learn_series_gt(series, save_model_as='./data/' + inc_model_name, epochs=100)
+use_regularization = True
+personalizer.incremental_learn_series_gt(series, save_model_as='./data/' + inc_model_name, epochs=100, use_regularization=use_regularization, freeze_feature_layers=not use_regularization)
+
 #trainings_manager.database['training_runs'][training_run_name] = []
 #trainings_manager.database['training_runs'][training_run_name].append(inc_model_name)
 # trainings_manager.db_update()
 ```

 %% Output

    Load base model: ./data/DeepConvLSTMA_statedict.pt
    Train on: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6
    tensor([  0.5024, 102.6791])
    Starting Training cuda
+    use regularization


-    loss before training: 0.09575944393873215
-    F1-Score Val:	0.2857142857142857
-    spec Val 0.9838187702265372
+    F1-Score Val:	1.0
+    spec Val 1.0
    sens Val 1.0
    
    Train on: daf902bc-01a6-43c5-a04e-f784c8f13c49
    tensor([ 0.5029, 87.6667])
    Starting Training cuda
+    use regularization


-    loss before training: 0.05737384781241417
-    stopped early! 55 0.06454429057107043
    F1-Score Val:	0.0
    spec Val 0
    sens Val 0
    
    Train on: 28fd37cc-c752-45ec-8adb-f6c008f72754
    tensor([  0.5012, 204.2273])
    Starting Training cuda
+    use regularization


-    loss before training: 0.08396054059267044
+    stopped early! 15 0.0017458487796283761
    F1-Score Val:	0.0
    spec Val 0
    sens Val 0
    
    Train on: 8af26275-c037-4291-ab79-dcb050a4686d
    tensor([  0.5015, 161.8913])
    Starting Training cuda
+    use regularization


-    loss before training: 0.04854266345500946
+    stopped early! 15 0.00018242024779790573
    F1-Score Val:	0.0
    spec Val 0
    sens Val 0
    
    Train on: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847
    tensor([  0.5021, 121.3232])
    Starting Training cuda
+    use regularization

+    /home/alex/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1492: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.
+      _warn_prf(

-    loss before training: 0.06260821223258972
-    F1-Score Val:	0.2857142857142857
-    spec Val 0.9888143176733781
-    sens Val 1.0
+
+    F1-Score Val:	0.0
+    spec Val 0
+    sens Val 0
    

+    /home/alex/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1492: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.
+      _warn_prf(
+
 %% Cell type:code id:56893cf5 tags:

 ``` python
 del trainings_manager.database['training_runs'][training_run_name]
 trainings_manager.db_update()
 ```

 %% Cell type:code id:d7464036 tags:

 ``` python
 personalizer.compare_model_evaluation_of_recordings(inc_model_name)
 display(Markdown(personalizer.render_evaluation()))
 ```

 %% Output

    | name | specificity | sensitivity |     f1     |
    |:---- | ----------- | ----------- | ---------- |
    | data: 34a0c627-45a9-4b96-9900-497907da94b1	 model: DeepConvLSTMA_statedict | 0.913863 | 0.866667 | 0.063363 |
    | data: 34a0c627-45a9-4b96-9900-497907da94b1	 model: recorded_inc_alex_tmp | 0.978447 | 1.000000 | 0.238727 |
    | data: b7584e6d-e21d-45cd-b1f7-3bac7b4c2612	 model: DeepConvLSTMA_statedict | 0.890805 | 0.933333 | 0.486957 |
    | data: b7584e6d-e21d-45cd-b1f7-3bac7b4c2612	 model: recorded_inc_alex_tmp | 0.936782 | 0.966667 | 0.630435 |
    | data: d729fbf5-48ed-4851-a5ea-eb41d2b1f688	 model: DeepConvLSTMA_statedict | 0.947441 | 0.578947 | 0.037898 |
    | data: d729fbf5-48ed-4851-a5ea-eb41d2b1f688	 model: recorded_inc_alex_tmp | 0.989736 | 0.868421 | 0.230769 |

 %% Cell type:code id:608d6b70 tags:

 ``` python
 ax = personalizer.learner_pipeline.plot_dataset()
 personalizer.learner_pipeline.plot_prediction(axes=ax)
 ```

 %% Output



    [[9.8303926e-01 1.6960735e-02]
     [6.2543350e-01 3.7456647e-01]
     [9.9613547e-01 3.8645437e-03]
     ...
     [9.9962878e-01 3.7124348e-04]
     [9.9982846e-01 1.7155796e-04]
     [9.9764913e-01 2.3508898e-03]]

    <AxesSubplot:>

 %% Cell type:code id:9193d12e tags:

 ``` python
 predictions = generate_predictions(series, base_model_name)
 ```

 %% Output

    run prediction on [cuda]: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6...0.6196446418762207 seconds
    run prediction on [cuda]: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847...0.8462226390838623 seconds
    run prediction on [cuda]: 88134a7c-00f9-4b82-b579-95ef47cb8a20...1.0194389820098877 seconds
    run prediction on [cuda]: 9c1913f3-e44d-40a0-a89a-feb2d3692494...0.030718564987182617 seconds

 %% Cell type:code id:f7096db7 tags:

 ``` python
 pseudo_filter = 'allnoise_correctedhwgt'
 for dataset in series:
    dataset.generate_feedback_areas(prediction=predictions[dataset.name])
    dataset.apply_pseudo_label_generators(pseudo_model_settings[pseudo_filter])
 ```

 %% Cell type:code id:d2d4574e tags:

 ``` python
 inc_model_name = 'gt_test3.pt'
 base_model_name = './data/DeepConvLSTMA_statedict.pt'
 ```

 %% Cell type:code id:387d9e22 tags:

 ``` python
 personalizer = Personalizer()
 personalizer.initialize(base_model_name)
 personalizer.incremental_learn_series_pseudo(series, save_model_as='./data/' + inc_model_name, epochs=100, use_regularization=False)
 ```

 %% Output

    Train on: e5fbd0a6-11c9-409b-a2ba-0d392fdb0af6
    Starting Training cuda


    loss before training: 0.09575944393873215
    F1-Score Val:	0.5333333333333333
    spec Val 0.9771241830065359
    sens Val 1.0
    
    Train on: 2252f3d0-cd90-42b4-9b4e-4172ed5fa847
    Starting Training cuda


    loss before training: 0.07606717944145203
    F1-Score Val:	0.15384615384615385
    spec Val 0.9753914988814317
    sens Val 1.0
    
    Train on: 88134a7c-00f9-4b82-b579-95ef47cb8a20
    Starting Training cuda


    loss before training: 0.025048060342669487
    stopped early! 50 0.05347004958285835
    F1-Score Val:	0.4
    spec Val 0.961038961038961
    sens Val 1.0
    
    Train on: 9c1913f3-e44d-40a0-a89a-feb2d3692494
    Starting Training cuda


    loss before training: 0.5482482314109802
    F1-Score Val:	0.0
    spec Val 0
    sens Val 0
    

 %% Cell type:code id:6981b0c1 tags:

 ``` python
 ```

--- a/src/notebooks/SyntheticLearning.ipynb
+++ b/src/notebooks/SyntheticLearning.ipynb
@@ -61,26 +61,26 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 15,
   "id": "d9aea435",
   "metadata": {},
   "outputs": [],
   "source": [
    "WINDOW_LENGTH = 150\n",
    "WINDOW_SHIFT = 75\n",
-    "inc_model_name = 'synthetic_inc_01.pt'\n",
+    "inc_model_name = 'synthetic_inc_reg_10.pt'\n",
    "predictions_db = './data/cluster/synthetic_learn_predictions_db'\n",
    "dataset_db = './data/synthetic_dataset_db'\n",
    "training_db = './data/cluster/pseudo_training_db'\n",
    "models_directory = './data/'\n",
-    "base_model = './data/HandWashingDeepConvLSTMA_trunc_01.pt'\n",
+    "base_model = './data/HandWashingDeepConvLSTMA_trunc_10.pt'\n",
    "training_run_name = 'test_base_synthetic_01'\n",
-    "training_collection = '01_training'"
+    "training_collection = '10_training'"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 16,
   "id": "4ea39d4e",
   "metadata": {},
   "outputs": [],
@@ -92,7 +92,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 12,
   "id": "b72e8659",
   "metadata": {},
   "outputs": [],
@@ -122,7 +122,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 17,
   "id": "1cb0a355",
   "metadata": {},
   "outputs": [
@@ -130,7 +130,7 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "dict_values([01_generated_2, 01_generated_3, 01_generated_4, 01_generated_5])\n"
+      "dict_values([10_generated_1, 10_generated_2, 10_generated_3])\n"
     ]
    }
   ],
@@ -1158,7 +1158,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 18,
   "id": "1fc8c4bf",
   "metadata": {
    "scrolled": false
@@ -1168,15 +1168,17 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "Train on: 01_generated_2\n",
-      "tensor([ 0.5030, 84.1154])\n",
-      "Starting Training cuda\n"
+      "Load base model: ./data/HandWashingDeepConvLSTMA_trunc_10.pt\n",
+      "Train on: 10_generated_1\n",
+      "tensor([ 0.5049, 51.6770])\n",
+      "Starting Training cuda\n",
+      "use regularization\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d458368f9e9e4f8981dcdf10dab204b3",
+       "model_id": "10f54099895a49bc890c0b6c82b109fc",
       "version_major": 2,
       "version_minor": 0
      },
@@ -1191,20 +1193,21 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "loss before training: 0.2243620604276657\n",
-      "F1-Score Val:\t0.20512820512820512\n",
-      "spec Val 0.9117647058823529\n",
-      "sens Val 0.8\n",
+      "stopped early! 15 0.07654275695616324\n",
+      "F1-Score Val:\t0.5\n",
+      "spec Val 0.9923664122137404\n",
+      "sens Val 1.0\n",
      " \n",
-      "Train on: 01_generated_3\n",
-      "tensor([ 0.5052, 48.5152])\n",
-      "Starting Training cuda\n"
+      "Train on: 10_generated_2\n",
+      "tensor([  0.5022, 112.9776])\n",
+      "Starting Training cuda\n",
+      "use regularization\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "016c01079f064fefa98495f9fe6da179",
+       "model_id": "59791d10726d497db1503f09c9e9c271",
       "version_major": 2,
       "version_minor": 0
      },
@@ -1219,20 +1222,21 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "loss before training: 0.34959739446640015\n",
-      "F1-Score Val:\t0.125\n",
-      "spec Val 0.88268156424581\n",
-      "sens Val 1.0\n",
+      "stopped early! 15 0.13930007410790635\n",
+      "F1-Score Val:\t0.6666666666666666\n",
+      "spec Val 1.0\n",
+      "sens Val 0.5\n",
      " \n",
-      "Train on: 01_generated_4\n",
-      "tensor([ 0.5074, 34.3200])\n",
-      "Starting Training cuda\n"
+      "Train on: 10_generated_3\n",
+      "tensor([ 0.5065, 39.0458])\n",
+      "Starting Training cuda\n",
+      "use regularization\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d21334ced4eb4e408aa0cd5af9f3fb81",
+       "model_id": "28cb92d457a84adba4ca784e7b45ac12",
       "version_major": 2,
       "version_minor": 0
      },
@@ -1247,44 +1251,25 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "loss before training: 0.190108522772789\n",
-      "F1-Score Val:\t0.1702127659574468\n",
-      "spec Val 0.8981723237597912\n",
-      "sens Val 1.0\n",
-      " \n",
-      "Train on: 01_generated_5\n",
-      "tensor([ 0.5049, 51.0110])\n",
-      "Starting Training cuda\n"
+      "stopped early! 15 0.04892555119461858\n",
+      "F1-Score Val:\t0.0\n",
+      "spec Val 0\n",
+      "sens Val 0\n",
+      " \n"
     ]
    },
    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "107d2c9db6194fceb3b8d365d1535d0f",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "  0%|          | 0/100 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
+     "name": "stderr",
     "output_type": "stream",
     "text": [
-      "loss before training: 0.5048531293869019\n",
-      "F1-Score Val:\t0.20689655172413793\n",
-      "spec Val 0.9258064516129032\n",
-      "sens Val 1.0\n",
-      " \n"
+      "/home/alex/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1492: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.\n",
+      "  _warn_prf(\n"
     ]
    }
   ],
   "source": [
-    "personalizer.incremental_learn_series_gt(list(series), save_model_as=models_directory + inc_model_name, epochs=100)\n",
+    "use_regularization = True\n",
+    "personalizer.incremental_learn_series_gt(list(series), save_model_as=models_directory + inc_model_name, epochs=100, use_regularization=use_regularization, freeze_feature_layers=not use_regularization)\n",
    "#trainings_manager.database['training_runs'][training_run_name] = []\n",
    "#trainings_manager.database['training_runs'][training_run_name].append(inc_model_name)\n",
    "#trainings_manager.db_update()"

 %% Cell type:code id:01c24b43 tags:

 ``` python
 %load_ext autoreload
 %autoreload 2

 %matplotlib notebook
 ```

 %% Cell type:code id:c68fe089 tags:

 ``` python
 import sys
 import os
 import numpy as np
 import matplotlib.pyplot as plt
 from IPython.display import display, Markdown
 import copy
 ```

 %% Cell type:code id:c7213856 tags:

 ``` python
 module_path = os.path.abspath(os.path.join('..'))
 os.chdir(module_path)
 if module_path not in sys.path:
    sys.path.append(module_path)
 ```

 %% Cell type:code id:3a8bad41 tags:

 ``` python
 from personalization_tools.load_data_sets import *
 from personalization_tools.helpers import *
 from personalization_tools.learner_pipeline import LearnerPipeline
 from personalization_tools.dataset_builder import *
 from personalization_tools.personalizer_synthetic import PersonalizerSynthetic
 from personalization_tools.dataset_manager import DatasetManager
 from personalization_tools.trainings_manager import TrainingsManager
 from personalization_tools.model_evaluation import ModelEvaluation
 from personalization_tools.pseudo_model_settings import pseudo_model_settings
 ```

 %% Cell type:code id:d9aea435 tags:

 ``` python
 WINDOW_LENGTH = 150
 WINDOW_SHIFT = 75
-inc_model_name = 'synthetic_inc_01.pt'
+inc_model_name = 'synthetic_inc_reg_10.pt'
 predictions_db = './data/cluster/synthetic_learn_predictions_db'
 dataset_db = './data/synthetic_dataset_db'
 training_db = './data/cluster/pseudo_training_db'
 models_directory = './data/'
-base_model = './data/HandWashingDeepConvLSTMA_trunc_01.pt'
+base_model = './data/HandWashingDeepConvLSTMA_trunc_10.pt'
 training_run_name = 'test_base_synthetic_01'
-training_collection = '01_training'
+training_collection = '10_training'
 ```

 %% Cell type:code id:4ea39d4e tags:

 ``` python
 personalizer = PersonalizerSynthetic()
 personalizer.initialize(base_model)
 # personalizer.load_data(['01', '02'])
 ```

 %% Cell type:code id:b72e8659 tags:

 ``` python
 dataset_manager = DatasetManager(dataset_db)
 ```

 %% Cell type:code id:e571f62e tags:

 ``` python
 trainings_manager = TrainingsManager(training_db)
 ```

 %% Cell type:code id:7e93e3c7 tags:

 ``` python
 series = dataset_manager.filter_by_category(training_collection).values()
 ```

 %% Cell type:code id:1cb0a355 tags:

 ``` python
 series = dataset_manager.filter_by_category(training_collection).values()
 print(series)
 ```

 %% Output

-    dict_values([01_generated_2, 01_generated_3, 01_generated_4, 01_generated_5])
+    dict_values([10_generated_1, 10_generated_2, 10_generated_3])

 %% Cell type:code id:ebac8347 tags:

 ``` python
 # series = personalizer.build_series_for_participant('01', 2, target_hand_wash_counts=2, target_recorded_hours = 1, reuse_data=False)
 personalizer.process_series(series)
 ```

 %% Output

    ---------------------------------------------------------------------------
    KeyboardInterrupt                         Traceback (most recent call last)
    <ipython-input-15-9ceed92af4b6> in <module>
          1 # series = personalizer.build_series_for_participant('01', 2, target_hand_wash_counts=2, target_recorded_hours = 1, reuse_data=False)
    ----> 2 personalizer.process_series(series)

    ~/gitrepos/uni/handwashing_personalizer/src/tools/personalizer_synthetic.py in process_series(self, series)
         13         for i, ser in enumerate(series):
         14             self.learner_pipeline.load_dataset(ser)
    ---> 15             predicted = self.learner_pipeline.predict_dataset()
         16             evaluation = self.learner_pipeline.evaluate_prediction()
         17             evaluations.append(evaluation)
    ~/gitrepos/uni/handwashing_personalizer/src/tools/learner_pipeline.py in predict_dataset(self)
        137
        138     def predict_dataset(self) -> np.ndarray:
    --> 139         return self.predict(self.data_set.x_win, self.data_set.name)
        140
        141     def plot_dataset(self, axes: plt.Axes = None):
    ~/gitrepos/uni/handwashing_personalizer/src/tools/learner_pipeline.py in predict(self, x_val, data_name)
        129                     x_pred = x_pred.to(device)
        130                     pred = model(x_pred)
    --> 131                     pred = pred.detach().cpu().numpy()
        132                     pred_complete[batch:batch + batch_size] = pred
        133                     del x_pred
    KeyboardInterrupt:

 %% Cell type:code id:bcc0fff7 tags:

 ``` python
 dataset_manager.update_dataset_collection(series)
 ```

 %% Cell type:code id:f36aed3f tags:

 ``` python
 personalizer.plot_series_windows(series)
 ```

 %% Output



 %% Cell type:code id:1fc8c4bf tags:

 ``` python
-personalizer.incremental_learn_series_gt(list(series), save_model_as=models_directory + inc_model_name, epochs=100)
+use_regularization = True
+personalizer.incremental_learn_series_gt(list(series), save_model_as=models_directory + inc_model_name, epochs=100, use_regularization=use_regularization, freeze_feature_layers=not use_regularization)
 #trainings_manager.database['training_runs'][training_run_name] = []
 #trainings_manager.database['training_runs'][training_run_name].append(inc_model_name)
 #trainings_manager.db_update()
 ```

 %% Output

-    Train on: 01_generated_2
-    tensor([ 0.5030, 84.1154])
-    Starting Training cuda
-
-
-    loss before training: 0.2243620604276657
-    F1-Score Val:	0.20512820512820512
-    spec Val 0.9117647058823529
-    sens Val 0.8
-    
-    Train on: 01_generated_3
-    tensor([ 0.5052, 48.5152])
+    Load base model: ./data/HandWashingDeepConvLSTMA_trunc_10.pt
+    Train on: 10_generated_1
+    tensor([ 0.5049, 51.6770])
    Starting Training cuda
+    use regularization


-    loss before training: 0.34959739446640015
-    F1-Score Val:	0.125
-    spec Val 0.88268156424581