Merge branch 'main' into framework/new_materials

README.md

@@ -34,7 +34,7 @@ InnoFW uses hydra to provide configuration structure that is suitable for the mo
 
 1. Create an experiment config file in the folder ```config/experiments/``` based on ```config/experiments/template.yaml```.
 2. Once you define your configuration file you can start training your model.
-    ```python train.py experiment=yolov5_cars```
+    ```python train.py experiments=yolov5_cars```
 3. InnoFW checks the configuration file for consistency of individual modules(model, dataset, loss, optimizer etc.) and if everything is fine then selects and adapter. Adapter is responsible for starting the training, testing, validation and inference pipeline.
 4. Model is being trained and checkpoints saved.
 

config/augmentations/preprocessing/sentinel2.yaml

@@ -0,0 +1,5 @@
+divide_by_10000:
+  _target_: innofw.core.augmentations.preprocessing.ToFloatWClip
+  max_value: 10000
+  p: 1.0
+  always_apply: True

config/augmentations_test/arable_sentinel2.yaml

@@ -0,0 +1,8 @@
+defaults:
+    - /augmentations/preprocessing:
+      - sentinel2
+    - /augmentations/combined:
+        - none
+    - /augmentations/position:
+    - /augmentations/color:
+    - /augmentations/postprocessing:

config/augmentations_train/arable-ndvi-min.yaml

@@ -0,0 +1,15 @@
+defaults:
+    - /augmentations/preprocessing:
+    - /augmentations/combined:
+        - none
+    - /augmentations/position:
+        - random_resized_crop
+    - /augmentations/color:
+    - /augmentations/postprocessing:
+
+
+augmentations:
+    position:
+        random_crop:
+            height: 224
+            width: 224

config/augmentations_train/arable-ndvi-strong.yaml

@@ -0,0 +1,31 @@
+defaults:
+    - /augmentations/preprocessing:
+    - /augmentations/combined:
+        - none
+    - /augmentations/position:
+        - random_resized_crop
+        - flip
+        - rotate90
+        - transpose
+        - grid_distortion
+    - /augmentations/color:
+        - random_gamma
+        - blur
+    - /augmentations/postprocessing:
+
+
+augmentations:
+    position:
+        flip:
+            p: 0.75
+        rotate90:
+            p: 0.75
+        transpose:
+            p: 0.75
+        grid_distortion:
+            p: 0.75
+    color:
+        random_gamma:
+            p: 0.75
+        blur:
+            p: 0.75

config/augmentations_train/arable-ndvi.yaml

@@ -0,0 +1,14 @@
+defaults:
+    - /augmentations/preprocessing:
+    - /augmentations/combined:
+        - none
+    - /augmentations/position:
+        - random_resized_crop
+        - flip
+        - rotate90
+        - transpose
+        - grid_distortion
+    - /augmentations/color:
+        - random_gamma
+        - blur
+    - /augmentations/postprocessing:

config/augmentations_train/arable_sentinel2.yaml

@@ -0,0 +1,8 @@
+defaults:
+    - /augmentations/preprocessing:
+      - sentinel2
+    - /augmentations/combined:
+        - none
+    - /augmentations/position:
+    - /augmentations/color:
+    - /augmentations/postprocessing:

config/augmentations_val/arable-ndvi-min.yaml

@@ -0,0 +1,15 @@
+defaults:
+    - /augmentations/preprocessing:
+    - /augmentations/combined:
+        - none
+    - /augmentations/position:
+        - random_resized_crop
+    - /augmentations/color:
+    - /augmentations/postprocessing:
+
+
+augmentations:
+    position:
+        random_crop:
+            height: 224
+            width: 224

config/augmentations_val/arable-ndvi-strong.yaml

@@ -0,0 +1,31 @@
+defaults:
+    - /augmentations/preprocessing:
+    - /augmentations/combined:
+        - none
+    - /augmentations/position:
+        - random_resized_crop
+        - flip
+        - rotate90
+        - transpose
+        - grid_distortion
+    - /augmentations/color:
+        - random_gamma
+        - blur
+    - /augmentations/postprocessing:
+
+
+augmentations:
+    position:
+        flip:
+            p: 0.75
+        rotate90:
+            p: 0.75
+        transpose:
+            p: 0.75
+        grid_distortion:
+            p: 0.75
+    color:
+        random_gamma:
+            p: 0.75
+        blur:
+            p: 0.75

config/augmentations_val/arable-ndvi.yaml

@@ -0,0 +1,14 @@
+defaults:
+    - /augmentations/preprocessing:
+    - /augmentations/combined:
+        - none
+    - /augmentations/position:
+        - random_resized_crop
+        - flip
+        - rotate90
+        - transpose
+        - grid_distortion
+    - /augmentations/color:
+        - random_gamma
+        - blur
+    - /augmentations/postprocessing:

config/augmentations_val/arable_sentinel2.yaml

@@ -0,0 +1,8 @@
+defaults:
+    - /augmentations/preprocessing:
+      - sentinel2
+    - /augmentations/combined:
+        - none
+    - /augmentations/position:
+    - /augmentations/color:
+    - /augmentations/postprocessing:

config/datasets/detection/air_data.yaml

@@ -0,0 +1,28 @@
+task:
+  - image-detection
+
+name: lep
+description: Набор данных содержит видео облетов c дронов над территорией линии электропередач
+
+markup_info: Набор данных содержит разметку bounding box, под формат детекции столбов ЛЭП.
+date_time: 18.07.2022
+
+_target_: innofw.core.integrations.ultralytics.datamodule.UltralyticsDataModuleAdapter
+
+train:
+test:
+infer: 
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/air_data/train.zip
+  target: ./data/air_data/
+
+num_workers: 8
+
+val_size: 0.2
+channels_num: 3
+image_size: 600
+num_classes: 4
+names:
+  - lep_1
+  - lep_2
+  - lep_3
+  - lep_4

config/datasets/detection/auto.yaml

@@ -0,0 +1,28 @@
+task:
+  - image-detection
+
+name: lep
+description: Набор данных содержит видео облетов c дронов над территорией линии электропередач
+
+markup_info: Набор данных содержит разметку bounding box, под формат детекции столбов ЛЭП.
+date_time: 18.07.2022
+
+_target_: innofw.core.integrations.ultralytics.datamodule.UltralyticsDataModuleAdapter
+
+train:
+test:
+infer: 
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/auto/train.zip
+  target: ./data/auto/
+
+num_workers: 8
+
+val_size: 0.2
+channels_num: 3
+image_size: 600
+num_classes: 4
+names:
+  - lep_1
+  - lep_2
+  - lep_3
+  - lep_4

config/datasets/detection/detection_rotten_fruits.yaml

@@ -7,8 +7,7 @@ description: Набор данных содержит фото гнилых и 
 markup_info: Набор данных содержит разметку bounding box, под формат детекции гнилых фруктов.
 date_time: 19.07.2022
 
-#_target_: innofw.core.datamodules.lightning_datamodules.YOLOv5DataModule
-_target_: innofw.core.integrations.ultralytics.datamodule.YOLOV5DataModuleAdapter
+_target_: innofw.core.integrations.ultralytics.datamodule.UltralyticsDataModuleAdapter
 
 train:
   source: https://api.blackhole.ai.innopolis.university/public-datasets/detection_rotten_fruits/train.zip

config/datasets/detection/wares_sort.yaml

@@ -0,0 +1,28 @@
+task:
+  - image-detection
+
+name: sort_data
+description: Набор данных содержит видео облетов c дронов над территорией линии электропередач
+
+markup_info: Набор данных содержит разметку bounding box, под формат детекции столбов ЛЭП.
+date_time: 18.07.2022
+
+_target_: innofw.core.integrations.ultralytics.datamodule.UltralyticsDataModuleAdapter
+
+train:
+test:
+infer: 
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/wares_sort/train.zip
+  target: ./data/wares_sort/
+
+num_workers: 8
+
+val_size: 0.2
+channels_num: 3
+image_size: 600
+num_classes: 4
+names:
+  - lep_1
+  - lep_2
+  - lep_3
+  - lep_4

config/datasets/detection/detection_lep.yaml → config/datasets/detection_lep.yaml

@@ -1,26 +1,23 @@
 task:
   - image-detection
 
-name: detection_lep
+name: lep
 description: Набор данных содержит видео облетов c дронов над территорией линии электропередач
 
 markup_info: Набор данных содержит разметку bounding box, под формат детекции столбов ЛЭП.
 date_time: 18.07.2022
 
-#_target_: innofw.core.datamodules.lightning_datamodules.detection.YOLOv5DataModule
-_target_: innofw.core.integrations.ultralytics.datamodule.YOLOV5DataModuleAdapter
-
+_target_: innofw.core.integrations.ultralytics.datamodule.UltralyticsDataModuleAdapter
 
 train:
   source: https://api.blackhole.ai.innopolis.university/public-datasets/testing/lep/train.zip
-  target: ./data/lep/train
+  target: ./data/lep/
 test:
   source: https://api.blackhole.ai.innopolis.university/public-datasets/testing/lep/test.zip
-  target: ./data/lep/test
+  target: ./data/lep/
 infer:
   source: https://api.blackhole.ai.innopolis.university/public-datasets/testing/lep/test.zip
-  target: ./data/lep/infer
-
+  target: ./data/lep/
 
 num_workers: 8
 
@@ -33,3 +30,26 @@ names:
   - lep_2
   - lep_3
   - lep_4
+
+#  Dataset structure is as follows:
+#  images/
+#    train/
+#      *.PNG
+#    test/
+#      *.PNG
+
+#  labels/
+#    train/
+#      *.txt
+#    test/
+#      *.txt
+
+#  Note that each archive consist only corresponding folders:
+#  train.zip/
+#    images/
+#      train/
+#        *.PNG
+#    labels/
+#      train/
+#        *.txt
+#  Similar for test.zip and infer.zip

config/datasets/semantic-segmentation/arable-bin-seg-ndvi/fields/tile-512-90-days.yaml

@@ -0,0 +1,22 @@
+_target_: innofw.core.datamodules.lightning_datamodules.semantic_segmentation.tiff.SegmentationDM
+train:
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/testing/arable_ndvi/train.zip
+  target: data/arable_ndvi/train/
+
+test:
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/testing/arable_ndvi/test.zip
+  target: data/arable_ndvi/test/
+
+img_foldername: img
+label_foldername: gt
+
+num_workers: 32
+batch_size: 16
+channels: 3
+
+
+name: Arable_NDVI
+task: image-segmentation
+description: Binary semantic segmentation dataset with arable images NDVI
+markup_info: some
+date_time: 010923

config/datasets/semantic-segmentation/landslides-kanopus/270223-hdf5.yaml

@@ -1,4 +1,4 @@
-_target_: innofw.core.datamodules.lightning_datamodules.segmentation_hdf5_dm.HDF5LightningDataModule  # todo:
+_target_: innofw.core.datamodules.lightning_datamodules.HDF5LightningDataModule  # todo:
 train:
   source: /mnt/nvmestorage/qb/data_n_weights/landslides-bin-seg-s2/210223/processed/train
 

config/datasets/semantic-segmentation/linear-road-bin-seg-oftp/tile-2048-original.yaml

@@ -1,4 +1,4 @@
-_target_: innofw.core.datamodules.lightning_datamodules.semantic_segmentation.tiff_dm.SegmentationDM
+_target_: innofw.core.datamodules.lightning_datamodules.semantic_segmentation.tiff.SegmentationDM
 # ${.root}
 train:
   source: /mnt/nvmestorage/qb/data_n_weights/linear-road-bin-seg-oftp/301122/processed/070123-167folders-2048/

config/datasets/semantic-segmentation/water-erosion-kanopus/220223-hdf5.yaml

@@ -1,4 +1,4 @@
-_target_: innofw.core.datamodules.lightning_datamodules.segmentation_hdf5_dm.HDF5LightningDataModule  # todo:
+_target_: innofw.core.datamodules.lightning_datamodules.HDF5LightningDataModule
 train:
   source: /mnt/nvmestorage/qb/data_n_weights/water_erosion/220223/processed/train/
 

config/experiments/anomaly-detection/IM_040822_rbe23lls_anomaly_detection_timeseries_lstm.yaml

@@ -3,7 +3,7 @@ defaults:
   - override /models: anomaly-detection/lstm_autoencoder.yaml
   - override /datasets: anomaly_detection_timeseries_ecg.yaml
   - override /losses: l1.yaml
-
+  - override /schedulers: cosine_annealing
 
 project: "ecg"
 task: "anomaly-detection-timeseries"

config/experiments/classification/SK_010822_jvslrmvk_resnet.yaml

@@ -7,6 +7,7 @@ defaults:
   - override /augmentations_test: none
   - override /losses: log_loss.yaml
   - override /optimizers: adam
+  - override /schedulers:
 
 project: "mnist_classification"
 task: "image-classification"

config/experiments/clustering/demo_queue.yaml

@@ -0,0 +1,9 @@
+# @package _global_
+defaults:
+- override /models: clustering/kmeans
+- override /datasets: clustering/clustering_credit_cards
+- override /callbacks: clustering
+- override /clear_ml: test_queue
+project: credit_cards
+random_seed: '42'
+task: table-clustering