Dev (#1)

* pose estimation integration to TCN

* pose generator

* version 6 features

* ptg data generator

* ptg datagenerator with poses and offset vectors

* resplit data

* top-n objects confs for vector generation

* clean and organize pose estimation into a single script

* pose generation in one script, union of datasets, selection by gids or vids

* step visualization, ptg model change, pose data generation in one script, new loss function

* ptg feat v6 with temporal features

---------

Co-authored-by: Peri Akiva <[email protected]>

.gitignore

@@ -143,6 +143,10 @@ dmypy.json
 *.h5
 *.tar
 *.tar.gz
+*.pth
+*.png
+*.jpg
+*.jpeg
 
 # Lightning-Hydra-Template
 configs/local/default.yaml

configs/experiment/m2/feat_v6.yaml

@@ -0,0 +1,73 @@
+# @package _global_
+
+# to execute this experiment run:
+# python train.py experiment=example
+
+defaults:
+  - override /data: ptg
+  - override /model: ptg
+  - override /callbacks: default
+  - override /trainer: gpu
+  - override /paths: default
+  - override /logger: aim
+
+# all parameters below will be merged with parameters from default configurations set above
+# this allows you to overwrite only specified parameters
+# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose" #[_v2_aug_False]
+exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_False_reshuffle_True" #[_v2_aug_False]
+# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_False_reshuffle_False" #[_v2_aug_False]
+# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_True" #[_v2_aug_False]
+# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_False" #[_v2_aug_False]
+
+
+tags: ["m2", "ms_tcn"]
+
+seed: 12345
+
+trainer:
+  min_epochs: 50
+  max_epochs: 500
+
+
+model:
+  compile: false
+
+  net:
+    dim: 182 # length of feature vector
+
+data:
+  num_classes: 9 # activities: includes background
+  batch_size: 512
+  num_workers: 0
+  epoch_length: 20000
+  window_size: 30
+  sample_rate: 1
+
+  all_transforms:
+    train_order: [] #["MoveCenterPts", "NormalizePixelPts"]
+    test_order: [] #["NormalizePixelPts"]
+
+    MoveCenterPts:
+        feat_version: 6
+        num_obj_classes: 9 # not including background
+    NormalizeFromCenter:
+      feat_version: 6
+    NormalizePixelPts:
+      feat_version: 6
+      num_obj_classes: 9 # not including background
+
+paths:
+  data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_False_reshuffle_True" #[_v2_aug_False]
+  # data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_False_reshuffle_False" #[_v2_aug_False]
+  # data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_True" #[_v2_aug_False]
+  # data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_False" #[_v2_aug_False]
+
+  # root_dir: "/data/PTG/medical/training/activity_classifier/TCN_HPL"
+  root_dir: "/data/users/peri.akiva/PTG/medical/training/activity_classifier/TCN_HPL"
+
+logger:
+  aim:
+    experiment: ${exp_name}
+    capture_terminal_logs: true
+
+task_name: ${exp_name}

configs/experiment/m2/feat_v6_no_pose.yaml

@@ -0,0 +1,64 @@
+# @package _global_
+
+# to execute this experiment run:
+# python train.py experiment=example
+
+defaults:
+  - override /data: ptg
+  - override /model: ptg
+  - override /callbacks: default
+  - override /trainer: gpu
+  - override /paths: default
+  - override /logger: aim
+
+# all parameters below will be merged with parameters from default configurations set above
+# this allows you to overwrite only specified parameters
+exp_name: "p_m2_tqt_data_test_feat_v6_no_pose" #[p_m2_tqt_data_test_feat_v6]
+
+tags: ["m2", "ms_tcn"]
+
+seed: 12345
+
+trainer:
+  min_epochs: 50
+  max_epochs: 400
+
+
+model:
+  compile: false
+
+  net:
+    dim: 288 # length of feature vector
+
+data:
+  num_classes: 9 # activities: includes background
+  batch_size: 256
+  num_workers: 0
+  epoch_length: 20000
+  window_size: 30
+  #sample_rate: 2
+
+  all_transforms:
+    train_order: [] #["MoveCenterPts", "NormalizePixelPts"]
+    test_order: [] #["NormalizePixelPts"]
+
+    MoveCenterPts:
+        feat_version: 5
+        num_obj_classes: 12 # not including background
+    NormalizeFromCenter:
+      feat_version: 5
+    NormalizePixelPts:
+      feat_version: 5
+      num_obj_classes: 12 # not including background
+
+paths:
+  data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_no_pose" #[p_m2_tqt_data_test_feat_v6]
+  # root_dir: "/data/PTG/medical/training/activity_classifier/TCN_HPL"
+  root_dir: "/data/users/peri.akiva/PTG/medical/training/activity_classifier/TCN_HPL"
+
+logger:
+  aim:
+    experiment: ${exp_name}
+    capture_terminal_logs: true
+
+task_name: ${exp_name}

configs/experiment/m2/feat_v6_only_hands_joints.yaml

@@ -0,0 +1,64 @@
+# @package _global_
+
+# to execute this experiment run:
+# python train.py experiment=example
+
+defaults:
+  - override /data: ptg
+  - override /model: ptg
+  - override /callbacks: default
+  - override /trainer: gpu
+  - override /paths: default
+  - override /logger: aim
+
+# all parameters below will be merged with parameters from default configurations set above
+# this allows you to overwrite only specified parameters
+exp_name: "p_m2_tqt_data_test_feat_v6_only_hands_joints_v2" #[p_m2_tqt_data_test_feat_v6]
+
+tags: ["m2", "ms_tcn"]
+
+seed: 12345
+
+trainer:
+  min_epochs: 50
+  max_epochs: 400
+
+
+model:
+  compile: false
+
+  net:
+    dim: 332 # length of feature vector
+
+data:
+  num_classes: 9 # activities: includes background
+  batch_size: 256
+  num_workers: 0
+  epoch_length: 20000
+  window_size: 30
+  #sample_rate: 2
+
+  all_transforms:
+    train_order: [] #["MoveCenterPts", "NormalizePixelPts"]
+    test_order: [] #["NormalizePixelPts"]
+
+    MoveCenterPts:
+        feat_version: 6
+        num_obj_classes: 12 # not including background
+    NormalizeFromCenter:
+      feat_version: 6
+    NormalizePixelPts:
+      feat_version: 6
+      num_obj_classes: 12 # not including background
+
+paths:
+  data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_only_hands_joints" #[p_m2_tqt_data_test_feat_v6]
+  # root_dir: "/data/PTG/medical/training/activity_classifier/TCN_HPL"
+  root_dir: "/data/users/peri.akiva/PTG/medical/training/activity_classifier/TCN_HPL"
+
+logger:
+  aim:
+    experiment: ${exp_name}
+    capture_terminal_logs: true
+
+task_name: ${exp_name}

configs/experiment/m2/feat_v6_only_object_joints.yaml

@@ -0,0 +1,64 @@
+# @package _global_
+
+# to execute this experiment run:
+# python train.py experiment=example
+
+defaults:
+  - override /data: ptg
+  - override /model: ptg
+  - override /callbacks: default
+  - override /trainer: gpu
+  - override /paths: default
+  - override /logger: aim
+
+# all parameters below will be merged with parameters from default configurations set above
+# this allows you to overwrite only specified parameters
+exp_name: "p_m2_tqt_data_test_feat_v6_only_objects_joints" #[p_m2_tqt_data_test_feat_v6]
+
+tags: ["m2", "ms_tcn"]
+
+seed: 12345
+
+trainer:
+  min_epochs: 50
+  max_epochs: 400
+
+
+model:
+  compile: false
+
+  net:
+    dim: 310 # length of feature vector
+
+data:
+  num_classes: 9 # activities: includes background
+  batch_size: 256
+  num_workers: 0
+  epoch_length: 20000
+  window_size: 30
+  #sample_rate: 2
+
+  all_transforms:
+    train_order: [] #["MoveCenterPts", "NormalizePixelPts"]
+    test_order: [] #["NormalizePixelPts"]
+
+    MoveCenterPts:
+        feat_version: 6
+        num_obj_classes: 12 # not including background
+    NormalizeFromCenter:
+      feat_version: 6
+    NormalizePixelPts:
+      feat_version: 6
+      num_obj_classes: 12 # not including background
+
+paths:
+  data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_only_objects_joints" #[p_m2_tqt_data_test_feat_v6]
+  # root_dir: "/data/PTG/medical/training/activity_classifier/TCN_HPL"
+  root_dir: "/data/users/peri.akiva/PTG/medical/training/activity_classifier/TCN_HPL"
+
+logger:
+  aim:
+    experiment: ${exp_name}
+    capture_terminal_logs: true
+
+task_name: ${exp_name}

configs/model/ptg.yaml

@@ -3,8 +3,8 @@ _target_: tcn_hpl.models.ptg_module.PTGLitModule
 optimizer:
   _target_: torch.optim.Adam
   _partial_: true
-  lr: 0.0005
-  weight_decay: 0.0
+  lr: 0.00001
+  weight_decay: 0.0001
 
 scheduler:
   _target_: torch.optim.lr_scheduler.ReduceLROnPlateau
@@ -15,23 +15,27 @@ scheduler:
 
 net:
   _target_: tcn_hpl.models.components.ms_tcs_net.MultiStageModel
-  num_stages: 4 
-  num_layers: 10 
-  num_f_maps: 64
-  dim: 204
+  num_stages: 4
+  num_layers: 5
+  num_f_maps: 128
+  # dim: 204
+  dim: 128
   num_classes: ${data.num_classes}
+  window_size: ${data.window_size}
 
 criterion:
   _target_: tcn_hpl.models.components.focal_loss.FocalLoss
-  alpha: 0.25 
-  gamma: 2
-  weight: None 
+  alpha: 0.25 # 0.25
+  gamma: 1
+  # weight: None 
+  weight: [1, 1, 1, 1, 1, 0.75, 0.75, 1, 1]
   reduction: "mean"
 
 data_dir: ${paths.data_dir}
 
 # Smoothing loss weight
-smoothing_loss: 0.015
+smoothing_loss: 0.0015
+use_smoothing_loss: True
 
 # Number of classes
 num_classes: ${data.num_classes}

environment.yaml

@@ -5,7 +5,7 @@
 # - conda allows for installing packages without requiring certain compilers or
 #   libraries to be available in the system, since it installs precompiled binaries
 
-name: myenv
+name: ptg
 
 channels:
   - pytorch

requirements.txt

@@ -1,13 +1,13 @@
 # --------- pytorch --------- #
-torch>=2.0.0
-torchvision>=0.15.0
-lightning>=2.0.0
-torchmetrics>=0.11.4
+# torch>=2.0.0
+# torchvision>=0.15.0
+# lightning>=2.0.0
+# torchmetrics>=0.11.4
 
 # --------- hydra --------- #
-hydra-core==1.3.2
-hydra-colorlog==1.2.0
-hydra-optuna-sweeper==1.2.0
+# hydra-core==1.3.2
+# hydra-colorlog==1.2.0
+# hydra-optuna-sweeper==1.2.0
 
 # --------- loggers --------- #
 # wandb

setup.py

@@ -9,8 +9,8 @@
     author="",
     author_email="",
     url="https://github.com/user/project",
-    install_requires=["lightning", "hydra-core", "hydra-colorlog"],
-    packages=find_packages(),
+    # install_requires=["lightning", "hydra-core", "hydra-colorlog"],
+    packages=find_packages(include=['tcn_hpl', 'tcn_hpl.*']),
     # use this to customize global commands available in the terminal after installing the package
     entry_points={
         "console_scripts": [

tcn_hpl/data/components/PTG_dataset.py

@@ -107,10 +107,26 @@ def __getitem__(self, idx):
 
         :return: features, targets, and mask of the window
         """
+
+        # print(f"size of dataset: {self.__len__()}")
+        # print(f"self.feature_frames: {self.feature_frames.shape}")
+        # print(f"self.target_frames: {self.target_frames.shape}")
+        # print(f"self.mask_frames: {self.mask_frames.shape}")
+        # print(f"self.source_vids: {self.source_vids.shape}")
+        # print(f"self.source_frames: {self.source_frames.shape}")
+        # print(f"self.mask_frames: {self.mask_frames}")
+
         features = self.feature_frames[idx:idx+self.window_size, :]
         target = self.target_frames[idx:idx+self.window_size]
         mask = self.mask_frames[idx:idx+self.window_size]
         source_vid = self.source_vids[idx:idx+self.window_size]
         source_frame = self.source_frames[idx:idx+self.window_size]
 
+        # print(f"mask: {mask}")
+        # print(f"features: {features.shape}")
+        # print(f"target: {target.shape}")
+        # print(f"mask: {mask.shape}")
+        # print(f"source_vid: {source_vid.shape}")
+        # print(f"source_frame: {source_frame.shape}")
+
         return features, target, mask, np.array(source_vid), source_frame