diff --git a/configs/experiment/m2/feat_v6.yaml b/configs/experiment/m2/feat_v6.yaml index b54e8fef1..5546bb2f0 100644 --- a/configs/experiment/m2/feat_v6.yaml +++ b/configs/experiment/m2/feat_v6.yaml @@ -3,6 +3,9 @@ # to execute this experiment run: # python train.py experiment=example task: "m2" +feature_version: 6 +topic: "medical" + defaults: - override /data: ptg - override /model: ptg @@ -13,16 +16,10 @@ defaults: # all parameters below will be merged with parameters from default configurations set above # this allows you to overwrite only specified parameters -# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose" #[_v2_aug_False] -# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v3_aug_False_reshuffle_True" #[_v2_aug_False] -exp_name: "p_m2_feat_v6_with_pose_v3_aug_False_reshuffle_True" -# exp_name: "p_m2_feat_v6_no_pose_v3_aug_False_reshuffle_True" -# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_False_reshuffle_False" #[_v2_aug_False] -# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_True" #[_v2_aug_False] -# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_False" #[_v2_aug_False] +# exp_name: "p_m2_feat_v6_with_pose_v3_aug_False_reshuffle_True" -tags: ["m2", "ms_tcn"] +tags: ["m2", "ms_tcn", "debug"] seed: 12345 @@ -35,15 +32,61 @@ model: compile: false net: - dim: 209 # length of feature vector + dim: 297 # length of feature vector + +IMAGE_HZ: 30 # zed bags +#IMAGE_HZ: 15 # BBN hololens live +OBJECT_DET_HZ: 15 +POSE_HZ: 4 + + + +data_gen: + top_k_objects: 1 + pose_repeat_rate: 7.5 + num_augs: 5 + + data_type: "pro" + # dataset_kwcoco: "/data/users/peri.akiva/datasets/ptg/m2_all_all_obj_results_with_dets_and_pose.mscoco.json" + dataset_kwcoco: "/data/PTG/medical/coco/m2_all_all_obj_results_with_dets_and_pose.mscoco.json" + + + activity_config_fn: "/home/local/KHQ/peri.akiva/projects/angel_system/config/activity_labels/medical/m2.yaml" + exp_ext: "_NEW_ORDER_fix_overlap_gt_step_gt" # anything unique about this run that isn't already in ``exp_name`` + exp_name: "${task}_${data_gen.data_type}_data_top_${data_gen.top_k_objects}_objs_feat_v${feature_version}_pose_rate_${data_gen.pose_repeat_rate}${data_gen.exp_ext}" + raw_data_root: /data/PTG/medical/bbn_data/Release_v0.5/v0.56 + dive_output_root: /data/PTG/medical/activity_anns + + filter_black_gloves: false + filter_blue_gloves: false + train_vid_ids: [1, 2, 4, 8, 9, 10, 11, 12, 16, 17,18, 20, 19, 30, 31, 32, 33, 34,35,36, + 7,132,133,50,51,54,56,52,61,53,57,65,66,67,68,69,58,60,64,125,126, + 127,129,131,134,135,136,119,122,124,70,72,92,93,94,95,97,98,100, + 101,102,103,104,105,107,108,112,114,117,118,73] + val_vid_ids: [5, 59,106,130,138, 77, 123, 71] + test_vid_ids: [3,14,55,62,96,109,128,137,139, 120, 75, 21, 13] + + names_black_gloves: [22,23,26,24,25,27,29,28,41,42,43,44,45,46,47,48,49,78, + 79,84,88,90,80,81,82,83,85,86,87,89,91,99,110,111,121,113,115,116] + names_blue_gloves: [132,133,50,51,54,55,56,52,61,59,53,57,62,65,66,67,68,69, + 58,60,63,64,125,126,127,129,131,134,135,136,128,130,137, + 138,139] + + train_vid_ids_bbn: [3, 4, 5, 9, 10, 11, 12, + 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 31, 32, 33, 34,35,36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 48, 49, + 51, 52, 53, 54] + val_vid_ids_bbn: [1, 6, 7, 29, 50] + test_vid_ids_bbn: [2, 8, 30, 47] data: num_classes: 9 # activities: includes background batch_size: 512 num_workers: 16 epoch_length: 20000 - window_size: 30 - sample_rate: 1 + window_size: 25 + sample_rate: 2 all_transforms: train_order: [] #["MoveCenterPts", "NormalizePixelPts"] @@ -58,46 +101,19 @@ data: feat_version: 6 num_obj_classes: 9 # not including background -data_gen: - reshuffle_datasets: true - augment: false - num_augs: 5 - feat_type: "with_pose" #[no_pose, with_pose, only_hands_joints, only_objects_joints] - filter_black_gloves: false - filter_blue_gloves: false - train_vid_ids: [1, 2, 4, 8, 9, 10, 11, 12, 16, 17,18, 20, 19, 30, 31, 32, 33, 34,35,36, - 7,132,133,50,51,54,56,52,61,53,57,65,66,67,68,69,58,60,64,125,126, - 127,129,131,134,135,136,119,122,124,70,72,92,93,94,95,97,98,100, - 101,102,103,104,105,107,108,112,114,117,118,73] - val_vid_ids: [5, 59,106,130,138, 77, 123, 71] - test_vid_ids: [3,14,55,62,96,109,128,137,139, 120, 75, 21, 13] - names_black_gloves: [22,23,26,24,25,27,29,28,41,42,43,44,45,46,47,48,49,78, - 79,84,88,90,80,81,82,83,85,86,87,89,91,99,110,111,121,113,115,116] - names_blue_gloves: [132,133,50,51,54,55,56,52,61,59,53,57,62,65,66,67,68,69, - 58,60,63,64,125,126,127,129,131,134,135,136,128,130,137, - 138,139] +exp_name: ${data_gen.exp_name}_win_${data.window_size}_obj_sample_${data.sample_rate} paths: - # data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_False_reshuffle_True" #[_v2_aug_False] - data_dir: "/data/PTG/TCN_data/m2/p_m2_feat_v6_with_pose_v3_aug_False_reshuffle_True" #[_v2_aug_False] - # data_dir: "/data/PTG/TCN_data/m2/p_m2_feat_v6_no_pose_v3_aug_False_reshuffle_True" #[_v2_aug_False] - # data_dir: "/data/PTG/TCN_data/m2/p_m2_feat_v6_no_pose_v3_aug_False_reshuffle_True" #[_v2_aug_False] - # data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_False_reshuffle_False" #[_v2_aug_False] - # data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_True" #[_v2_aug_False] - # data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_False" #[_v2_aug_False] - - # root_dir: "/data/PTG/medical/training/activity_classifier/TCN_HPL" - root_dir: "/data/users/peri.akiva/PTG/medical/training/activity_classifier/TCN_HPL" - - # dataset_kwcoco: "/data/users/peri.akiva/datasets/ptg/m2_good_images_only_no_amputation_stump_train_activity_obj_results_with_dets_and_pose.mscoco.json" + data_dir: "/data/PTG/medical/training/activity_classifier/TCN_data/${task}/${data_gen.exp_name}" + root_dir: "/data/PTG/medical/training/activity_classifier/TCN_HPL/" dataset_kwcoco: "/data/users/peri.akiva/datasets/ptg/m2_all_all_obj_results_with_dets_and_pose.mscoco.json" - activity_config_root: "/home/local/KHQ/peri.akiva/projects/angel_system/config/activity_labels" + dataset_kwcoco_lab: + activity_config_root: "/home/local/KHQ/peri.akiva/projects/angel_system/config/activity_labels/medical" activity_config_fn: "${paths.activity_config_root}/${task}" ptg_root: "/home/local/KHQ/peri.akiva/angel_system" activity_config_path: "${paths.ptg_root}/config/activity_labels/medical" output_data_dir_root: "/data/PTG/TCN_data" - # bbn_data_root: "{bbn_data_dir}/Release_v0.5/v0.56" bbn_data_dir: "/data/PTG/medical/bbn_data" diff --git a/configs/experiment/m3/feat_v6.yaml b/configs/experiment/m3/feat_v6.yaml index d90fb84bc..75a2e7e0e 100644 --- a/configs/experiment/m3/feat_v6.yaml +++ b/configs/experiment/m3/feat_v6.yaml @@ -3,6 +3,9 @@ # to execute this experiment run: # python train.py experiment=example task: "m3" +feature_version: 6 +topic: "medical" + defaults: - override /data: ptg - override /model: ptg @@ -13,16 +16,10 @@ defaults: # all parameters below will be merged with parameters from default configurations set above # this allows you to overwrite only specified parameters -# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose" #[_v2_aug_False] -# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v3_aug_False_reshuffle_True" #[_v2_aug_False] -exp_name: "p_m3_feat_v6_with_pose_v3_aug_False_reshuffle_True" -# exp_name: "p_m2_feat_v6_no_pose_v3_aug_False_reshuffle_True" -# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_False_reshuffle_False" #[_v2_aug_False] -# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_True" #[_v2_aug_False] -# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_False" #[_v2_aug_False] +# exp_name: "p_m3_feat_v6_with_pose_v3_aug_False_reshuffle_True" -tags: ["m3", "ms_tcn"] +tags: ["m3", "ms_tcn", "debug"] seed: 12345 @@ -35,15 +32,51 @@ model: compile: false net: - dim: 188 # length of feature vector + dim: 246 # length of feature vector + + + +data_gen: + top_k_objects: 1 + pose_repeat_rate: 7.5 + num_augs: 5 + feat_type: "with_pose" #[no_pose, with_pose, only_hands_joints, only_objects_joints] + + data_type: "pro" + # dataset_kwcoco: "/data/users/peri.akiva/datasets/ptg/m2_all_all_obj_results_with_dets_and_pose.mscoco.json" + dataset_kwcoco: "/data/PTG/medical/coco/m3_all_all_obj_results_with_dets_and_pose.mscoco.json" + + activity_config_fn: "/home/local/KHQ/peri.akiva/projects/angel_system/config/activity_labels/medical/m3.yaml" + exp_ext: "_NEW_ORDER_fix_overlap_gt_step_gt" # anything unique about this run that isn't already in ``exp_name`` + exp_name: "${task}_${data_gen.data_type}_data_top_${data_gen.top_k_objects}_objs_feat_v${feature_version}_pose_rate_${data_gen.pose_repeat_rate}${data_gen.exp_ext}" + raw_data_root: /data/PTG/medical/bbn_data/Release_v0.5/v0.56 + dive_output_root: /data/PTG/medical/activity_anns + + filter_black_gloves: false + filter_blue_gloves: false + train_vid_ids: [1, 2, 4, 5, 6, 8, 9, 11, 12, 15, 16, 17, 19, 20, 21, + 22, 23, 24, 26, 28, 29, 30, 31, 33, 34, 35, 36, 37, 38, 39, 40, + 42, 44, 45, 48, 49, 51, 52, 53, 54] + val_vid_ids: [3, 7, 10, 18, 27, 32] + test_vid_ids: [50, 13, 47, 25] + names_black_gloves: [22,23,26,24,25,27,29,28,41,42,43,44,45,46,47,48,49,78, + 79,84,88,90,80,81,82,83,85,86,87,89,91,99,110,111,121,113,115,116] + names_blue_gloves: [132,133,50,51,54,55,56,52,61,59,53,57,62,65,66,67,68,69, + 58,60,63,64,125,126,127,129,131,134,135,136,128,130,137, + 138,139] + train_vid_ids_bbn: [1, 2, 3, 4, 5, 6, 7, 8] + val_vid_ids_bbn: [9] + test_vid_ids_bbn: [10] + + data: num_classes: 6 # activities: includes background batch_size: 512 - num_workers: 0 + num_workers: 16 epoch_length: 20000 - window_size: 45 - sample_rate: 1 + window_size: 25 + sample_rate: 2 all_transforms: train_order: [] #["MoveCenterPts", "NormalizePixelPts"] @@ -58,45 +91,19 @@ data: feat_version: 6 num_obj_classes: 4 # not including background -data_gen: - reshuffle_datasets: true - augment: false - num_augs: 5 - feat_type: "with_pose" #[no_pose, with_pose, only_hands_joints, only_objects_joints] - filter_black_gloves: false - filter_blue_gloves: false - train_vid_ids: [1, 2, 4, 5, 6, 8, 9, 11, 12, 14, 15, 16, 17, 19, 20, 21, - 22, 23, 24, 26, 28, 29, 30, 31, 33, 34, 35, 36, 37, 38, 39, 40, - 42, 43, 44, 45, 46, 48, 49, 51, 52, 53, 54] - val_vid_ids: [3, 7, 10, 18, 27, 32, 41] - test_vid_ids: [50, 13, 47, 25] - names_black_gloves: [22,23,26,24,25,27,29,28,41,42,43,44,45,46,47,48,49,78, - 79,84,88,90,80,81,82,83,85,86,87,89,91,99,110,111,121,113,115,116] - names_blue_gloves: [132,133,50,51,54,55,56,52,61,59,53,57,62,65,66,67,68,69, - 58,60,63,64,125,126,127,129,131,134,135,136,128,130,137, - 138,139] - +exp_name: ${data_gen.exp_name}_win_${data.window_size}_obj_sample_${data.sample_rate} paths: - # data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_False_reshuffle_True" #[_v2_aug_False] - data_dir: "/data/PTG/TCN_data/m3/p_m3_feat_v6_with_pose_v3_aug_False_reshuffle_True" #[_v2_aug_False] - # data_dir: "/data/PTG/TCN_data/m2/p_m2_feat_v6_no_pose_v3_aug_False_reshuffle_True" #[_v2_aug_False] - # data_dir: "/data/PTG/TCN_data/m2/p_m2_feat_v6_no_pose_v3_aug_False_reshuffle_True" #[_v2_aug_False] - # data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_False_reshuffle_False" #[_v2_aug_False] - # data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_True" #[_v2_aug_False] - # data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_False" #[_v2_aug_False] - - # root_dir: "/data/PTG/medical/training/activity_classifier/TCN_HPL" - root_dir: "/data/users/peri.akiva/PTG/medical/training/activity_classifier/TCN_HPL" - - # dataset_kwcoco: "/data/users/peri.akiva/datasets/ptg/m2_good_images_only_no_amputation_stump_train_activity_obj_results_with_dets_and_pose.mscoco.json" + # data_dir: "/data/PTG/TCN_data/m3/p_m3_feat_v6_with_pose_v3_aug_False_reshuffle_True" #[_v2_aug_False] + data_dir: "/data/PTG/medical/training/activity_classifier/TCN_data/${task}/${data_gen.exp_name}" + root_dir: "/data/PTG/medical/training/activity_classifier/TCN_HPL/" dataset_kwcoco: "/data/users/peri.akiva/datasets/ptg/m3_all_all_obj_results_with_dets_and_pose.mscoco.json" - activity_config_root: "/home/local/KHQ/peri.akiva/projects/angel_system/config/activity_labels" + dataset_kwcoco_lab: /data/PTG/medical/coco/m3_lab_all_all_obj_results_with_dets_and_pose.mscoco.json + activity_config_root: "/home/local/KHQ/peri.akiva/projects/angel_system/config/activity_labels/medical" activity_config_fn: "${paths.activity_config_root}/${task}" ptg_root: "/home/local/KHQ/peri.akiva/angel_system" activity_config_path: "${paths.ptg_root}/config/activity_labels/medical" output_data_dir_root: "/data/PTG/TCN_data" - # bbn_data_root: "{bbn_data_dir}/Release_v0.5/v0.56" bbn_data_dir: "/data/PTG/medical/bbn_data" diff --git a/configs/experiment/m5/feat_v6.yaml b/configs/experiment/m5/feat_v6.yaml index 96da42f8e..5c08d692b 100644 --- a/configs/experiment/m5/feat_v6.yaml +++ b/configs/experiment/m5/feat_v6.yaml @@ -2,7 +2,10 @@ # to execute this experiment run: # python train.py experiment=example +topic: "medical" task: "m5" +feature_version: 6 + defaults: - override /data: ptg - override /model: ptg @@ -14,12 +17,10 @@ defaults: # all parameters below will be merged with parameters from default configurations set above # this allows you to overwrite only specified parameters # exp_name: "p_m2_tqt_data_test_feat_v6_with_pose" #[_v2_aug_False] -exp_name: "p_m5_feat_v6_with_pose_v3_aug_False_reshuffle_True" #[_v2_aug_False] -# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_False_reshuffle_False" #[_v2_aug_False] -# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_True" #[_v2_aug_False] -# exp_name: "p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_False" #[_v2_aug_False] +# exp_name: "p_m5_feat_v6_with_pose_v3_aug_False_reshuffle_True" #[_v2_aug_False] +# exp_name: "p_m5_feat_v6_with_pose_v3_aug_False_reshuffle_True_lab" -tags: ["m5", "ms_tcn"] +tags: ["m5", "ms_tcn", "debug"] seed: 12345 @@ -32,15 +33,47 @@ model: compile: false net: - dim: 188 # length of feature vector + dim: 246 # length of feature vector + +IMAGE_HZ: 30 # zed bags +#IMAGE_HZ: 15 # BBN hololens live +OBJECT_DET_HZ: 15 +POSE_HZ: 4 + + +data_gen: + top_k_objects: 1 + pose_repeat_rate: 7.5 + data_type: "pro" + # feat_type: "with_pose" #[no_pose, with_pose, only_hands_joints, only_objects_joints] + dataset_kwcoco: "/data/users/peri.akiva/datasets/ptg/m5_all_all_obj_results_with_dets_and_pose.mscoco.json" + + filter_black_gloves: false + filter_blue_gloves: false + train_vid_ids: [1, 2, 4, 5, 6, 8, 9, 11, 12, 16, 19, 20, 21, + 22, 23, 24, 26, 28, 29, 30, 31, 33, 34, 35, 36, 37, + 42, 43, 44, 45, 46, 48, 49] + val_vid_ids: [3, 7, 10, 18, 27, 32, 41] + test_vid_ids: [50, 13, 47, 25] + + train_vid_ids_bbn: [3, 4, 6, 7, 9, 11, 12, 13, + 14, 16, 17, 18, 20, 21, 19, 22, 23] + val_vid_ids_bbn: [1, 5, 10] + test_vid_ids_bbn: [2, 8] + + activity_config_fn: "/home/local/KHQ/peri.akiva/projects/angel_system/config/activity_labels/medical/m5.yaml" + exp_ext: "_NEW_ORDER_fix_overlap_gt_step_GT_corrected" # anything unique about this run that isn't already in ``exp_name`` + exp_name: "${task}_${data_gen.data_type}_data_top_${data_gen.top_k_objects}_objs_feat_v${feature_version}_pose_rate_${data_gen.pose_repeat_rate}${data_gen.exp_ext}" + raw_data_root: /data/PTG/medical/bbn_data/Release_v0.5/v0.56 + dive_output_root: /data/PTG/medical/activity_anns data: num_classes: 6 # activities: includes background batch_size: 512 - num_workers: 0 + num_workers: 12 epoch_length: 20000 - window_size: 30 - sample_rate: 1 + window_size: 25 + sample_rate: 2 all_transforms: @@ -55,29 +88,15 @@ data: feat_version: 6 num_obj_classes: 6 # not including background - -data_gen: - reshuffle_datasets: true - augment: false - num_augs: 5 - feat_type: "with_pose" #[no_pose, with_pose, only_hands_joints, only_objects_joints] - filter_black_gloves: false - filter_blue_gloves: false - train_vid_ids: [1, 2, 4, 5, 6, 8, 9, 11, 12, 14, 15, 16, 17, 19, 20, 21, - 22, 23, 24, 26, 28, 29, 30, 31, 33, 34, 35, 36, 37, 38, 39, 40, - 42, 43, 44, 45, 46, 48, 49] - val_vid_ids: [3, 7, 10, 18, 27, 32, 41] - test_vid_ids: [50, 13, 47, 25] +exp_name: ${data_gen.exp_name}_win_${data.window_size}_obj_sample_${data.sample_rate} paths: - data_dir: "/data/PTG/TCN_data/m5/p_m5_feat_v6_with_pose_v3_aug_False_reshuffle_True" #[_v2_aug_False] - # data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_False_reshuffle_False" #[_v2_aug_False] - # data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_True" #[_v2_aug_False] - # data_dir: "/data/PTG/TCN_data/m2/p_m2_tqt_data_test_feat_v6_with_pose_v2_aug_True_reshuffle_False" #[_v2_aug_False] - # root_dir: "/data/PTG/medical/training/activity_classifier/TCN_HPL" - root_dir: "/data/users/peri.akiva/PTG/medical/training/activity_classifier/TCN_HPL" + data_dir: "/data/PTG/medical/training/activity_classifier/TCN_data/${task}/${data_gen.exp_name}" + root_dir: "/data/PTG/medical/training/activity_classifier/TCN_HPL/" + dataset_kwcoco: "/data/users/peri.akiva/datasets/ptg/m5_all_all_obj_results_with_dets_and_pose.mscoco.json" - activity_config_root: "/home/local/KHQ/peri.akiva/projects/angel_system/config/activity_labels" + dataset_kwcoco_lab: "/data/users/peri.akiva/datasets/ptg/m5_lab_all_all_obj_results_with_dets_and_pose.mscoco.json" + activity_config_root: "/home/local/KHQ/peri.akiva/projects/angel_system/config/activity_labels/medical" activity_config_fn: "${paths.activity_config_root}/${task}" ptg_root: "/home/local/KHQ/peri.akiva/angel_system" activity_config_path: "${paths.ptg_root}/config/activity_labels/medical" diff --git a/configs/model/ptg.yaml b/configs/model/ptg.yaml index 926d55513..ef7b0bace 100644 --- a/configs/model/ptg.yaml +++ b/configs/model/ptg.yaml @@ -3,7 +3,7 @@ _target_: tcn_hpl.models.ptg_module.PTGLitModule optimizer: _target_: torch.optim.Adam _partial_: true - lr: 0.000001 + lr: 0.00001 weight_decay: 0.0001 scheduler: diff --git a/tcn_hpl/data/components/PTG_dataset.py b/tcn_hpl/data/components/PTG_dataset.py index f59a92f7e..faf9eb2bc 100644 --- a/tcn_hpl/data/components/PTG_dataset.py +++ b/tcn_hpl/data/components/PTG_dataset.py @@ -115,25 +115,10 @@ def __getitem__(self, idx): :return: features, targets, and mask of the window """ - # print(f"size of dataset: {self.__len__()}") - # print(f"self.feature_frames: {self.feature_frames.shape}") - # print(f"self.target_frames: {self.target_frames.shape}") - # print(f"self.mask_frames: {self.mask_frames.shape}") - # print(f"self.source_vids: {self.source_vids.shape}") - # print(f"self.source_frames: {self.source_frames.shape}") - # print(f"self.mask_frames: {self.mask_frames}") - features = self.feature_frames[idx : idx + self.window_size, :] target = self.target_frames[idx : idx + self.window_size] mask = self.mask_frames[idx : idx + self.window_size] source_vid = self.source_vids[idx : idx + self.window_size] source_frame = self.source_frames[idx : idx + self.window_size] - # print(f"mask: {mask}") - # print(f"features: {features.shape}") - # print(f"target: {target.shape}") - # print(f"mask: {mask.shape}") - # print(f"source_vid: {source_vid.shape}") - # print(f"source_frame: {source_frame.shape}") - return features, target, mask, np.array(source_vid), source_frame diff --git a/tcn_hpl/data/utils/pose_generation/configs/main.yaml b/tcn_hpl/data/utils/pose_generation/configs/main.yaml index 612169f77..59b8ddd53 100644 --- a/tcn_hpl/data/utils/pose_generation/configs/main.yaml +++ b/tcn_hpl/data/utils/pose_generation/configs/main.yaml @@ -1,5 +1,5 @@ -task: r18 -data_type: bbn #[lab, pro] +task: m2 +data_type: lab #[lab, pro] root: /data/PTG/medical/bbn_data/Release_v0.5/v0.52/M2_Tourniquet/Data img_save_path: /data/users/peri.akiva/datasets/m2_tourniquet/imgs pose_model_config: /home/local/KHQ/peri.akiva/projects/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/ViTPose_base_medic_casualty_256x192.py @@ -20,7 +20,10 @@ data: m3: /data/PTG/medical/training/yolo_object_detector/detect/m3_all/m3_all_all_obj_results.mscoco.json m5: /data/PTG/medical/training/yolo_object_detector/detect//m5_all/m5_all_all_obj_results.mscoco.json r18: /data/PTG/medical/training/yolo_object_detector/detect//r18_all/r18_all_all_obj_results.mscoco.json - save_root: /data/users/peri.akiva/datasets/ptg + save_root: /data/PTG/medical/coco bbn_lab: r18: /data/PTG/medical/training/yolo_object_detector/detect//r18_all_bbn_lab_data/r18_all_bbn_lab_data_all_obj_results.mscoco.json - save_root: /data/users/peri.akiva/datasets/ptg \ No newline at end of file + m5: /data/PTG/medical/training/yolo_object_detector/detect/m5_lab_all/m5_lab_all_all_obj_results.mscoco.json + m3: /data/PTG/medical/training/yolo_object_detector/detect//m3_lab_all/m3_lab_all_all_obj_results.mscoco.json + m2: /data/PTG/medical/training/yolo_object_detector/detect//m2_lab_all/m2_lab_all_all_obj_results.mscoco.json + save_root: /data/PTG/medical/coco \ No newline at end of file diff --git a/tcn_hpl/data/utils/pose_generation/generate_pose_data.py b/tcn_hpl/data/utils/pose_generation/generate_pose_data.py index a73430c6c..bbb658aa4 100644 --- a/tcn_hpl/data/utils/pose_generation/generate_pose_data.py +++ b/tcn_hpl/data/utils/pose_generation/generate_pose_data.py @@ -232,14 +232,6 @@ def generate_bbs_and_pose( img = read_image(path, format="BGR") - # bs, ls, kps = self.predict_single(img) - - # print(f"boxes: {bs}") - # print(f"ls: {ls}") - # print(f"kps: {kps}") - - # continue - predictions, visualized_output = self.predictor.run_on_image(img) instances = predictions["instances"].to("cpu") @@ -304,12 +296,12 @@ def generate_bbs_and_pose( } pose_keypoints_list.append(kp_dict) - # print(f"pose_keypoints_list: {pose_keypoints_list}") current_ann["keypoints"] = pose_keypoints_list # current_ann['image_features'] = image_features self.dataset.add_annotation(**current_ann) + # if we want to visualize output # import matplotlib.pyplot as plt # image_show = dset.draw_image(gid=img_id) # plt.imshow(image_show) diff --git a/tcn_hpl/data/utils/pose_generation/rt_pose_generation.py b/tcn_hpl/data/utils/pose_generation/rt_pose_generation.py index 1330afdf0..913819a0d 100644 --- a/tcn_hpl/data/utils/pose_generation/rt_pose_generation.py +++ b/tcn_hpl/data/utils/pose_generation/rt_pose_generation.py @@ -99,19 +99,4 @@ def predict_single(det_model, pose_model, image: torch.tensor) -> list: pose_keypoints = pose_results[0]["keypoints"].tolist() keypoints_list.append(pose_keypoints) - # pose_keypoints_list = [] - # for kp_index, keypoint in enumerate(pose_keypoints): - # kp_dict = {'xy': [keypoint[0], keypoint[1]], - # 'keypoint_category_id': kp_index, - # 'keypoint_category': keypoints_cats[kp_index]} - # pose_keypoints_list.append(kp_dict) - - # # keypoints_list.append(pose_keypoints_list) - # # print(f"pose_keypoints_list: {pose_keypoints_list}") - # current_ann['keypoints'] = pose_keypoints_list - # current_ann['image_features'] = image_features - - # dset.add_annotation(**current_ann) - - # results = [] return boxes_list, labels_list, keypoints_list diff --git a/tcn_hpl/data/utils/pose_generation/video_to_frames.py b/tcn_hpl/data/utils/pose_generation/video_to_frames.py index 735683e0e..c07d928d2 100644 --- a/tcn_hpl/data/utils/pose_generation/video_to_frames.py +++ b/tcn_hpl/data/utils/pose_generation/video_to_frames.py @@ -22,9 +22,7 @@ def main(): videos = utils.dictionary_contents( videos_src_path, types=["*.mp4", "*.MP4"], recursive=True ) - # print(videos) - # exit() - # videos = filter(lambda x: x.endswith('MP4'), videos) + coco_json = { "info": { "description": "Medical Pose Estimation", @@ -82,7 +80,6 @@ def main(): frame_count = frame_count + 1 print("Final frame:", num) - # coco_json_save_path = f"{videos_src_path}/medical_coco.json" coco_json_save_path = "medical_coco.json" with open(coco_json_save_path, "w") as outfile: diff --git a/tcn_hpl/data/utils/ptg_datagenerator.py b/tcn_hpl/data/utils/ptg_datagenerator.py index 00bbbc391..d5b688106 100644 --- a/tcn_hpl/data/utils/ptg_datagenerator.py +++ b/tcn_hpl/data/utils/ptg_datagenerator.py @@ -22,7 +22,14 @@ from angel_system.data.medical.data_paths import TASK_TO_NAME from angel_system.data.medical.data_paths import LAB_TASK_TO_NAME +from angel_system.data.medical.load_bbn_data import bbn_activity_txt_to_csv +from angel_system.data.common.kwcoco_utils import add_activity_gt_to_kwcoco +def bbn_to_dive(raw_data_root, dive_output_dir, task, label_mapping, label_version=1): + used_videos = bbn_activity_txt_to_csv(task=task, root_dir=raw_data_root, + output_dir=dive_output_dir, label_mapping=label_mapping, + label_version=label_version) + return used_videos def create_training_data(config_path): ##################### @@ -49,22 +56,18 @@ def create_training_data(config_path): top_k_objects = config["data_gen"]["top_k_objects"] pose_repeat_rate = config["data_gen"]["pose_repeat_rate"] exp_ext = config["data_gen"]["exp_ext"] + raw_data_root = f"{config['data_gen']['raw_data_root']}/{TASK_TO_NAME[task_name]}/Data" + dive_output_root = config["data_gen"]["dive_output_root"] - dset = kwcoco.CocoDataset(config["data_gen"]["dataset_kwcoco"]) - # Check if the dest has activity gt, if it doesn't then add it - if not "activity_gt" in list(dset.imgs.values())[0].keys(): - print("adding activity ground truth to the dataset") - from angel_system.data.common.kwcoco_utils import add_activity_gt_to_kwcoco - - dset = add_activity_gt_to_kwcoco(topic, task_name, dset, activity_config_fn) - def filter_dset_by_split(split): + def filter_dset_by_split(split, dataset_to_split): # Filter by video names - video_lookup = dset.index.name_to_video + video_lookup = dataset_to_split.index.name_to_video split_vidids = [] split_img_ids = [] for index in config["data_gen"][split]: video_name = f"{task_name.upper()}-{index}" + print(f"video name: {video_name}") # Make sure we have the video if video_name in video_lookup: @@ -80,9 +83,9 @@ def filter_dset_by_split(split): continue split_vidids.append(video_name) - split_img_ids.extend(list(dset.index.vidid_to_gids[vid])) + split_img_ids.extend(list(dataset_to_split.index.vidid_to_gids[vid])) - split_dset = dset.subset(gids=split_img_ids, copy=True) + split_dset = dataset_to_split.subset(gids=split_img_ids, copy=True) return split_dset ##################### @@ -122,14 +125,42 @@ def filter_dset_by_split(split): activity_config = yaml.safe_load(stream) activity_labels = activity_config["labels"] + activity_labels_desc_mapping = {} with open(f"{output_data_dir}/mapping.txt", "w") as mapping: for label in activity_labels: i = label["id"] label_str = label["label"] + if "description" in label.keys(): + # activity_labels_desc_mapping[label["description"]] = label["label"] + activity_labels_desc_mapping[label["description"]] = label["id"] + elif "full_str" in label.keys(): + # activity_labels_desc_mapping[label["full_str"]] = label["label"] + activity_labels_desc_mapping[label["full_str"]] = label["id"] if label_str == "done": continue mapping.write(f"{i} {label_str}\n") - + + print(f"label mapping: {activity_labels_desc_mapping}") + # exit() + dset = kwcoco.CocoDataset(config["data_gen"]["dataset_kwcoco"]) + # Check if the dest has activity gt, if it doesn't then add it + if not "activity_gt" in list(dset.imgs.values())[0].keys(): + print("adding activity ground truth to the dataset") + + #generate dive files for videos in dataset if it does not exist + video_id = list(dset.index.videos.keys())[0] + video = dset.index.videos[video_id] + video_name = video["name"] + activity_gt_dir = f"{dive_output_root}/{task_name}_labels/" + activity_gt_fn = f"{activity_gt_dir}/{video_name}_activity_labels_v1.csv" + print(f"activity_gt_dir: {activity_gt_dir}, activity_gt_fn: {activity_gt_fn}") + + used_videos = bbn_to_dive(raw_data_root, activity_gt_dir, task_name, activity_labels_desc_mapping) + video_ids_to_remove = [vid for vid, value in dset.index.videos.items() if value['name'] not in used_videos] + dset.remove_videos(video_ids_to_remove) + + dset = add_activity_gt_to_kwcoco(topic, task_name, dset, activity_config_fn) + ##################### # Features, # groundtruth and @@ -137,7 +168,7 @@ def filter_dset_by_split(split): ##################### for split in ["train", "val", "test"]: print(f"==== {split} ====") - split_dset = filter_dset_by_split(f"{split}_vid_ids") + split_dset = filter_dset_by_split(f"{split}_vid_ids", dset) print(f"{split} num_images: {len(split_dset.imgs)}") for video_id in ub.ProgIter( diff --git a/tcn_hpl/models/components/ms_tcs_net.py b/tcn_hpl/models/components/ms_tcs_net.py index 0e4f0000a..11b7f9ad6 100644 --- a/tcn_hpl/models/components/ms_tcs_net.py +++ b/tcn_hpl/models/components/ms_tcs_net.py @@ -35,6 +35,7 @@ def __init__( ] ) + # normalizing network self.fc = nn.Sequential( nn.Linear(dim * window_size, 4096), nn.GELU(), @@ -51,30 +52,14 @@ def __init__( nn.Dropout(0.25), nn.Linear(4096, dim * window_size), ) - # self.fc1 = nn.Linear(dim*30, 4096) - # self.act = nn.GELU() - # self.drop1 = nn.Dropout(0.1) - # self.fc2 = nn.Linear(4096, 8192) - # self.drop2 = nn.Dropout(0.1) - - # self.fc3 = nn.Linear(8192, 16384) - # self.act3 = nn.GELU() - # self.drop3 = nn.Dropout(0.1) - # self.fc4 = nn.Linear(16384, dim*30) - - # self.fc = nn.Linear(1280, 2048) def forward(self, x, mask): b, d, c = x.shape # [batch_size, feat_dim, window_size] # mask shape: [batch_size, window_size] - # print(f"x: {x.shape}") - # print(f"mask: {mask.shape}") re_x = einops.rearrange(x, "b d c -> b (d c)") re_x = self.fc(re_x) x = einops.rearrange(re_x, "b (d c) -> b d c", d=d, c=c) - # print(f"re_x: {re_x.shape}") - # print(f"x: {x.shape}") out = self.stage1(x, mask) outputs = out.unsqueeze(0) @@ -82,7 +67,6 @@ def forward(self, x, mask): out = s(F.softmax(out, dim=1) * mask[:, None, :], mask) outputs = torch.cat((outputs, out.unsqueeze(0)), dim=0) - # print(f"outputs: {outputs.shape}") return outputs diff --git a/tcn_hpl/models/ptg_module.py b/tcn_hpl/models/ptg_module.py index 191ee6518..40d3802be 100644 --- a/tcn_hpl/models/ptg_module.py +++ b/tcn_hpl/models/ptg_module.py @@ -108,9 +108,7 @@ def __init__( self.class_ids = list(actions_dict.values()) self.classes = list(actions_dict.keys()) - # print(f"CLASSES IN MODEL: {self.classes}") - # print(f"actions_dict: {actions_dict}") - # exit() + self.action_id_to_str = dict(zip(self.class_ids, self.classes)) # loss functions @@ -171,11 +169,6 @@ def plot_gt_vs_preds(self, per_video_frame_gt_preds, split="train", max_items=30 gt.append(int(video_gt_preds[ind][0])) preds.append(int(video_gt_preds[ind][1])) - # plt.plot(gt, label="gt") - # plt.plot(preds, label="preds") - - # sns.barplot(x=inds, y=gt, linestyle='dotted', color='magenta', label='GT', ax=ax) - # ax.stackplot(inds, gt, alpha=0.5, labels=['GT'], color=['magenta']) sns.lineplot( x=inds, y=preds, @@ -191,12 +184,7 @@ def plot_gt_vs_preds(self, per_video_frame_gt_preds, split="train", max_items=30 ) sns.lineplot(x=inds, y=gt, color="magenta", label="GT", ax=ax, linewidth=3) - # ax.xaxis.label.set_visible(False) - # ax.spines['bottom'].set_visible(False) ax.legend() - # /confusion_mat_val.png" - # title = f"plot_pred_vs_gt_vid{video}.png" Path(folder).mkdir(parents=True, exist_ok=True) - # fig.savefig(f"{self.hparams.output_dir}/{title}") root_dir = f"{self.hparams.output_dir}/steps_vs_preds" if not os.path.exists(root_dir): @@ -212,7 +200,6 @@ def forward(self, x: torch.Tensor, m: torch.Tensor) -> torch.Tensor: :param m: A tensor of mask of the valid frames. :return: A tensor of logits. """ - return self.net(x, m) def on_train_start(self) -> None: @@ -222,9 +209,6 @@ def on_train_start(self) -> None: self.val_loss.reset() self.val_acc.reset() self.val_acc_best.reset() - # self.train_loss.reset() - # self.train_acc.reset() - # self.train_acc_best.reset() def compute_loss(self, p, y, mask): """Compute the total loss for a batch @@ -239,19 +223,8 @@ def compute_loss(self, p, y, mask): probs = torch.softmax(p, dim=1) # shape (batch size, self.hparams.num_classes) preds = torch.argmax(probs, dim=1).float() # shape: batch size - # print(f"prediction: {p}, GT: {y}"), # [bs, num_classes, window_size] - # print(f"prediction: {p.shape}, GT: {y.shape}") - loss = torch.zeros((1)).to(p[0]) - # print(f"loss: {loss.shape}") - # print(f"p loss: {p[:,:,-1].shape}") - # print(f"y: {y.view(-1).shape}") - - # p = einops.rearrange(p, 'b c w -> (b w) c') - # print(f"prediction: {p.shape}, GT: {y.shape}") - # print(f"prediction: {p}, GT: {y}"), # [bs, num_classes, window_size] - # TODO: Use only last frame per window loss += self.criterion( @@ -265,25 +238,13 @@ def compute_loss(self, p, y, mask): # ) # need to penalize high volatility of predictions within a window - - # std, mean = torch.std_mean(preds, dim=-1) mode, _ = torch.mode(y, dim=-1) mode = einops.repeat(mode, "b -> b c", c=preds.shape[-1]) - # print(f"mode: {mode.shape}") - # print(f"preds: {preds.shape}") - # print(f"mode: {mode[0,:]}") - # eps = 1e10 - # variation_coef = std/(mean+eps) - # variation_coef = torch.zeros_like(mode) + variation_coef = torch.abs(preds - mode) variation_coef = torch.sum(variation_coef, dim=-1) gt_variation_coef = torch.zeros_like(variation_coef) - # print(f"variation_coef: {variation_coef[0]}") - # print(f"variation_coef mean: {variation_coef.mean()}") - # print(f"p: {p.shape}") - # print(f"p[:, :, 1:]: {p[0, 0, 1:]}") - # print(f"F.log_softmax(p[:, :, 1:], dim=1): {F.log_softmax(p[:, :, 1:], dim=1)}") - + if self.hparams.use_smoothing_loss: loss += self.hparams.smoothing_loss * torch.mean( self.mse( @@ -353,9 +314,6 @@ def training_step( """ loss, probs, preds, targets, source_vid, source_frame = self.model_step(batch) - # print(f"targets: {targets}") - # print(f"preds: {preds}") - # update and log metrics self.train_loss(loss) self.train_acc(preds, targets[:, -1]) @@ -367,9 +325,9 @@ def training_step( "train/acc", self.train_acc, on_step=False, on_epoch=True, prog_bar=True ) - self.log( - "train/lr", self.lr_schedulers().get_last_lr()[0], on_step=False, on_epoch=True, prog_bar=True - ) + # self.log( + # "train/lr", self.lr_schedulers().state_dict()[[0], on_step=False, on_epoch=True, prog_bar=True + # ) self.training_step_outputs_target.append(targets[:, -1]) self.training_step_outputs_source_vid.append(source_vid[:, -1]) @@ -385,13 +343,7 @@ def on_train_epoch_end(self) -> None: # acc = self.train_acc.compute() # get current val acc # self.train_acc_best(acc) # update best so far val acc - - # all_targets = torch.concat(self.training_step_outputs_target) # shape: #frames - # all_preds = torch.concat(self.training_step_outputs_pred) # shape: #frames - # all_probs = torch.concat(self.training_step_outputs_prob) # shape (#frames, #act labels) - # all_source_vids = torch.concat(self.training_step_outputs_source_vid) - # all_source_frames = torch.concat(self.training_step_outputs_source_frame) - + all_targets = torch.cat(self.training_step_outputs_target) # shape: #frames all_preds = torch.cat(self.training_step_outputs_pred) # shape: #frames all_probs = torch.cat( @@ -514,10 +466,6 @@ def validation_step( windowed_preds = torch.tensor(windowed_preds).to(targets) windowed_ys = torch.tensor(windowed_ys).to(targets) - # print(f"preds: {preds.shape}, targets: {targets.shape}") - # print(f"windowed_preds: {windowed_preds.shape}, windowed_ys: {windowed_ys.shape}") - - # self.val_acc(windowed_preds, windowed_ys) self.val_acc(preds, targets[:, -1]) self.log("val/loss", self.val_loss, on_step=False, on_epoch=True, prog_bar=True) self.log("val/acc", self.val_acc, on_step=False, on_epoch=True, prog_bar=True) @@ -528,26 +476,6 @@ def validation_step( self.validation_step_outputs_pred.append(preds[inds]) self.validation_step_outputs_prob.append(probs[inds]) - # def plot_gt_vs_activations(self, step_gts, fname_suffix=None): - # # Plot gt vs predicted class across all vid frames - # fig = plt.figure() - # sns.set(font_scale=1) - # step_gts = [float(i) for i in step_gts] - # plt.plot(step_gts, label="gt") - # starting_zero_value = 0 - # for i in range(len(self.avg_probs)): #TODO - swap len(self.avg_probs) with the number of activities you're tracking - # starting_zero_value -= 2 - # plot_line = np.asarray(self.activity_conf_history)[:, i]. #TODO: this 1D array is the activity confidences for one activity class i. - # plt.plot(2 * plot_line + starting_zero_value, label=f"act_preds[{i}]") - - # plt.legend() - # if not fname_suffix: - # fname_suffix = f"vid{vid_id}" - # recipe_type = foo # TODO: fill with the task name - # title = f"plot_pred_vs_gt_{recipe_type}_{fname_suffix}.png" - # plt.title(title) - # fig.savefig(f"./outputs/{title}") # TODO: output this wherever you want - # plt.close() def on_validation_epoch_end(self) -> None: "Lightning hook that is called when a validation epoch ends." @@ -560,14 +488,6 @@ def on_validation_epoch_end(self) -> None: best_val_acc = self.val_acc_best.compute() self.log("val/acc_best", best_val_acc, sync_dist=True, prog_bar=True) - # import collections - # counter = collections.Counter(self.validation_step_outputs_target) - - # all_targets = torch.concat(self.validation_step_outputs_target) # shape: #frames - # all_preds = torch.concat(self.validation_step_outputs_pred) # shape: #frames - # all_probs = torch.concat(self.validation_step_outputs_prob) # shape (#frames, #act labels) - # all_source_vids = torch.concat(self.validation_step_outputs_source_vid) - # all_source_frames = torch.concat(self.validation_step_outputs_source_frame) all_targets = torch.cat(self.validation_step_outputs_target) # shape: #frames all_preds = torch.cat(self.validation_step_outputs_pred) # shape: #frames @@ -577,8 +497,6 @@ def on_validation_epoch_end(self) -> None: all_source_vids = torch.cat(self.validation_step_outputs_source_vid) all_source_frames = torch.cat(self.validation_step_outputs_source_frame) - # print(f"Per class occurences in GT: {torch.unique(all_targets, return_counts=True)}") - # print(f"all_targets: {all_targets.shape}") # Load val vidoes if self.val_frames is None: diff --git a/tcn_hpl/train.py b/tcn_hpl/train.py index 402581271..252356e4e 100644 --- a/tcn_hpl/train.py +++ b/tcn_hpl/train.py @@ -55,8 +55,6 @@ def train(cfg: DictConfig) -> Tuple[Dict[str, Any], Dict[str, Any]]: log.info(f"Instantiating datamodule <{cfg.data._target_}>") datamodule: LightningDataModule = hydra.utils.instantiate(cfg.data) - # print(f"datamodule: {datamodule.__dict__}") - # exit() log.info(f"Instantiating model <{cfg.model._target_}>") model: LightningModule = hydra.utils.instantiate(cfg.model)