diff --git a/setup.py b/setup.py index fe089fc4c..c06818381 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ author_email="", url="https://github.com/user/project", # install_requires=["lightning", "hydra-core", "hydra-colorlog"], - packages=find_packages(include=['tcn_hpl', 'tcn_hpl.*']), + packages=find_packages(include=["tcn_hpl", "tcn_hpl.*"]), # use this to customize global commands available in the terminal after installing the package entry_points={ "console_scripts": [ diff --git a/tcn_hpl/data/components/augmentations.py b/tcn_hpl/data/components/augmentations.py index 43df4898b..e351238e0 100644 --- a/tcn_hpl/data/components/augmentations.py +++ b/tcn_hpl/data/components/augmentations.py @@ -727,15 +727,16 @@ def forward(self, features): def __repr__(self) -> str: detail = f"(im_w={self.im_w}, im_h={self.im_h}, feat_version={self.feat_version}, top_k_objects={self.top_k_objects})" return f"{self.__class__.__name__}{detail}" - class DropoutObjects(torch.nn.Module): - """Drop out Objects """ + """Drop out Objects""" - def __init__(self, dropout_probablity, num_obj_classes, feat_version, top_k_objects): + def __init__( + self, dropout_probablity, num_obj_classes, feat_version, top_k_objects + ): """ - :param dropout_probablity: probablity that a given frame will NOT have an object + :param dropout_probablity: probablity that a given frame will NOT have an object """ super().__init__() @@ -769,8 +770,8 @@ def __init__(self, dropout_probablity, num_obj_classes, feat_version, top_k_obje self.obj_feature_mask.append(0) if self.use_hand_dist: - self.obj_feature_mask += [0]*2*self.num_good_obj_classes - ind += 2*self.num_good_obj_classes + self.obj_feature_mask += [0] * 2 * self.num_good_obj_classes + ind += 2 * self.num_good_obj_classes if self.use_center_dist: # right hand - image center distance @@ -785,8 +786,8 @@ def __init__(self, dropout_probablity, num_obj_classes, feat_version, top_k_obje if self.use_hand_dist: # Left hand distances - self.obj_feature_mask += [0]*2*self.num_good_obj_classes - ind += 2*self.num_good_obj_classes + self.obj_feature_mask += [0] * 2 * self.num_good_obj_classes + ind += 2 * self.num_good_obj_classes if self.use_center_dist: # left hand - image center distance @@ -825,34 +826,33 @@ def __init__(self, dropout_probablity, num_obj_classes, feat_version, top_k_obje if self.use_joint_hand_offset: # left hand - joints distances ind += 44 - self.obj_feature_mask += [1]*44 + self.obj_feature_mask += [1] * 44 # right hand - joints distances ind += 44 - self.obj_feature_mask += [1]*44 + self.obj_feature_mask += [1] * 44 # OBJS-JOINTS if self.use_joint_object_offset: - self.obj_feature_mask += [0]*44*self.top_k_objects*self.num_good_obj_classes - ind += 44*self.top_k_objects*self.num_good_obj_classes - + self.obj_feature_mask += ( + [0] * 44 * self.top_k_objects * self.num_good_obj_classes + ) + ind += 44 * self.top_k_objects * self.num_good_obj_classes + self.obj_feature_mask = torch.tensor(self.obj_feature_mask) def forward(self, features): num_frames = features.shape[0] # Pick starting location of random mask - start = random.randint(0,self.skip_stride) + start = random.randint(0, self.skip_stride) # Create mask (one element for each frame) mask = torch.rand(num_frames) > self.dropout_probablity if self.dropout_last: mask[-1] = 0 - - return features def __repr__(self) -> str: detail = f"(im_w={self.im_w}, im_h={self.im_h}, num_obj_classes={self.num_obj_classes}, feat_version={self.feat_version}, top_k_objects={self.top_k_objects})" return f"{self.__class__.__name__}{detail}" - diff --git a/tcn_hpl/data/utils/ptg_datagenerator.py b/tcn_hpl/data/utils/ptg_datagenerator.py index d5b688106..fdc32c4bb 100644 --- a/tcn_hpl/data/utils/ptg_datagenerator.py +++ b/tcn_hpl/data/utils/ptg_datagenerator.py @@ -25,12 +25,18 @@ from angel_system.data.medical.load_bbn_data import bbn_activity_txt_to_csv from angel_system.data.common.kwcoco_utils import add_activity_gt_to_kwcoco + def bbn_to_dive(raw_data_root, dive_output_dir, task, label_mapping, label_version=1): - used_videos = bbn_activity_txt_to_csv(task=task, root_dir=raw_data_root, - output_dir=dive_output_dir, label_mapping=label_mapping, - label_version=label_version) + used_videos = bbn_activity_txt_to_csv( + task=task, + root_dir=raw_data_root, + output_dir=dive_output_dir, + label_mapping=label_mapping, + label_version=label_version, + ) return used_videos + def create_training_data(config_path): ##################### # Input @@ -56,10 +62,11 @@ def create_training_data(config_path): top_k_objects = config["data_gen"]["top_k_objects"] pose_repeat_rate = config["data_gen"]["pose_repeat_rate"] exp_ext = config["data_gen"]["exp_ext"] - raw_data_root = f"{config['data_gen']['raw_data_root']}/{TASK_TO_NAME[task_name]}/Data" + raw_data_root = ( + f"{config['data_gen']['raw_data_root']}/{TASK_TO_NAME[task_name]}/Data" + ) dive_output_root = config["data_gen"]["dive_output_root"] - def filter_dset_by_split(split, dataset_to_split): # Filter by video names video_lookup = dataset_to_split.index.name_to_video @@ -139,15 +146,15 @@ def filter_dset_by_split(split, dataset_to_split): if label_str == "done": continue mapping.write(f"{i} {label_str}\n") - + print(f"label mapping: {activity_labels_desc_mapping}") # exit() dset = kwcoco.CocoDataset(config["data_gen"]["dataset_kwcoco"]) # Check if the dest has activity gt, if it doesn't then add it if not "activity_gt" in list(dset.imgs.values())[0].keys(): print("adding activity ground truth to the dataset") - - #generate dive files for videos in dataset if it does not exist + + # generate dive files for videos in dataset if it does not exist video_id = list(dset.index.videos.keys())[0] video = dset.index.videos[video_id] video_name = video["name"] @@ -155,12 +162,18 @@ def filter_dset_by_split(split, dataset_to_split): activity_gt_fn = f"{activity_gt_dir}/{video_name}_activity_labels_v1.csv" print(f"activity_gt_dir: {activity_gt_dir}, activity_gt_fn: {activity_gt_fn}") - used_videos = bbn_to_dive(raw_data_root, activity_gt_dir, task_name, activity_labels_desc_mapping) - video_ids_to_remove = [vid for vid, value in dset.index.videos.items() if value['name'] not in used_videos] + used_videos = bbn_to_dive( + raw_data_root, activity_gt_dir, task_name, activity_labels_desc_mapping + ) + video_ids_to_remove = [ + vid + for vid, value in dset.index.videos.items() + if value["name"] not in used_videos + ] dset.remove_videos(video_ids_to_remove) - + dset = add_activity_gt_to_kwcoco(topic, task_name, dset, activity_config_fn) - + ##################### # Features, # groundtruth and @@ -223,7 +236,7 @@ def filter_dset_by_split(split, dataset_to_split): ann_by_image, feat_version=feat_version, top_k_objects=top_k_objects, - pose_repeat_rate=pose_repeat_rate + pose_repeat_rate=pose_repeat_rate, ) print(X.shape) diff --git a/tcn_hpl/models/ptg_module.py b/tcn_hpl/models/ptg_module.py index 40d3802be..fd5ab8d8b 100644 --- a/tcn_hpl/models/ptg_module.py +++ b/tcn_hpl/models/ptg_module.py @@ -244,7 +244,7 @@ def compute_loss(self, p, y, mask): variation_coef = torch.abs(preds - mode) variation_coef = torch.sum(variation_coef, dim=-1) gt_variation_coef = torch.zeros_like(variation_coef) - + if self.hparams.use_smoothing_loss: loss += self.hparams.smoothing_loss * torch.mean( self.mse( @@ -343,7 +343,7 @@ def on_train_epoch_end(self) -> None: # acc = self.train_acc.compute() # get current val acc # self.train_acc_best(acc) # update best so far val acc - + all_targets = torch.cat(self.training_step_outputs_target) # shape: #frames all_preds = torch.cat(self.training_step_outputs_pred) # shape: #frames all_probs = torch.cat( @@ -369,7 +369,7 @@ def on_train_epoch_end(self) -> None: self.train_frames[video[:-4]] = train_fns per_video_frame_gt_preds = {} - + for (gt, pred, source_vid, source_frame) in zip( all_targets, all_preds, all_source_vids, all_source_frames ): @@ -476,7 +476,6 @@ def validation_step( self.validation_step_outputs_pred.append(preds[inds]) self.validation_step_outputs_prob.append(probs[inds]) - def on_validation_epoch_end(self) -> None: "Lightning hook that is called when a validation epoch ends." acc = self.val_acc.compute() # get current val acc @@ -488,7 +487,6 @@ def on_validation_epoch_end(self) -> None: best_val_acc = self.val_acc_best.compute() self.log("val/acc_best", best_val_acc, sync_dist=True, prog_bar=True) - all_targets = torch.cat(self.validation_step_outputs_target) # shape: #frames all_preds = torch.cat(self.validation_step_outputs_pred) # shape: #frames all_probs = torch.cat( @@ -497,7 +495,6 @@ def on_validation_epoch_end(self) -> None: all_source_vids = torch.cat(self.validation_step_outputs_source_vid) all_source_frames = torch.cat(self.validation_step_outputs_source_frame) - # Load val vidoes if self.val_frames is None: self.val_frames = {} diff --git a/tcn_hpl/train.py b/tcn_hpl/train.py index 252356e4e..fff4f05d2 100644 --- a/tcn_hpl/train.py +++ b/tcn_hpl/train.py @@ -55,7 +55,6 @@ def train(cfg: DictConfig) -> Tuple[Dict[str, Any], Dict[str, Any]]: log.info(f"Instantiating datamodule <{cfg.data._target_}>") datamodule: LightningDataModule = hydra.utils.instantiate(cfg.data) - log.info(f"Instantiating model <{cfg.model._target_}>") model: LightningModule = hydra.utils.instantiate(cfg.model) diff --git a/tcn_hpl/utils/logging_utils.py b/tcn_hpl/utils/logging_utils.py index a36a6b02b..03079e91d 100644 --- a/tcn_hpl/utils/logging_utils.py +++ b/tcn_hpl/utils/logging_utils.py @@ -53,7 +53,6 @@ def log_hyperparameters(object_dict: Dict[str, Any]) -> None: hparams["tags"] = cfg.get("tags") hparams["ckpt_path"] = cfg.get("ckpt_path") hparams["seed"] = cfg.get("seed") - hparams["cfg"] = cfg diff --git a/tests/conftest.py b/tests/conftest.py index b5dea333c..23d5fc418 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,7 +43,9 @@ def cfg_eval_global() -> DictConfig: :return: A DictConfig containing a default Hydra configuration for evaluation. """ with initialize(version_base="1.3", config_path="../configs"): - cfg = compose(config_name="eval.yaml", return_hydra_config=True, overrides=["ckpt_path=."]) + cfg = compose( + config_name="eval.yaml", return_hydra_config=True, overrides=["ckpt_path=."] + ) # set defaults for all tests with open_dict(cfg): diff --git a/tests/helpers/run_if.py b/tests/helpers/run_if.py index 9703af425..dd33b3c73 100644 --- a/tests/helpers/run_if.py +++ b/tests/helpers/run_if.py @@ -92,9 +92,7 @@ def __new__( reasons.append(f"torch<{max_torch}") if min_python: - py_version = ( - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ) + py_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" conditions.append(Version(py_version) < Version(min_python)) reasons.append(f"python>={min_python}") diff --git a/tests/test_augs.py b/tests/test_augs.py index 5c4da0088..5ff607c03 100644 --- a/tests/test_augs.py +++ b/tests/test_augs.py @@ -9,9 +9,7 @@ class MoveCenterPts(torch.nn.Module): adjusting the distances for each frame """ - def __init__( - self, num_obj_classes, feat_version - ): + def __init__(self, num_obj_classes, feat_version): """ :param hand_dist_delta: Decimal percentage to calculate the +-offset in pixels for the hands @@ -49,9 +47,11 @@ def forward(self, features): [1, left_dist_idx1], [right_dist_idx2, left_dist_idx2], ): - frame[start_idx:end_idx:2] += " + hand_delta_x + obj_delta_x" + frame[start_idx:end_idx:2] += " + hand_delta_x + obj_delta_x" - frame[start_idx + 1 : end_idx : 2] += " + hand_delta_y + obj_delta_y" + frame[ + start_idx + 1 : end_idx : 2 + ] += " + hand_delta_y + obj_delta_y" # Distance between hands hands_dist_idx = left_dist_idx2 @@ -59,17 +59,19 @@ def forward(self, features): frame[hands_dist_idx] += " + rhand_delta_x + lhand_delta_x" frame[hands_dist_idx + 1] += " + rhand_delta_y + lhand_delta_y" - + elif self.feat_version == 3: # Right and left hand distances - right_idx1 = 1; right_idx2 = 2; - left_idx1 = 4; left_idx2 = 5 + right_idx1 = 1 + right_idx2 = 2 + left_idx1 = 4 + left_idx2 = 5 for start_idx, end_idx in zip( [right_idx1, left_idx1], [right_idx2, left_idx2], ): frame[start_idx] = frame[start_idx] + " + hand_delta_x" - + frame[end_idx] = frame[end_idx] + " + hand_delta_y" # Object distances @@ -85,6 +87,7 @@ def forward(self, features): features[i] = frame return features + class ActivationDelta(torch.nn.Module): """Update the activation feature of each class by +-``conf_delta``""" @@ -167,9 +170,9 @@ def forward(self, features): [1, left_dist_idx1], [right_dist_idx2, left_dist_idx2] ): features[:, start_idx:end_idx:2] += " / self.im_w" - + features[:, start_idx + 1 : end_idx : 2] += " / self.im_h" - + # Distance between hands hands_dist_idx = left_dist_idx2 @@ -182,12 +185,12 @@ def forward(self, features): else: NotImplementedError(f"Unhandled version '{self.feat_version}'") - return features + class NormalizeFromCenter(torch.nn.Module): """Normalize the distances from -1 to 1 with respect to the image center - + Missing objects will be set to (2, 2) """ @@ -210,22 +213,25 @@ def forward(self, features): elif self.feat_version == 3: # Right and left hand distances - right_idx1 = 1; right_idx2 = 2; - left_idx1 = 4; left_idx2 = 5 + right_idx1 = 1 + right_idx2 = 2 + left_idx1 = 4 + left_idx2 = 5 for start_idx, end_idx in zip( [right_idx1, left_idx1], [right_idx2, left_idx2], ): features[:, start_idx] += " / self.half_w" - - features[:, end_idx] += " / self.half_h" + features[:, end_idx] += " / self.half_h" # Object distances start_idx = 10 while start_idx < features.shape[1]: features[:, start_idx] = features[:, start_idx] + " / self.half_w" - features[:, start_idx + 1] = features[:, start_idx + 1] + " / self.half_h" + features[:, start_idx + 1] = ( + features[:, start_idx + 1] + " / self.half_h" + ) start_idx += 5 else: @@ -233,48 +239,51 @@ def forward(self, features): return features - + def main(): num_obj_classes = 40 feat_version = 5 - mv_center = MoveCenterPts(num_obj_classes+2, feat_version) - act_delta = ActivationDelta(num_obj_classes+2, feat_version) - norm = NormalizePixelPts(num_obj_classes+2, feat_version) + mv_center = MoveCenterPts(num_obj_classes + 2, feat_version) + act_delta = ActivationDelta(num_obj_classes + 2, feat_version) + norm = NormalizePixelPts(num_obj_classes + 2, feat_version) if feat_version == 2: feat_v2 = ["A[right hand]"] for i in range(num_obj_classes): feat_v2.append(f"D[right hand, obj{i+1}]x") feat_v2.append(f"D[right hand, obj{i+1}]y") - + feat_v2.append("A[left hand]") for i in range(num_obj_classes): feat_v2.append(f"D[left hand, obj{i+1}]x") feat_v2.append(f"D[left hand, obj{i+1}]y") - + feat_v2.append("D[right hand, left hand]x") feat_v2.append("D[right hand, left hand]y") for i in range(num_obj_classes): feat_v2.append(f"A[obj{i+1}]") - feat_v2 = np.array([feat_v2], dtype='object') + feat_v2 = np.array([feat_v2], dtype="object") assert feat_v2.shape == (1, 204) - #feat_v2_mv_center = mv_center(feat_v2) - #feat_v2_act_delta = act_delta(feat_v2) + # feat_v2_mv_center = mv_center(feat_v2) + # feat_v2_act_delta = act_delta(feat_v2) feat_v2_norm = norm(feat_v2) - + for i, e in enumerate(feat_v2_norm[0]): print(f"{i}: {e}") elif feat_version == 3: - feat_v3 = ["A[right hand]", - "D[right hand, center]x", "D[right hand, center]y", + feat_v3 = [ + "A[right hand]", + "D[right hand, center]x", + "D[right hand, center]y", "A[left hand]", - "D[left hand, center]x", "D[left hand, center]y", - "I[right hand, left hand]" + "D[left hand, center]x", + "D[left hand, center]y", + "I[right hand, left hand]", ] for i in range(num_obj_classes): @@ -284,14 +293,13 @@ def main(): feat_v3.append(f"D[obj{i+1}, center]x") feat_v3.append(f"D[obj{i+1}, center]y") - feat_v3 = np.array([feat_v3], dtype='object') + feat_v3 = np.array([feat_v3], dtype="object") assert feat_v3.shape == (1, 207) - #feat_v3_mv_center = mv_center(feat_v3) - #feat_v3_act_delta = act_delta(feat_v3) + # feat_v3_mv_center = mv_center(feat_v3) + # feat_v3_act_delta = act_delta(feat_v3) feat_v3_norm_center = norm_center(feat_v3) - for i, e in enumerate(feat_v3_norm_center[0]): print(f"{i}: {e}") @@ -300,12 +308,12 @@ def main(): for i in range(num_obj_classes): feat_v5.append(f"D[right hand, obj{i+1}]x") feat_v5.append(f"D[right hand, obj{i+1}]y") - + feat_v5.append("A[left hand]") for i in range(num_obj_classes): feat_v5.append(f"D[left hand, obj{i+1}]x") feat_v5.append(f"D[left hand, obj{i+1}]y") - + feat_v5.append("D[right hand, left hand]x") feat_v5.append("D[right hand, left hand]y") @@ -316,11 +324,11 @@ def main(): feat_v5.append(f"I[right hand, obj{i+1}]") feat_v5.append(f"I[left hand, obj{i+1}]") - feat_v5 = np.array([feat_v5], dtype='object') + feat_v5 = np.array([feat_v5], dtype="object") assert feat_v5.shape == (1, 285) - #feat_v5_mv_center = mv_center(feat_v5) - #feat_v5_act_delta = act_delta(feat_v5) + # feat_v5_mv_center = mv_center(feat_v5) + # feat_v5_act_delta = act_delta(feat_v5) feat_v5_norm = norm(feat_v5) for i, e in enumerate(feat_v5_norm[0]): @@ -328,4 +336,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tests/test_eval.py b/tests/test_eval.py index 865be5c8d..99fb86de4 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -10,7 +10,9 @@ @pytest.mark.slow -def test_train_eval(tmp_path: Path, cfg_train: DictConfig, cfg_eval: DictConfig) -> None: +def test_train_eval( + tmp_path: Path, cfg_train: DictConfig, cfg_eval: DictConfig +) -> None: """Tests training and evaluation by training for 1 epoch with `train.py` then evaluating with `eval.py`. @@ -36,4 +38,7 @@ def test_train_eval(tmp_path: Path, cfg_train: DictConfig, cfg_eval: DictConfig) test_metric_dict, _ = evaluate(cfg_eval) assert test_metric_dict["test/acc"] > 0.0 - assert abs(train_metric_dict["test/acc"].item() - test_metric_dict["test/acc"].item()) < 0.001 + assert ( + abs(train_metric_dict["test/acc"].item() - test_metric_dict["test/acc"].item()) + < 0.001 + )