From 331c9aa03aa999e72231233fe1ab932deaf573a6 Mon Sep 17 00:00:00 2001 From: bbearce Date: Fri, 21 Jun 2024 13:38:29 -0600 Subject: [PATCH] bens_current --- .gitignore | 1 + .../run/medsam/miccai_experiments/infer.sh | 12 +++++++ .../run/medsam/miccai_experiments/train1-2.sh | 14 ++++++++ .../run/medsam/miccai_experiments/train3-4.sh | 15 +++++++++ .../run/medsam/miccai_experiments/train5-6.sh | 15 +++++++++ .../medsam/miccai_experiments/train7-10.sh | 28 ++++++++++++++++ .../medsam/miccai_experiments/train_frac10.sh | 4 +-- .../miccai_experiments/train_frac100.sh | 7 ++++ .../medsam/miccai_experiments/train_frac20.sh | 6 ++-- .../medsam/miccai_experiments/train_frac30.sh | 7 ++++ .../medsam/miccai_experiments/train_frac40.sh | 7 ++++ .../medsam/miccai_experiments/train_frac50.sh | 7 ++++ .../medsam/miccai_experiments/train_frac60.sh | 7 ++++ .../medsam/miccai_experiments/train_frac70.sh | 7 ++++ .../medsam/miccai_experiments/train_frac80.sh | 7 ++++ .../medsam/miccai_experiments/train_frac90.sh | 7 ++++ src/segmentation/generic/run/run_infer.py | 15 ++++++--- src/segmentation/generic/run/run_train.py | 4 +-- src/segmentation/generic/utils/utils_infer.py | 10 +++--- src/segmentation/generic/utils/utils_train.py | 28 +++++++++++----- .../medsam/data_modules/datasets_medsam.py | 20 +++++++++-- yamls/system/lt2.yaml | 2 +- .../miccai_experiments/experiment_10.yaml | 12 ++++--- .../miccai_experiments/experiment_100.yaml | 33 +++++++++++++++++++ .../miccai_experiments/experiment_20.yaml | 14 ++++---- .../miccai_experiments/experiment_30.yaml | 33 +++++++++++++++++++ .../miccai_experiments/experiment_40.yaml | 33 +++++++++++++++++++ .../miccai_experiments/experiment_50.yaml | 33 +++++++++++++++++++ .../miccai_experiments/experiment_60.yaml | 33 +++++++++++++++++++ .../miccai_experiments/experiment_70.yaml | 33 +++++++++++++++++++ .../miccai_experiments/experiment_80.yaml | 33 +++++++++++++++++++ .../miccai_experiments/experiment_90.yaml | 33 +++++++++++++++++++ 32 files changed, 481 insertions(+), 39 deletions(-) create mode 100755 scripts/run/medsam/miccai_experiments/infer.sh create mode 100644 scripts/run/medsam/miccai_experiments/train1-2.sh create mode 100644 scripts/run/medsam/miccai_experiments/train3-4.sh create mode 100644 scripts/run/medsam/miccai_experiments/train5-6.sh create mode 100644 scripts/run/medsam/miccai_experiments/train7-10.sh mode change 100644 => 100755 scripts/run/medsam/miccai_experiments/train_frac10.sh create mode 100755 scripts/run/medsam/miccai_experiments/train_frac100.sh mode change 100644 => 100755 scripts/run/medsam/miccai_experiments/train_frac20.sh create mode 100755 scripts/run/medsam/miccai_experiments/train_frac30.sh create mode 100755 scripts/run/medsam/miccai_experiments/train_frac40.sh create mode 100755 scripts/run/medsam/miccai_experiments/train_frac50.sh create mode 100755 scripts/run/medsam/miccai_experiments/train_frac60.sh create mode 100755 scripts/run/medsam/miccai_experiments/train_frac70.sh create mode 100755 scripts/run/medsam/miccai_experiments/train_frac80.sh create mode 100755 scripts/run/medsam/miccai_experiments/train_frac90.sh create mode 100644 yamls/training/sweeps/medsam/miccai_experiments/experiment_100.yaml create mode 100644 yamls/training/sweeps/medsam/miccai_experiments/experiment_30.yaml create mode 100644 yamls/training/sweeps/medsam/miccai_experiments/experiment_40.yaml create mode 100644 yamls/training/sweeps/medsam/miccai_experiments/experiment_50.yaml create mode 100644 yamls/training/sweeps/medsam/miccai_experiments/experiment_60.yaml create mode 100644 yamls/training/sweeps/medsam/miccai_experiments/experiment_70.yaml create mode 100644 yamls/training/sweeps/medsam/miccai_experiments/experiment_80.yaml create mode 100644 yamls/training/sweeps/medsam/miccai_experiments/experiment_90.yaml diff --git a/.gitignore b/.gitignore index db47db1..2ca66f9 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,4 @@ wandb/ # Other # yamls/ # scripts/run +lightning_logs/ \ No newline at end of file diff --git a/scripts/run/medsam/miccai_experiments/infer.sh b/scripts/run/medsam/miccai_experiments/infer.sh new file mode 100755 index 0000000..631e297 --- /dev/null +++ b/scripts/run/medsam/miccai_experiments/infer.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +python src/segmentation/generic/run/run_infer.py \ + --all_models_root_dir /sddata/data/geographic_atrophy/output_logger/label \ + --config_augmentations_path /sddata/projects/segmentation_generic/yamls/augmentations/medsam/high_augs.yaml \ + --holdout_csv_path /sddata/data/geographic_atrophy/nj_110/csvs/test_bb.csv \ + --model_arch medsam \ + --image_root_dir /sddata/data/geographic_atrophy/nj_110/images \ + --label_root_dir /sddata/data/geographic_atrophy/nj_110/labels \ + --num_workers 4 \ + --gpu_id 3 \ + --label_bbox_option label diff --git a/scripts/run/medsam/miccai_experiments/train1-2.sh b/scripts/run/medsam/miccai_experiments/train1-2.sh new file mode 100644 index 0000000..def8ac2 --- /dev/null +++ b/scripts/run/medsam/miccai_experiments/train1-2.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_10.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 0 + + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_20.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 0 diff --git a/scripts/run/medsam/miccai_experiments/train3-4.sh b/scripts/run/medsam/miccai_experiments/train3-4.sh new file mode 100644 index 0000000..be1d600 --- /dev/null +++ b/scripts/run/medsam/miccai_experiments/train3-4.sh @@ -0,0 +1,15 @@ +#!/bin/sh + + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_30.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 1 + + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_40.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 1 diff --git a/scripts/run/medsam/miccai_experiments/train5-6.sh b/scripts/run/medsam/miccai_experiments/train5-6.sh new file mode 100644 index 0000000..0aea5a2 --- /dev/null +++ b/scripts/run/medsam/miccai_experiments/train5-6.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_50.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 2 + + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_60.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 2 + diff --git a/scripts/run/medsam/miccai_experiments/train7-10.sh b/scripts/run/medsam/miccai_experiments/train7-10.sh new file mode 100644 index 0000000..5687442 --- /dev/null +++ b/scripts/run/medsam/miccai_experiments/train7-10.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_70.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 3 + + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_80.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 3 + + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_90.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 3 + + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_100.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 3 \ No newline at end of file diff --git a/scripts/run/medsam/miccai_experiments/train_frac10.sh b/scripts/run/medsam/miccai_experiments/train_frac10.sh old mode 100644 new mode 100755 index 531bcee..741669a --- a/scripts/run/medsam/miccai_experiments/train_frac10.sh +++ b/scripts/run/medsam/miccai_experiments/train_frac10.sh @@ -2,6 +2,6 @@ python src/segmentation/generic/run/run_train.py \ --model_arch medsam \ - --train_yaml /home/kindersc/repos/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_10.yaml \ - --system_yaml /home/kindersc/repos/segmentation_generic/yamls/system/lt2.yaml \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_10.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ --gpu_id 0 \ No newline at end of file diff --git a/scripts/run/medsam/miccai_experiments/train_frac100.sh b/scripts/run/medsam/miccai_experiments/train_frac100.sh new file mode 100755 index 0000000..4d02fc3 --- /dev/null +++ b/scripts/run/medsam/miccai_experiments/train_frac100.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_100.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 3 \ No newline at end of file diff --git a/scripts/run/medsam/miccai_experiments/train_frac20.sh b/scripts/run/medsam/miccai_experiments/train_frac20.sh old mode 100644 new mode 100755 index 5a71fe5..934957a --- a/scripts/run/medsam/miccai_experiments/train_frac20.sh +++ b/scripts/run/medsam/miccai_experiments/train_frac20.sh @@ -2,6 +2,6 @@ python src/segmentation/generic/run/run_train.py \ --model_arch medsam \ - --train_yaml /home/kindersc/repos/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_20.yaml \ - --system_yaml /home/kindersc/repos/segmentation_generic/yamls/system/lt2.yaml \ - --gpu_id 0 \ No newline at end of file + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_20.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 1 \ No newline at end of file diff --git a/scripts/run/medsam/miccai_experiments/train_frac30.sh b/scripts/run/medsam/miccai_experiments/train_frac30.sh new file mode 100755 index 0000000..0bbe2c5 --- /dev/null +++ b/scripts/run/medsam/miccai_experiments/train_frac30.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_30.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 2 \ No newline at end of file diff --git a/scripts/run/medsam/miccai_experiments/train_frac40.sh b/scripts/run/medsam/miccai_experiments/train_frac40.sh new file mode 100755 index 0000000..4f966dc --- /dev/null +++ b/scripts/run/medsam/miccai_experiments/train_frac40.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_40.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 2 \ No newline at end of file diff --git a/scripts/run/medsam/miccai_experiments/train_frac50.sh b/scripts/run/medsam/miccai_experiments/train_frac50.sh new file mode 100755 index 0000000..d2a4fca --- /dev/null +++ b/scripts/run/medsam/miccai_experiments/train_frac50.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_50.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 3 \ No newline at end of file diff --git a/scripts/run/medsam/miccai_experiments/train_frac60.sh b/scripts/run/medsam/miccai_experiments/train_frac60.sh new file mode 100755 index 0000000..b6ff4bf --- /dev/null +++ b/scripts/run/medsam/miccai_experiments/train_frac60.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_60.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 0 \ No newline at end of file diff --git a/scripts/run/medsam/miccai_experiments/train_frac70.sh b/scripts/run/medsam/miccai_experiments/train_frac70.sh new file mode 100755 index 0000000..a8cfbb8 --- /dev/null +++ b/scripts/run/medsam/miccai_experiments/train_frac70.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_70.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 1 \ No newline at end of file diff --git a/scripts/run/medsam/miccai_experiments/train_frac80.sh b/scripts/run/medsam/miccai_experiments/train_frac80.sh new file mode 100755 index 0000000..9ec476c --- /dev/null +++ b/scripts/run/medsam/miccai_experiments/train_frac80.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_80.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 2 \ No newline at end of file diff --git a/scripts/run/medsam/miccai_experiments/train_frac90.sh b/scripts/run/medsam/miccai_experiments/train_frac90.sh new file mode 100755 index 0000000..7de2fc1 --- /dev/null +++ b/scripts/run/medsam/miccai_experiments/train_frac90.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +python src/segmentation/generic/run/run_train.py \ + --model_arch medsam \ + --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_90.yaml \ + --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \ + --gpu_id 3 \ No newline at end of file diff --git a/src/segmentation/generic/run/run_infer.py b/src/segmentation/generic/run/run_infer.py index 6ddfb0a..87036b0 100644 --- a/src/segmentation/generic/run/run_infer.py +++ b/src/segmentation/generic/run/run_infer.py @@ -23,6 +23,7 @@ def parse_args(): parser.add_argument('--label_root_dir', type=str, required=True, help="Directory where test labels are stored") parser.add_argument('--num_workers', type=int, default=1, help="Number of workers to use") parser.add_argument('--gpu_id', type=int, default=0, help="Which GPU to run on (distributed not available yet)") + parser.add_argument('--label_bbox_option', type=str, default='label', help="Which bounding box type do you want ['label','image','padded','yolo']") return parser.parse_args() @@ -42,11 +43,12 @@ def parse_args(): outputs = {} # Walk through the root directory - for root, dirs, files in os.walk(all_models_root_dir): # iterate over /output_logger/... folder - folder_name = os.path.basename(root) - for file in files: + for dir in os.listdir(all_models_root_dir): # iterate over /output_logger/... folder + root = os.path.join(all_models_root_dir, f"{dir}/wandb-lightning/{dir.split('_')[0]}/checkpoints") + for file in os.listdir(root): # Check if the file ends with .pt or .ckpt if file.endswith('.pt') or file.endswith('.ckpt'): + # import pdb; pdb.set_trace() # Print the full path to the file file_path = os.path.join(root, file) print(f"weights file: {file_path}") @@ -58,10 +60,13 @@ def parse_args(): image_root_dir=image_root_dir, label_root_dir=label_root_dir, gpu_id=0, - num_workers=4 + num_workers=4, + label_bbox_option = args.label_bbox_option ) - outputs[folder_name] = test_val + outputs[dir] = test_val + print("outputs: ", outputs) print('output:') + import pdb; pdb.set_trace() print(outputs) diff --git a/src/segmentation/generic/run/run_train.py b/src/segmentation/generic/run/run_train.py index 692918f..4dfa198 100644 --- a/src/segmentation/generic/run/run_train.py +++ b/src/segmentation/generic/run/run_train.py @@ -1,3 +1,4 @@ +import os import argparse import wandb import lightning.pytorch as pl @@ -28,7 +29,6 @@ def parse_args(): def train_model(config=None): with wandb.init(): config = wandb.config # I think this is {...parameters} - print('using: ', config.frac_num) # config.method @@ -71,7 +71,7 @@ def train_model(config=None): project='wandb-lightning', job_type='train', # log_model='all', - save_dir=config.output_dir + wandb_run_id + save_dir=os.path.join(config.output_dir, f"{wandb_run_id}_{config.frac_num}") ) # log gradients, parameter histogram and model topology diff --git a/src/segmentation/generic/utils/utils_infer.py b/src/segmentation/generic/utils/utils_infer.py index 5ddeec0..35cabfd 100644 --- a/src/segmentation/generic/utils/utils_infer.py +++ b/src/segmentation/generic/utils/utils_infer.py @@ -14,7 +14,7 @@ def infer_model(config_augmentations_path, holdout_csv_path, weights_path, model_arch, image_root_dir, label_root_dir, - gpu_id=0, num_workers=4, csv_img_path_col='image', csv_label_path_col='mask'): + gpu_id=0, num_workers=4, label_bbox_option='label', csv_img_path_col='image', csv_label_path_col='mask'): device = torch.device(f'cuda:{gpu_id}' if torch.cuda.is_available() else "cpu") # # Need for output size etc. # config_augmentations = '/home/kindersc/repos/segmentation_generic/yamls/augmentations/medsam/medium_augs.yaml' @@ -39,7 +39,8 @@ def infer_model(config_augmentations_path, holdout_csv_path, weights_path, model transform=val_transform, preprocessor=preprocess, batch_size=1, - num_workers=num_workers + num_workers=num_workers, + label_bbox_option=label_bbox_option ) configs = { @@ -59,11 +60,12 @@ def infer_model(config_augmentations_path, holdout_csv_path, weights_path, model ) trainer = pl.Trainer( - devices='auto', + # devices='auto', + devices=[gpu_id], accelerator='gpu', ) # Evaluate the model on the test set - test_acc = trainer.test(dataloaders=holdout_dataloader)['test_acc'] + test_acc = trainer.test(model, dataloaders=holdout_dataloader)[0]['test_acc'] return test_acc diff --git a/src/segmentation/generic/utils/utils_train.py b/src/segmentation/generic/utils/utils_train.py index 40142b9..7357640 100644 --- a/src/segmentation/generic/utils/utils_train.py +++ b/src/segmentation/generic/utils/utils_train.py @@ -40,7 +40,7 @@ def get_model_and_processor(config): # TODO: fix def get_dataloader_from_csv(model_arch, csv_path, csv_img_path_col, csv_label_path_col, image_root_dir, label_root_dir, transform, preprocessor, - batch_size, num_workers): + batch_size, num_workers, label_bbox_option): data_df = pd.read_csv(csv_path) image_paths = data_df[csv_img_path_col].tolist() @@ -63,7 +63,8 @@ def get_dataloader_from_csv(model_arch, csv_path, csv_img_path_col, csv_label_pa transform=transform, preprocess=preprocessor, image_root_dir=image_root_dir, - label_root_dir=label_root_dir + label_root_dir=label_root_dir, + label_bbox_option=label_bbox_option ) data_loader = DataLoader(data_set, batch_size=batch_size, shuffle=False, num_workers=num_workers) @@ -77,16 +78,21 @@ def get_dataloaders(config, train_transform, val_transform, preprocess, frac_num image_root_dir = data_root_dir + 'images' label_root_dir = data_root_dir + 'labels' # get csvs - train_csv = data_root_dir + f'csvs/miccai_nj/train_{frac_num}.csv' - val_csv = data_root_dir + f'csvs/miccai_nj/val_{frac_num}.csv' - test_csv = data_root_dir + 'csvs/miccai_nj/test_bb.csv' + # train_csv = data_root_dir + f'csvs/miccai_nj/train_{frac_num}.csv' + # val_csv = data_root_dir + f'csvs/miccai_nj/val_{frac_num}.csv' + # test_csv = data_root_dir + 'csvs/miccai_nj/test_bb.csv' + train_csv = data_root_dir + f'csvs/train_{frac_num}.csv' + val_csv = data_root_dir + f'csvs/val_{frac_num}.csv' + # val_csv = data_root_dir + f'csvs/val_100.csv' + test_csv = data_root_dir + 'csvs/test_bb.csv' # get column names # TODO: fix hardcode? csv_img_path_col = 'image' csv_label_path_col = 'mask' - + # import pdb; pdb.set_trace() train_dataloader = get_dataloader_from_csv( + # config.label_bbox_option model_arch=config.model_arch, csv_path=train_csv, csv_img_path_col=csv_img_path_col, @@ -96,7 +102,8 @@ def get_dataloaders(config, train_transform, val_transform, preprocess, frac_num transform=train_transform, preprocessor=preprocess, batch_size=config.gpu_max_batch_size, - num_workers=config.num_workers + num_workers=config.num_workers, + label_bbox_option=config.label_bbox_option ) val_dataloader = get_dataloader_from_csv( @@ -109,7 +116,8 @@ def get_dataloaders(config, train_transform, val_transform, preprocess, frac_num transform=val_transform, preprocessor=preprocess, batch_size=config.gpu_max_batch_size, - num_workers=config.num_workers//2 + num_workers=config.num_workers//2, + label_bbox_option=config.label_bbox_option ) test_dataloader = get_dataloader_from_csv( @@ -122,7 +130,8 @@ def get_dataloaders(config, train_transform, val_transform, preprocess, frac_num transform=val_transform, preprocessor=preprocess, batch_size=config.gpu_max_batch_size, - num_workers=config.num_workers//2 + num_workers=config.num_workers//2, + label_bbox_option=config.label_bbox_option ) return train_dataloader, val_dataloader, test_dataloader @@ -205,5 +214,6 @@ def get_sweep_config(model_arch, train_params, system_params, gpu_id): 'num_workers': {'values': [system_params['num_workers']]}, 'gpu_max_batch_size': {'values': [system_params['gpu_max_batch_size']]}, 'gpu_id': {'values': [gpu_id]}, + 'label_bbox_option': train_params['label_bbox_option'], }, } diff --git a/src/segmentation/medsam/data_modules/datasets_medsam.py b/src/segmentation/medsam/data_modules/datasets_medsam.py index d1d2c8f..c8b5b1c 100644 --- a/src/segmentation/medsam/data_modules/datasets_medsam.py +++ b/src/segmentation/medsam/data_modules/datasets_medsam.py @@ -11,8 +11,10 @@ class MedSAMDataset(GenericDataset): Mask2Former dataset. """ - def __init__(self, images, masks, transform, preprocess, image_root_dir, label_root_dir): + def __init__(self, images, masks, transform, preprocess, image_root_dir, label_root_dir, label_bbox_option="label"): super().__init__(images, masks, transform, preprocess, image_root_dir, label_root_dir) + self.label_bbox_option = label_bbox_option + def __len__(self): return len(self.images) @@ -36,7 +38,21 @@ def __getitem__(self, idx): image = image.transpose(2, 0, 1) # Get the bounding boxes of the labels - input_boxes = [get_label_bbox(mask_label)] + + if self.label_bbox_option == 'label': + print(f"lself.label_bbox_option: {self.label_bbox_option}") + input_boxes = [get_label_bbox(mask_label, bbox_shift = 0)] # Label bbox + elif self.label_bbox_option == 'image': + print(f"iself.label_bbox_option: {self.label_bbox_option}") + input_boxes = [0., 0., float(image.shape[2]), float(image.shape[1])] # Image bbox + elif self.label_bbox_option == 'padded_label': + print(f"pself.label_bbox_option: {self.label_bbox_option}") + input_boxes = [get_label_bbox(mask_label)] # Label bbox + elif self.label_bbox_option == 'yolo': + print(f"yself.label_bbox_option: {self.label_bbox_option}") + pass + else: + raise Exception("You didn't specify label type") # Get data in way that MedSAM wants batched = self.preprocessor( diff --git a/yamls/system/lt2.yaml b/yamls/system/lt2.yaml index ea0d9df..3b04c32 100644 --- a/yamls/system/lt2.yaml +++ b/yamls/system/lt2.yaml @@ -1,4 +1,4 @@ data_dir: '/sddata/data/geographic_atrophy/nj_110/' -output_dir: '/sddata/projects/segmentation_ga/wandb_outputs/output_logger/' +output_dir: '/sddata/data/geographic_atrophy/output_logger/' num_workers: 8 gpu_max_batch_size: 1 \ No newline at end of file diff --git a/yamls/training/sweeps/medsam/miccai_experiments/experiment_10.yaml b/yamls/training/sweeps/medsam/miccai_experiments/experiment_10.yaml index d06b4d2..055c57d 100644 --- a/yamls/training/sweeps/medsam/miccai_experiments/experiment_10.yaml +++ b/yamls/training/sweeps/medsam/miccai_experiments/experiment_10.yaml @@ -1,5 +1,5 @@ # Info -wb_project: 'miccai_dev' +wb_project: 'miccai_fractionations' wb_sweepname: 'bens_trials_10' # Sweep type sweep_method: 'random' @@ -14,9 +14,9 @@ lr: {'values': [0.00003]} grad_clip_val: {'max': 3.0, 'min': 0.01, 'distribution': 'uniform'} augmentations: { 'values': [ - '/home/kindersc/repos/segmentation_generic/yamls/augmentations/medsam/low_augs.yaml', - '/home/kindersc/repos/segmentation_generic/yamls/augmentations/medsam/medium_augs.yaml', - '/home/kindersc/repos/segmentation_generic/yamls/augmentations/medsam/high_augs.yaml' + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/low_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/medium_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/high_augs.yaml' ] } patience: {'values': [10]} @@ -28,4 +28,6 @@ adamw_weight_decay: {'max': 0.01, 'min': 0.0000001, 'distribution': 'log_uniform sgd_momentum: {'max': 0.99, 'min': 0.01, 'distribution': 'log_uniform'} log_every_n_steps: {'values': [10]} # Frac num -frac_num: {'values': [10]} \ No newline at end of file +frac_num: {'values': [10]} +# label options (label, image, padded_label, yolo) +label_bbox_option: {'values': ['image']} \ No newline at end of file diff --git a/yamls/training/sweeps/medsam/miccai_experiments/experiment_100.yaml b/yamls/training/sweeps/medsam/miccai_experiments/experiment_100.yaml new file mode 100644 index 0000000..f1891c1 --- /dev/null +++ b/yamls/training/sweeps/medsam/miccai_experiments/experiment_100.yaml @@ -0,0 +1,33 @@ +# Info +wb_project: 'miccai_fractionations' +wb_sweepname: 'bens_trials' +# Sweep type +sweep_method: 'random' +sweep_goal: 'minimize' +sweep_metric_name: 'validation_loss' +sweep_count: 3 +# Hyperparams (formatted) +# Still need to specify the dataset frac split to use... +batch_size: {'values': [8]} +# lr: {'max': 0.00007, 'min': 0.000005, 'distribution': 'log_uniform'} +lr: {'values': [0.00005]} +grad_clip_val: {'max': 3.0, 'min': 0.01, 'distribution': 'uniform'} +augmentations: { + 'values': [ + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/low_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/medium_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/high_augs.yaml' + ] +} +patience: {'values': [10]} +max_epochs: {'values': [500]} +early_stopping_monitor: {'values': ['val_loss']} +optimizer_name: {'values': ['adam', 'adamw']} +scheduler_name: {'values': ['exponential_decay', 'cosine_annealing', 'cyclic_lr', 'none']} +adamw_weight_decay: {'max': 0.01, 'min': 0.0000001, 'distribution': 'log_uniform'} +sgd_momentum: {'max': 0.99, 'min': 0.01, 'distribution': 'log_uniform'} +log_every_n_steps: {'values': [10]} +# Frac num +frac_num: {'values': [100]} +# label options (label, image, padded_label, yolo) +label_bbox_option: {'values': ['image']} \ No newline at end of file diff --git a/yamls/training/sweeps/medsam/miccai_experiments/experiment_20.yaml b/yamls/training/sweeps/medsam/miccai_experiments/experiment_20.yaml index 7becb97..42b8498 100644 --- a/yamls/training/sweeps/medsam/miccai_experiments/experiment_20.yaml +++ b/yamls/training/sweeps/medsam/miccai_experiments/experiment_20.yaml @@ -1,5 +1,5 @@ # Info -wb_project: 'miccai_dev' +wb_project: 'miccai_fractionations' wb_sweepname: 'bens_trials' # Sweep type sweep_method: 'random' @@ -10,13 +10,13 @@ sweep_count: 3 # Still need to specify the dataset frac split to use... batch_size: {'values': [8]} # lr: {'max': 0.00007, 'min': 0.000005, 'distribution': 'log_uniform'} -lr: {'values': [0.00003]} +lr: {'values': [0.00005]} grad_clip_val: {'max': 3.0, 'min': 0.01, 'distribution': 'uniform'} augmentations: { 'values': [ - '/home/kindersc/repos/segmentation_generic/yamls/augmentations/medsam/low_augs.yaml', - '/home/kindersc/repos/segmentation_generic/yamls/augmentations/medsam/medium_augs.yaml', - '/home/kindersc/repos/segmentation_generic/yamls/augmentations/medsam/high_augs.yaml' + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/low_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/medium_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/high_augs.yaml' ] } patience: {'values': [10]} @@ -28,4 +28,6 @@ adamw_weight_decay: {'max': 0.01, 'min': 0.0000001, 'distribution': 'log_uniform sgd_momentum: {'max': 0.99, 'min': 0.01, 'distribution': 'log_uniform'} log_every_n_steps: {'values': [10]} # Frac num -frac_num: {'values': [20]} \ No newline at end of file +frac_num: {'values': [20]} +# label options (label, image, padded_label, yolo) +label_bbox_option: {'values': ['image']} \ No newline at end of file diff --git a/yamls/training/sweeps/medsam/miccai_experiments/experiment_30.yaml b/yamls/training/sweeps/medsam/miccai_experiments/experiment_30.yaml new file mode 100644 index 0000000..07bdd81 --- /dev/null +++ b/yamls/training/sweeps/medsam/miccai_experiments/experiment_30.yaml @@ -0,0 +1,33 @@ +# Info +wb_project: 'miccai_fractionations' +wb_sweepname: 'bens_trials' +# Sweep type +sweep_method: 'random' +sweep_goal: 'minimize' +sweep_metric_name: 'validation_loss' +sweep_count: 3 +# Hyperparams (formatted) +# Still need to specify the dataset frac split to use... +batch_size: {'values': [8]} +# lr: {'max': 0.00007, 'min': 0.000005, 'distribution': 'log_uniform'} +lr: {'values': [0.00005]} +grad_clip_val: {'max': 3.0, 'min': 0.01, 'distribution': 'uniform'} +augmentations: { + 'values': [ + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/low_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/medium_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/high_augs.yaml' + ] +} +patience: {'values': [10]} +max_epochs: {'values': [500]} +early_stopping_monitor: {'values': ['val_loss']} +optimizer_name: {'values': ['adam', 'adamw']} +scheduler_name: {'values': ['exponential_decay', 'cosine_annealing', 'cyclic_lr', 'none']} +adamw_weight_decay: {'max': 0.01, 'min': 0.0000001, 'distribution': 'log_uniform'} +sgd_momentum: {'max': 0.99, 'min': 0.01, 'distribution': 'log_uniform'} +log_every_n_steps: {'values': [10]} +# Frac num +frac_num: {'values': [30]} +# label options (label, image, padded_label, yolo) +label_bbox_option: {'values': ['image']} \ No newline at end of file diff --git a/yamls/training/sweeps/medsam/miccai_experiments/experiment_40.yaml b/yamls/training/sweeps/medsam/miccai_experiments/experiment_40.yaml new file mode 100644 index 0000000..de80838 --- /dev/null +++ b/yamls/training/sweeps/medsam/miccai_experiments/experiment_40.yaml @@ -0,0 +1,33 @@ +# Info +wb_project: 'miccai_fractionations' +wb_sweepname: 'bens_trials' +# Sweep type +sweep_method: 'random' +sweep_goal: 'minimize' +sweep_metric_name: 'validation_loss' +sweep_count: 3 +# Hyperparams (formatted) +# Still need to specify the dataset frac split to use... +batch_size: {'values': [8]} +# lr: {'max': 0.00007, 'min': 0.000005, 'distribution': 'log_uniform'} +lr: {'values': [0.00005]} +grad_clip_val: {'max': 3.0, 'min': 0.01, 'distribution': 'uniform'} +augmentations: { + 'values': [ + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/low_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/medium_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/high_augs.yaml' + ] +} +patience: {'values': [10]} +max_epochs: {'values': [500]} +early_stopping_monitor: {'values': ['val_loss']} +optimizer_name: {'values': ['adam', 'adamw']} +scheduler_name: {'values': ['exponential_decay', 'cosine_annealing', 'cyclic_lr', 'none']} +adamw_weight_decay: {'max': 0.01, 'min': 0.0000001, 'distribution': 'log_uniform'} +sgd_momentum: {'max': 0.99, 'min': 0.01, 'distribution': 'log_uniform'} +log_every_n_steps: {'values': [10]} +# Frac num +frac_num: {'values': [40]} +# label options (label, image, padded_label, yolo) +label_bbox_option: {'values': ['image']} \ No newline at end of file diff --git a/yamls/training/sweeps/medsam/miccai_experiments/experiment_50.yaml b/yamls/training/sweeps/medsam/miccai_experiments/experiment_50.yaml new file mode 100644 index 0000000..7230549 --- /dev/null +++ b/yamls/training/sweeps/medsam/miccai_experiments/experiment_50.yaml @@ -0,0 +1,33 @@ +# Info +wb_project: 'miccmiccai_fractionationsai_dev' +wb_sweepname: 'bens_trials' +# Sweep type +sweep_method: 'random' +sweep_goal: 'minimize' +sweep_metric_name: 'validation_loss' +sweep_count: 3 +# Hyperparams (formatted) +# Still need to specify the dataset frac split to use... +batch_size: {'values': [8]} +# lr: {'max': 0.00007, 'min': 0.000005, 'distribution': 'log_uniform'} +lr: {'values': [0.00005]} +grad_clip_val: {'max': 3.0, 'min': 0.01, 'distribution': 'uniform'} +augmentations: { + 'values': [ + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/low_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/medium_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/high_augs.yaml' + ] +} +patience: {'values': [10]} +max_epochs: {'values': [500]} +early_stopping_monitor: {'values': ['val_loss']} +optimizer_name: {'values': ['adam', 'adamw']} +scheduler_name: {'values': ['exponential_decay', 'cosine_annealing', 'cyclic_lr', 'none']} +adamw_weight_decay: {'max': 0.01, 'min': 0.0000001, 'distribution': 'log_uniform'} +sgd_momentum: {'max': 0.99, 'min': 0.01, 'distribution': 'log_uniform'} +log_every_n_steps: {'values': [10]} +# Frac num +frac_num: {'values': [50]} +# label options (label, image, padded_label, yolo) +label_bbox_option: {'values': ['image']} \ No newline at end of file diff --git a/yamls/training/sweeps/medsam/miccai_experiments/experiment_60.yaml b/yamls/training/sweeps/medsam/miccai_experiments/experiment_60.yaml new file mode 100644 index 0000000..05fe2a7 --- /dev/null +++ b/yamls/training/sweeps/medsam/miccai_experiments/experiment_60.yaml @@ -0,0 +1,33 @@ +# Info +wb_project: 'miccai_fractionations' +wb_sweepname: 'bens_trials' +# Sweep type +sweep_method: 'random' +sweep_goal: 'minimize' +sweep_metric_name: 'validation_loss' +sweep_count: 3 +# Hyperparams (formatted) +# Still need to specify the dataset frac split to use... +batch_size: {'values': [8]} +# lr: {'max': 0.00007, 'min': 0.000005, 'distribution': 'log_uniform'} +lr: {'values': [0.00005]} +grad_clip_val: {'max': 3.0, 'min': 0.01, 'distribution': 'uniform'} +augmentations: { + 'values': [ + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/low_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/medium_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/high_augs.yaml' + ] +} +patience: {'values': [10]} +max_epochs: {'values': [500]} +early_stopping_monitor: {'values': ['val_loss']} +optimizer_name: {'values': ['adam', 'adamw']} +scheduler_name: {'values': ['exponential_decay', 'cosine_annealing', 'cyclic_lr', 'none']} +adamw_weight_decay: {'max': 0.01, 'min': 0.0000001, 'distribution': 'log_uniform'} +sgd_momentum: {'max': 0.99, 'min': 0.01, 'distribution': 'log_uniform'} +log_every_n_steps: {'values': [10]} +# Frac num +frac_num: {'values': [60]} +# label options (label, image, padded_label, yolo) +label_bbox_option: {'values': ['image']} \ No newline at end of file diff --git a/yamls/training/sweeps/medsam/miccai_experiments/experiment_70.yaml b/yamls/training/sweeps/medsam/miccai_experiments/experiment_70.yaml new file mode 100644 index 0000000..6737f36 --- /dev/null +++ b/yamls/training/sweeps/medsam/miccai_experiments/experiment_70.yaml @@ -0,0 +1,33 @@ +# Info +wb_project: 'miccai_fractionations' +wb_sweepname: 'bens_trials' +# Sweep type +sweep_method: 'random' +sweep_goal: 'minimize' +sweep_metric_name: 'validation_loss' +sweep_count: 3 +# Hyperparams (formatted) +# Still need to specify the dataset frac split to use... +batch_size: {'values': [8]} +# lr: {'max': 0.00007, 'min': 0.000005, 'distribution': 'log_uniform'} +lr: {'values': [0.00005]} +grad_clip_val: {'max': 3.0, 'min': 0.01, 'distribution': 'uniform'} +augmentations: { + 'values': [ + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/low_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/medium_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/high_augs.yaml' + ] +} +patience: {'values': [10]} +max_epochs: {'values': [500]} +early_stopping_monitor: {'values': ['val_loss']} +optimizer_name: {'values': ['adam', 'adamw']} +scheduler_name: {'values': ['exponential_decay', 'cosine_annealing', 'cyclic_lr', 'none']} +adamw_weight_decay: {'max': 0.01, 'min': 0.0000001, 'distribution': 'log_uniform'} +sgd_momentum: {'max': 0.99, 'min': 0.01, 'distribution': 'log_uniform'} +log_every_n_steps: {'values': [10]} +# Frac num +frac_num: {'values': [70]} +# label options (label, image, padded_label, yolo) +label_bbox_option: {'values': ['image']} \ No newline at end of file diff --git a/yamls/training/sweeps/medsam/miccai_experiments/experiment_80.yaml b/yamls/training/sweeps/medsam/miccai_experiments/experiment_80.yaml new file mode 100644 index 0000000..6b6112a --- /dev/null +++ b/yamls/training/sweeps/medsam/miccai_experiments/experiment_80.yaml @@ -0,0 +1,33 @@ +# Info +wb_project: 'miccai_fractionations' +wb_sweepname: 'bens_trials' +# Sweep type +sweep_method: 'random' +sweep_goal: 'minimize' +sweep_metric_name: 'validation_loss' +sweep_count: 3 +# Hyperparams (formatted) +# Still need to specify the dataset frac split to use... +batch_size: {'values': [8]} +# lr: {'max': 0.00007, 'min': 0.000005, 'distribution': 'log_uniform'} +lr: {'values': [0.00005]} +grad_clip_val: {'max': 3.0, 'min': 0.01, 'distribution': 'uniform'} +augmentations: { + 'values': [ + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/low_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/medium_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/high_augs.yaml' + ] +} +patience: {'values': [10]} +max_epochs: {'values': [500]} +early_stopping_monitor: {'values': ['val_loss']} +optimizer_name: {'values': ['adam', 'adamw']} +scheduler_name: {'values': ['exponential_decay', 'cosine_annealing', 'cyclic_lr', 'none']} +adamw_weight_decay: {'max': 0.01, 'min': 0.0000001, 'distribution': 'log_uniform'} +sgd_momentum: {'max': 0.99, 'min': 0.01, 'distribution': 'log_uniform'} +log_every_n_steps: {'values': [10]} +# Frac num +frac_num: {'values': [80]} +# label options (label, image, padded_label, yolo) +label_bbox_option: {'values': ['image']} \ No newline at end of file diff --git a/yamls/training/sweeps/medsam/miccai_experiments/experiment_90.yaml b/yamls/training/sweeps/medsam/miccai_experiments/experiment_90.yaml new file mode 100644 index 0000000..90b92cd --- /dev/null +++ b/yamls/training/sweeps/medsam/miccai_experiments/experiment_90.yaml @@ -0,0 +1,33 @@ +# Info +wb_project: 'miccai_fractionations' +wb_sweepname: 'bens_trials' +# Sweep type +sweep_method: 'random' +sweep_goal: 'minimize' +sweep_metric_name: 'validation_loss' +sweep_count: 3 +# Hyperparams (formatted) +# Still need to specify the dataset frac split to use... +batch_size: {'values': [8]} +# lr: {'max': 0.00007, 'min': 0.000005, 'distribution': 'log_uniform'} +lr: {'values': [0.00005]} +grad_clip_val: {'max': 3.0, 'min': 0.01, 'distribution': 'uniform'} +augmentations: { + 'values': [ + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/low_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/medium_augs.yaml', + '/sddata/projects/segmentation_generic/yamls/augmentations/medsam/high_augs.yaml' + ] +} +patience: {'values': [10]} +max_epochs: {'values': [500]} +early_stopping_monitor: {'values': ['val_loss']} +optimizer_name: {'values': ['adam', 'adamw']} +scheduler_name: {'values': ['exponential_decay', 'cosine_annealing', 'cyclic_lr', 'none']} +adamw_weight_decay: {'max': 0.01, 'min': 0.0000001, 'distribution': 'log_uniform'} +sgd_momentum: {'max': 0.99, 'min': 0.01, 'distribution': 'log_uniform'} +log_every_n_steps: {'values': [10]} +# Frac num +frac_num: {'values': [90]} +# label options (label, image, padded_label, yolo) +label_bbox_option: {'values': ['image']} \ No newline at end of file