bens_current

.gitignore

@@ -50,3 +50,4 @@ wandb/
 # Other
 # yamls/
 # scripts/run
+lightning_logs/

scripts/run/medsam/miccai_experiments/infer.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+python src/segmentation/generic/run/run_infer.py \
+    --all_models_root_dir /sddata/data/geographic_atrophy/output_logger/label \
+    --config_augmentations_path /sddata/projects/segmentation_generic/yamls/augmentations/medsam/high_augs.yaml \
+    --holdout_csv_path /sddata/data/geographic_atrophy/nj_110/csvs/test_bb.csv \
+    --model_arch medsam \
+    --image_root_dir /sddata/data/geographic_atrophy/nj_110/images \
+    --label_root_dir /sddata/data/geographic_atrophy/nj_110/labels \
+    --num_workers 4 \
+    --gpu_id 3 \
+    --label_bbox_option label

scripts/run/medsam/miccai_experiments/train1-2.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_10.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 0
+
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_20.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 0

scripts/run/medsam/miccai_experiments/train3-4.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_30.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 1
+
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_40.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 1

scripts/run/medsam/miccai_experiments/train5-6.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_50.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 2
+
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_60.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 2
+

scripts/run/medsam/miccai_experiments/train7-10.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_70.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 3
+
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_80.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 3
+
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_90.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 3
+
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_100.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 3

scripts/run/medsam/miccai_experiments/train_frac10.sh

@@ -2,6 +2,6 @@
 
 python src/segmentation/generic/run/run_train.py \
     --model_arch medsam \
-    --train_yaml /home/kindersc/repos/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_10.yaml \
-    --system_yaml /home/kindersc/repos/segmentation_generic/yamls/system/lt2.yaml \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_10.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
     --gpu_id 0

scripts/run/medsam/miccai_experiments/train_frac100.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_100.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 3

scripts/run/medsam/miccai_experiments/train_frac20.sh

@@ -2,6 +2,6 @@
 
 python src/segmentation/generic/run/run_train.py \
     --model_arch medsam \
-    --train_yaml /home/kindersc/repos/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_20.yaml \
-    --system_yaml /home/kindersc/repos/segmentation_generic/yamls/system/lt2.yaml \
-    --gpu_id 0
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_20.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 1

scripts/run/medsam/miccai_experiments/train_frac30.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_30.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 2

scripts/run/medsam/miccai_experiments/train_frac40.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_40.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 2

scripts/run/medsam/miccai_experiments/train_frac50.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_50.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 3

scripts/run/medsam/miccai_experiments/train_frac60.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_60.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 0

scripts/run/medsam/miccai_experiments/train_frac70.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_70.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 1

scripts/run/medsam/miccai_experiments/train_frac80.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_80.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 2

scripts/run/medsam/miccai_experiments/train_frac90.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+python src/segmentation/generic/run/run_train.py \
+    --model_arch medsam \
+    --train_yaml /sddata/projects/segmentation_generic/yamls/training/sweeps/medsam/miccai_experiments/experiment_90.yaml \
+    --system_yaml /sddata/projects/segmentation_generic/yamls/system/lt2.yaml \
+    --gpu_id 3

src/segmentation/generic/run/run_infer.py

@@ -23,6 +23,7 @@ def parse_args():
     parser.add_argument('--label_root_dir', type=str, required=True, help="Directory where test labels are stored")
     parser.add_argument('--num_workers', type=int, default=1, help="Number of workers to use")
     parser.add_argument('--gpu_id', type=int, default=0, help="Which GPU to run on (distributed not available yet)")
+    parser.add_argument('--label_bbox_option', type=str, default='label', help="Which bounding box type do you want ['label','image','padded','yolo']")
 
     return parser.parse_args()
 
@@ -42,11 +43,12 @@ def parse_args():
     outputs = {}
 
     # Walk through the root directory
-    for root, dirs, files in os.walk(all_models_root_dir):  # iterate over /output_logger/... folder
-        folder_name = os.path.basename(root)
-        for file in files:
+    for dir in os.listdir(all_models_root_dir):  # iterate over /output_logger/... folder
+        root = os.path.join(all_models_root_dir, f"{dir}/wandb-lightning/{dir.split('_')[0]}/checkpoints")
+        for file in os.listdir(root):
             # Check if the file ends with .pt or .ckpt
             if file.endswith('.pt') or file.endswith('.ckpt'):
+                # import pdb; pdb.set_trace()
                 # Print the full path to the file
                 file_path = os.path.join(root, file)
                 print(f"weights file: {file_path}")
@@ -58,10 +60,13 @@ def parse_args():
                     image_root_dir=image_root_dir,
                     label_root_dir=label_root_dir,
                     gpu_id=0,
-                    num_workers=4
+                    num_workers=4,
+                    label_bbox_option = args.label_bbox_option
                 )
 
-                outputs[folder_name] = test_val
+                outputs[dir] = test_val
+                print("outputs: ", outputs)
 
     print('output:')
+    import pdb; pdb.set_trace()
     print(outputs)

src/segmentation/generic/run/run_train.py

@@ -1,3 +1,4 @@
+import os
 import argparse
 import wandb
 import lightning.pytorch as pl
@@ -28,7 +29,6 @@ def parse_args():
 def train_model(config=None):
     with wandb.init():
         config = wandb.config  # I think this is {...parameters}
-
         print('using: ', config.frac_num)
         # config.method
 
@@ -71,7 +71,7 @@ def train_model(config=None):
             project='wandb-lightning',
             job_type='train',
             # log_model='all',
-            save_dir=config.output_dir + wandb_run_id
+            save_dir=os.path.join(config.output_dir, f"{wandb_run_id}_{config.frac_num}")
         )
 
         # log gradients, parameter histogram and model topology

src/segmentation/generic/utils/utils_infer.py

@@ -14,7 +14,7 @@
 
 
 def infer_model(config_augmentations_path, holdout_csv_path, weights_path, model_arch, image_root_dir, label_root_dir,
-                gpu_id=0, num_workers=4, csv_img_path_col='image', csv_label_path_col='mask'):
+                gpu_id=0, num_workers=4, label_bbox_option='label', csv_img_path_col='image', csv_label_path_col='mask'):
     device = torch.device(f'cuda:{gpu_id}' if torch.cuda.is_available() else "cpu")
     # # Need for output size etc.
     # config_augmentations = '/home/kindersc/repos/segmentation_generic/yamls/augmentations/medsam/medium_augs.yaml'
@@ -39,7 +39,8 @@ def infer_model(config_augmentations_path, holdout_csv_path, weights_path, model
         transform=val_transform,
         preprocessor=preprocess,
         batch_size=1,
-        num_workers=num_workers
+        num_workers=num_workers,
+        label_bbox_option=label_bbox_option
     )
 
     configs = {
@@ -59,11 +60,12 @@ def infer_model(config_augmentations_path, holdout_csv_path, weights_path, model
     )
 
     trainer = pl.Trainer(
-        devices='auto',
+        # devices='auto',
+        devices=[gpu_id],
         accelerator='gpu',
     )
 
     # Evaluate the model on the test set
-    test_acc = trainer.test(dataloaders=holdout_dataloader)['test_acc']
+    test_acc = trainer.test(model, dataloaders=holdout_dataloader)[0]['test_acc']
 
     return test_acc

src/segmentation/generic/utils/utils_train.py

@@ -40,7 +40,7 @@ def get_model_and_processor(config):
 # TODO: fix
 def get_dataloader_from_csv(model_arch, csv_path, csv_img_path_col, csv_label_path_col,
                             image_root_dir, label_root_dir, transform, preprocessor,
-                            batch_size, num_workers):
+                            batch_size, num_workers, label_bbox_option):
     data_df = pd.read_csv(csv_path)
 
     image_paths = data_df[csv_img_path_col].tolist()
@@ -63,7 +63,8 @@ def get_dataloader_from_csv(model_arch, csv_path, csv_img_path_col, csv_label_pa
             transform=transform,
             preprocess=preprocessor,
             image_root_dir=image_root_dir,
-            label_root_dir=label_root_dir
+            label_root_dir=label_root_dir,
+            label_bbox_option=label_bbox_option
         )
 
     data_loader = DataLoader(data_set, batch_size=batch_size, shuffle=False, num_workers=num_workers)
@@ -77,16 +78,21 @@ def get_dataloaders(config, train_transform, val_transform, preprocess, frac_num
     image_root_dir = data_root_dir + 'images'
     label_root_dir = data_root_dir + 'labels'
     # get csvs
-    train_csv = data_root_dir + f'csvs/miccai_nj/train_{frac_num}.csv'
-    val_csv = data_root_dir + f'csvs/miccai_nj/val_{frac_num}.csv'
-    test_csv = data_root_dir + 'csvs/miccai_nj/test_bb.csv'
+    # train_csv = data_root_dir + f'csvs/miccai_nj/train_{frac_num}.csv'
+    # val_csv = data_root_dir + f'csvs/miccai_nj/val_{frac_num}.csv'
+    # test_csv = data_root_dir + 'csvs/miccai_nj/test_bb.csv'
+    train_csv = data_root_dir + f'csvs/train_{frac_num}.csv'
+    val_csv = data_root_dir + f'csvs/val_{frac_num}.csv'
+    # val_csv = data_root_dir + f'csvs/val_100.csv'
+    test_csv = data_root_dir + 'csvs/test_bb.csv'
 
     # get column names
     # TODO: fix hardcode?
     csv_img_path_col = 'image'
     csv_label_path_col = 'mask'
-
+    # import pdb; pdb.set_trace()
     train_dataloader = get_dataloader_from_csv(
+        # config.label_bbox_option
         model_arch=config.model_arch,
         csv_path=train_csv,
         csv_img_path_col=csv_img_path_col,
@@ -96,7 +102,8 @@ def get_dataloaders(config, train_transform, val_transform, preprocess, frac_num
         transform=train_transform,
         preprocessor=preprocess,
         batch_size=config.gpu_max_batch_size,
-        num_workers=config.num_workers
+        num_workers=config.num_workers,
+        label_bbox_option=config.label_bbox_option
     )
 
     val_dataloader = get_dataloader_from_csv(
@@ -109,7 +116,8 @@ def get_dataloaders(config, train_transform, val_transform, preprocess, frac_num
         transform=val_transform,
         preprocessor=preprocess,
         batch_size=config.gpu_max_batch_size,
-        num_workers=config.num_workers//2
+        num_workers=config.num_workers//2,
+        label_bbox_option=config.label_bbox_option
     )
 
     test_dataloader = get_dataloader_from_csv(
@@ -122,7 +130,8 @@ def get_dataloaders(config, train_transform, val_transform, preprocess, frac_num
         transform=val_transform,
         preprocessor=preprocess,
         batch_size=config.gpu_max_batch_size,
-        num_workers=config.num_workers//2
+        num_workers=config.num_workers//2,
+        label_bbox_option=config.label_bbox_option
     )
 
     return train_dataloader, val_dataloader, test_dataloader
@@ -205,5 +214,6 @@ def get_sweep_config(model_arch, train_params, system_params, gpu_id):
             'num_workers': {'values': [system_params['num_workers']]},
             'gpu_max_batch_size': {'values': [system_params['gpu_max_batch_size']]},
             'gpu_id': {'values': [gpu_id]},
+            'label_bbox_option': train_params['label_bbox_option'],
         },
     }