update docs, recreate py files

3.evaluate_model/README.md

@@ -7,9 +7,12 @@ These predictions are saved in [model_predictions.tsv](evaluations/model_predict
 
 We evaluate these 4 sets of predictions with a confusion matrix to see the true/false positives and negatives (see [sklearn.metrics.confusion_matrix](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html) for more details).
 
-We also evaluate these 6 sets of predictions with [sklearn.metrics.f1_score](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html) to determine the final/shuffled baseline model's predictive performance on each subset.
+We also evaluate these 4 sets of predictions with [sklearn.metrics.f1_score](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html) to determine the final/shuffled baseline model's predictive performance on each subset.
 F1 score measures the models precision and recall performance for each phenotypic class.
 
+In [class_PR_curves.ipynb](class_PR_curves.ipynb), we use [sklearn.metrics.precision_recall_curve](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_curve.html) to derive the precision-recall curves for final and shuffled baseline models on the training and testing data subsets.
+The precision recall curves and their data are saved to [class_precision_recall_curves](class_precision_recall_curves/).
+
 ## Step 1: Evaluate Model
 
 Use the commands below to evaluate the ML models:

3.evaluate_model/evaluate_model.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Convert notebook to python file and execute
+# Convert notebooks to python file and execute
 jupyter nbconvert --to python \
         --FilesWriter.build_directory=scripts/nbconverted \
-        --execute evaluate_model.ipynb
+        --execute *.ipynb

3.evaluate_model/scripts/nbconverted/class_PR_curves.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# ### Import Libraries
+
+# In[1]:
+
+
+import pandas as pd
+import pathlib
+from joblib import load
+
+import sys
+sys.path.append("../utils")
+from split_utils import get_features_data
+from train_utils import get_dataset
+from evaluate_utils import class_PR_curves
+
+
+# ### Load models and datasets
+
+# In[2]:
+
+
+model_dir = pathlib.Path("../2.train_model/models/")
+log_reg_model_path = pathlib.Path(f"{model_dir}/log_reg_model.joblib")
+log_reg_model = load(log_reg_model_path)
+shuffled_baseline_log_reg_model_path = pathlib.Path(f"{model_dir}/shuffled_baseline_log_reg_model.joblib")
+shuffled_baseline_log_reg_model = load(shuffled_baseline_log_reg_model_path)
+
+# load features data from indexes and features dataframe
+data_split_path = pathlib.Path("../1.split_data/indexes/data_split_indexes.tsv")
+data_split_indexes = pd.read_csv(data_split_path, sep="\t", index_col=0)
+features_dataframe_path = pathlib.Path("../0.download_data/data/training_data.csv.gz")
+features_dataframe = get_features_data(features_dataframe_path)
+
+train_data = get_dataset(features_dataframe, data_split_indexes, "train")
+test_data = get_dataset(features_dataframe, data_split_indexes, "test")
+
+save_dir = pathlib.Path("class_precision_recall_curves/")
+save_dir.mkdir(parents=True, exist_ok=True)
+
+
+# ### PR Curves for combinations of models and data subsets
+
+# In[3]:
+
+
+fig, PR_data  = class_PR_curves(train_data, log_reg_model)
+fig.suptitle("Precision Recall Curves for Final Model on Training Data")
+fig.savefig(f"{save_dir}/final_model_train_data.png")
+PR_data.to_csv(f"{save_dir}/final_model_train_data.tsv", sep="\t")
+
+
+# In[4]:
+
+
+fig, PR_data  = class_PR_curves(train_data, shuffled_baseline_log_reg_model)
+fig.suptitle("Precision Recall Curves for Shuffled Baseline Model on Training Data")
+fig.savefig(f"{save_dir}/shuffled_baseline_model_train_data.png")
+PR_data.to_csv(f"{save_dir}/shuffled_baseline_model_train_data.tsv", sep="\t")
+
+
+# In[5]:
+
+
+fig, PR_data  = class_PR_curves(test_data, log_reg_model)
+fig.suptitle("Precision Recall Curves for Final Model on Testing Data")
+fig.savefig(f"{save_dir}/final_model_test_data.png")
+PR_data.to_csv(f"{save_dir}/final_model_test_data.tsv", sep="\t")
+
+
+# In[6]:
+
+
+fig, PR_data  = class_PR_curves(test_data, shuffled_baseline_log_reg_model)
+fig.suptitle("Precision Recall Curves for Shuffled Baseline Model on Testing Data")
+fig.savefig(f"{save_dir}/shuffled_baseline_model_test_data.png")
+PR_data.to_csv(f"{save_dir}/shuffled_baseline_model_test_data.tsv", sep="\t")
+

3.evaluate_model/scripts/nbconverted/evaluate_model.py

@@ -16,7 +16,7 @@
 import sys
 sys.path.append("../utils")
 from split_utils import get_features_data
-from train_utils import get_dataset, get_X_y_data
+from train_utils import get_dataset
 from evaluate_utils import evaluate_model_cm, evaluate_model_score