diff --git a/docs/howto/grouped_data.ipynb b/docs/howto/grouped_data.ipynb new file mode 100644 index 00000000..1bfbdd23 --- /dev/null +++ b/docs/howto/grouped_data.ipynb @@ -0,0 +1,273 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# How to work with grouped data\n", + "\n", + "One of the often appearing properties of the Data Science problems is the natural grouping of the data. You could for instance have multiple samples for the same customer. In such case, you need to make sure that all samples from a given group are in the same fold e.g. in Cross-Validation.\n", + "\n", + "Let's prepare a dataset with groups." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[0, 1, 2, 3, 4, 0, 1, 2, 3, 4]" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.datasets import make_classification\n", + "\n", + "X, y = make_classification(n_samples=100, n_features=10, random_state=42)\n", + "groups = [i % 5 for i in range(100)]\n", + "groups[:10]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The integers in `groups` variable indicate the group id, to which a given sample belongs.\n", + "\n", + "One of the easiest ways to ensure that the data is split using the information about groups is using `from sklearn.model_selection import GroupKFold`. You can also read more about other ways of splitting data with groups in sklearn [here](https://scikit-learn.org/stable/modules/cross_validation.html#cross-validation-iterators-for-grouped-data)." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import GroupKFold\n", + "\n", + "cv = GroupKFold(n_splits=5).split(X, y, groups=groups)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Such variable can be passed to the `cv` parameter in `probatus` functions" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from probatus.feature_elimination import ShapRFECV\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.model_selection import RandomizedSearchCV\n", + "\n", + "clf = RandomForestClassifier(random_state=42)\n", + "\n", + "param_grid = {\n", + " 'n_estimators': [5, 7, 10],\n", + " 'max_leaf_nodes': [3, 5, 7, 10],\n", + "}\n", + "search = RandomizedSearchCV(clf, param_grid, n_iter=1, random_state=42)\n", + "\n", + "shap_elimination = ShapRFECV(\n", + " clf=search, step=0.2, cv=cv, scoring='roc_auc', n_jobs=3, random_state=42)\n", + "report = shap_elimination.fit_compute(X, y)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
num_featuresfeatures_seteliminated_featurestrain_metric_meantrain_metric_stdval_metric_meanval_metric_std
110[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][8, 7]1.0000.0010.9570.086
28[0, 1, 2, 3, 4, 5, 6, 9][5]0.9990.0010.9660.055
37[0, 1, 2, 3, 4, 6, 9][4]1.0000.0000.9420.114
46[0, 1, 2, 3, 6, 9][9]0.9990.0010.9800.032
55[0, 1, 2, 3, 6][6]1.0000.0000.9600.073
64[0, 1, 2, 3][1]0.9990.0010.9510.091
73[0, 2, 3][3]0.9990.0010.9710.052
82[0, 2][0]0.9980.0020.9250.122
91[2][]0.9980.0020.9380.098
\n", + "
" + ], + "text/plain": [ + " num_features features_set eliminated_features \\\n", + "1 10 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] [8, 7] \n", + "2 8 [0, 1, 2, 3, 4, 5, 6, 9] [5] \n", + "3 7 [0, 1, 2, 3, 4, 6, 9] [4] \n", + "4 6 [0, 1, 2, 3, 6, 9] [9] \n", + "5 5 [0, 1, 2, 3, 6] [6] \n", + "6 4 [0, 1, 2, 3] [1] \n", + "7 3 [0, 2, 3] [3] \n", + "8 2 [0, 2] [0] \n", + "9 1 [2] [] \n", + "\n", + " train_metric_mean train_metric_std val_metric_mean val_metric_std \n", + "1 1.000 0.001 0.957 0.086 \n", + "2 0.999 0.001 0.966 0.055 \n", + "3 1.000 0.000 0.942 0.114 \n", + "4 0.999 0.001 0.980 0.032 \n", + "5 1.000 0.000 0.960 0.073 \n", + "6 0.999 0.001 0.951 0.091 \n", + "7 0.999 0.001 0.971 0.052 \n", + "8 0.998 0.002 0.925 0.122 \n", + "9 0.998 0.002 0.938 0.098 " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "report" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/mkdocs.yml b/mkdocs.yml index 1b17b6eb..fb0985c1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -18,7 +18,8 @@ nav: - Univariate Sample Similarity: tutorials/nb_distribution_statistics.ipynb - Custom Scoring Metrics: tutorials/nb_custom_scoring.ipynb - HowTo: - - Reproducibility of the results: howto/reproducibility.ipynb + - Reproduce the results: howto/reproducibility.ipynb + - Work with grouped data: howto/grouped_data.ipynb - API: - probatus.feature_elimination: api/feature_elimination.md - probatus.interpret: api/model_interpret.md