From 4d5ca54873a09d327e21323283295e9549f8f95a Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 7 Jun 2018 13:57:45 +0300 Subject: [PATCH 01/13] simplify holdout index extraction --- polara/recommender/data.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index c026498..40748f0 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -685,7 +685,7 @@ def _sample_holdout(self, test_split, group_id=None): selector = selector.sample(frac=1, random_state=random_state) group_id = group_id or self.fields.userid - grouper = selector.groupby(self._data[group_id], sort=False) + grouper = selector.groupby(self._data[group_id], sort=False, group_keys=False) if self._random_holdout: # randomly sample data for evaluation random_state = np.random.RandomState(self.seed) @@ -708,8 +708,7 @@ def sample_largest(x): return x.iloc[np.argpartition(x, -size)[-size:]] holdout = grouper.apply(sample_largest) - holdout_index = holdout.index.get_level_values(1) - return self._data.loc[holdout_index] + return self._data.loc[holdout.index] def _sample_testset(self, test_split, holdout_index): From 0b19dd0a7871f866bfacca86134b1716af630a7b Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 7 Jun 2018 13:58:56 +0300 Subject: [PATCH 02/13] improve random holdout sampling efficiency --- polara/recommender/data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index 40748f0..1af7a6b 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -12,8 +12,10 @@ def random_choice(df, num, random_state): n = df.shape[0] - k = min(num, n) - return df.iloc[random_state.choice(n, k, replace=False)] + if n > num: + return df.take(random_state.choice(n, num, replace=False), is_copy=False) + else: + return df def random_sample(df, frac, random_state): From e314a129543c6f5b459e3ae29b98b99a9f7fb82d Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 7 Jun 2018 13:59:32 +0300 Subject: [PATCH 03/13] fix empty feedback indexing bug --- polara/recommender/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index 1af7a6b..9941d9e 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -677,7 +677,7 @@ def reindex(data, col, sort=True, inplace=True): def _sample_holdout(self, test_split, group_id=None): # TODO order_field may also change - need to check it as well - order_field = self._custom_order or self.fields.feedback + order_field = self._custom_order or self.fields.feedback or [] selector = self._data.loc[test_split, order_field] # data may have many items with the same top ratings From 82353ed993fe6b3697b2e4eb75f782d8092c57b2 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 7 Jun 2018 14:48:08 +0300 Subject: [PATCH 04/13] assign developer version tag --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eb47339..3525a33 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ opts = dict(name="polara", description="Fast and flexible recommender system framework", keywords = "recommender system", - version = "0.6.0", + version = "0.6.0.dev", license="MIT", author="Evgeny Frolov", platforms=["any"], From 8d62e3ed2b91ca93067992da8003a94958e3bf9b Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 7 Jun 2018 17:55:12 +0300 Subject: [PATCH 05/13] fix bug with data preparation completion message --- polara/recommender/data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index 9941d9e..ba161c1 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -221,8 +221,10 @@ def prepare(self): self._try_sort_test_data() if self.verbose: + num_train_events = self.training.shape[0] if self.training is not None else 0 + num_holdout_events = self.test.holdout.shape[0] if self.test.holdout is not None else 0 stats_msg = 'Done.\nThere are {} events in the training and {} events in the holdout.' - print(stats_msg.format(self.training.shape[0], self.test.holdout.shape[0])) + print(stats_msg.format(num_train_events, num_holdout_events)) def prepare_training_only(self): self.holdout_size = 0 # do not form holdout From 8d0a46f727c35d0cd844ac5846f8037815583c66 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 7 Jun 2018 20:33:39 +0300 Subject: [PATCH 06/13] implement feedback value thresholding for all models --- polara/recommender/data.py | 26 ++++++++-- polara/recommender/defaults.py | 1 + .../external/mymedialite/mmlwrapper.py | 6 +-- polara/recommender/models.py | 49 ++++++++++++++----- 4 files changed, 63 insertions(+), 19 deletions(-) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index ba161c1..e4a8234 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -735,7 +735,24 @@ def _sample_testset(self, test_split, holdout_index): return sampled - def to_coo(self, tensor_mode=False): + @staticmethod + def threshold_data(idx, val, threshold, filter_values=True): + if threshold is None: + return idx, val + + value_filter = val >= threshold + if filter_values: + val = val[value_filter] + if isinstance(idx, tuple): + idx = tuple([x[value_filter] for x in idx]) + else: + idx = idx[value_filter, :] + else: + val[~value_filter] = 0 + return idx, val + + + def to_coo(self, tensor_mode=False, feedback_threshold=None): userid, itemid, feedback = self.fields user_item_data = self.training[[userid, itemid]].values @@ -755,6 +772,7 @@ def to_coo(self, tensor_mode=False): val = self.training[feedback].values shp = tuple(idx.max(axis=0) + 1) + idx, val = self.threshold_data(idx, val, feedback_threshold) idx = idx.astype(np.intp) val = np.ascontiguousarray(val) return idx, val, shp @@ -775,7 +793,7 @@ def _recover_testset(self, update_data=False): return testset - def test_to_coo(self, tensor_mode=False): + def test_to_coo(self, tensor_mode=False, feedback_threshold=None): userid, itemid, feedback = self.fields testset = self.test.testset @@ -801,8 +819,8 @@ def test_to_coo(self, tensor_mode=False): else: fdbk_val = testset[feedback].values test_coo = (user_idx, item_idx, fdbk_val) - - return test_coo + test_coo, val = self.threshold_data(test_coo[:-1], test_coo[-1], feedback_threshold, filter_values=False) + return test_coo + (val,) def get_test_shape(self, tensor_mode=False): diff --git a/polara/recommender/defaults.py b/polara/recommender/defaults.py index 161d728..6065d05 100644 --- a/polara/recommender/defaults.py +++ b/polara/recommender/defaults.py @@ -15,6 +15,7 @@ #MODELS +feedback_threshold = None switch_positive = None #feedback values below are treated as negative feedback verify_integrity = True #svd diff --git a/polara/recommender/external/mymedialite/mmlwrapper.py b/polara/recommender/external/mymedialite/mmlwrapper.py index cf4cf7c..96f84b0 100644 --- a/polara/recommender/external/mymedialite/mmlwrapper.py +++ b/polara/recommender/external/mymedialite/mmlwrapper.py @@ -69,14 +69,14 @@ def command(self): ' --save-item-mapping={{item_mapping}}').format(command_template) else: command = ('{} --no-id-mapping' - ' --rating-threshold={{switch_positive}}').format(command_template) + ' --rating-threshold={{feedback_threshold}}').format(command_template) return command def _save_to_disk(self): if self.positive_only: feedback = self.data.fields.feedback - pos_ind = self.data.training[feedback] >= self.switch_positive + pos_ind = self.data.training[feedback] >= self.feedback_threshold pos_data = self.data.training.loc[pos_ind] pos_data.to_csv(self.train_data_path, index=False, header=False) else: @@ -99,7 +99,7 @@ def _run_external(self, debug=False): program=self.program, train_path=self.train_data_path, saved_model_path=self.saved_model_path, - switch_positive=self.switch_positive, + feedback_threshold=self.feedback_threshold, topk=self.topk, algo=method_name, options=self.options, diff --git a/polara/recommender/models.py b/polara/recommender/models.py index c2e47ab..52eecc1 100644 --- a/polara/recommender/models.py +++ b/polara/recommender/models.py @@ -70,22 +70,19 @@ def __new__(mcs, name, bases, clsdict): @with_metaclass(MetaModel) class RecommenderModel(object): - _config = ('topk', 'filter_seen', 'switch_positive', 'verify_integrity') + _config = ('topk', 'filter_seen', 'switch_positive', 'feedback_threshold', 'verify_integrity') _pad_const = -1 # used for sparse data - def __init__(self, recommender_data, switch_positive=None): + def __init__(self, recommender_data, feedback_threshold=None): self.data = recommender_data self._recommendations = None self.method = 'ABC' self._topk = get_default('topk') - self.filter_seen = get_default('filter_seen') - # `switch_positive` can be used by other models during construction process - # (e.g. mymedialite wrapper or any other implicit model); hence, it's - # better to make it a model attribute, not a simple evaluation argument - # (in contrast to `on_feedback_level` argument of self.evaluate) - self.switch_positive = switch_positive or get_default('switch_positive') + self._filter_seen = get_default('filter_seen') + self._feedback_threshold = feedback_threshold or get_default('feedback_threshold') + self.switch_positive = get_default('switch_positive') self.verify_integrity = get_default('verify_integrity') self.max_test_workers = get_default('max_test_workers') @@ -130,13 +127,34 @@ def topk(self, new_value): self._recommendations = None # if topk is too high - recalculate recommendations self._topk = new_value + @property + def feedback_threshold(self): + return self._feedback_threshold + + @feedback_threshold.setter + def feedback_threshold(self, new_value): + if self._feedback_threshold != new_value: + self._feedback_threshold = new_value + self._renew_model() + + @property + def filter_seen(self): + return self._filter_seen + + @filter_seen.setter + def filter_seen(self, new_value): + if self._filter_seen != new_value: + self._filter_seen = new_value + self._refresh_model() + def build(self): raise NotImplementedError('This must be implemented in subclasses') - def get_training_matrix(self, dtype=None): - idx, val, shp = self.data.to_coo(tensor_mode=False) + def get_training_matrix(self, feedback_threshold=None, dtype=None): + threshold = feedback_threshold or self.feedback_threshold + idx, val, shp = self.data.to_coo(tensor_mode=False, feedback_threshold=threshold) dtype = dtype or val.dtype matrix = csr_matrix((val, (idx[:, 0], idx[:, 1])), shape=shp, dtype=dtype) @@ -160,6 +178,12 @@ def get_test_matrix(self, test_data=None, shape=None, user_slice=None): coo_data = test_data user_coo, item_coo, fdbk_coo = coo_data + valid_fdbk = fdbk_coo != 0 + if not valid_fdbk.all(): + user_coo = user_coo[valid_fdbk] + item_coo = item_coo[valid_fdbk] + fdbk_coo = fdbk_coo[valid_fdbk] + num_items = shape[1] test_matrix = csr_matrix((fdbk_coo, (user_coo, item_coo)), shape=(num_users, num_items), @@ -180,14 +204,15 @@ def _get_slices_idx(self, shape, result_width=None, scores_multiplier=None, dtyp return slices_idx - def _get_test_data(self): + def _get_test_data(self, feedback_threshold=None): try: tensor_mode = self.factors.get(self.data.fields.feedback, None) is not None except AttributeError: tensor_mode = False - user_idx, item_idx, feedback = self.data.test_to_coo(tensor_mode=tensor_mode) test_shape = self.data.get_test_shape(tensor_mode=tensor_mode) + threshold = feedback_threshold or self.feedback_threshold + user_idx, item_idx, feedback = self.data.test_to_coo(tensor_mode=tensor_mode, feedback_threshold=threshold) idx_diff = np.diff(user_idx) # TODO sorting by self._predition_key From df731f25d0695804e8468a52347b5304041c09eb Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Thu, 7 Jun 2018 20:38:18 +0300 Subject: [PATCH 07/13] allow return some common model config values --- polara/recommender/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/polara/recommender/models.py b/polara/recommender/models.py index 52eecc1..1a0a9d5 100644 --- a/polara/recommender/models.py +++ b/polara/recommender/models.py @@ -148,6 +148,11 @@ def filter_seen(self, new_value): self._refresh_model() + def get_base_configuration(self): + config = {attr: getattr(self, attr) for attr in self._config} + return config + + def build(self): raise NotImplementedError('This must be implemented in subclasses') From 74cda790a11d6a1a8f81adb34eaeb7b909facd97 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 3 Aug 2018 13:52:10 +0300 Subject: [PATCH 08/13] fix attribute naming bug in mp model --- polara/recommender/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polara/recommender/models.py b/polara/recommender/models.py index 1a0a9d5..370ec98 100644 --- a/polara/recommender/models.py +++ b/polara/recommender/models.py @@ -607,7 +607,7 @@ def build(self): item_groups = self.data.training.groupby(itemid, sort=True) if self.by_feedback_value: feedback = self.data.fields.feedback - self.items_scores = item_groups[feedback].sum().values + self.item_scores = item_groups[feedback].sum().values else: self.item_scores = item_groups.size().values From b7ad6b0042ced73ca60c1fde6a55be81cb88e6fa Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Sat, 11 Aug 2018 08:00:37 +0300 Subject: [PATCH 09/13] fix _prediction_key attribute naming --- polara/recommender/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/polara/recommender/models.py b/polara/recommender/models.py index 370ec98..fde8c35 100644 --- a/polara/recommender/models.py +++ b/polara/recommender/models.py @@ -86,8 +86,8 @@ def __init__(self, recommender_data, feedback_threshold=None): self.verify_integrity = get_default('verify_integrity') self.max_test_workers = get_default('max_test_workers') - # TODO sorting in data must be by self._predition_key, also need to change get_test_data - self._predition_key = self.data.fields.userid + # TODO sorting in data must be by self._prediction_key, also need to change get_test_data + self._prediction_key = self.data.fields.userid self._prediction_target = self.data.fields.itemid self._is_ready = False @@ -220,7 +220,7 @@ def _get_test_data(self, feedback_threshold=None): user_idx, item_idx, feedback = self.data.test_to_coo(tensor_mode=tensor_mode, feedback_threshold=threshold) idx_diff = np.diff(user_idx) - # TODO sorting by self._predition_key + # TODO sorting by self._prediction_key assert (idx_diff >= 0).all() # calculations assume testset is sorted by users! # TODO only required when testset consists of known users @@ -411,7 +411,7 @@ def evaluate(self, method='hits', topk=None, not_rated_penalty=None, on_feedback feedback = None if ignore_feedback else feedback scoring_data = assemble_scoring_matrices(recommendations, eval_data, - self._predition_key, self._prediction_target, + self._prediction_key, self._prediction_target, is_positive, feedback=feedback) if method == 'relevance': # no need for feedback From 84fcdd5844bb2d138ca61133990a3bc44d2439da Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 17 Aug 2018 20:09:29 +0300 Subject: [PATCH 10/13] allow to define which index values to use for test data alignment critical for reindexing test data in custom scenarios --- polara/recommender/data.py | 61 +++++++------------------------------- 1 file changed, 11 insertions(+), 50 deletions(-) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index e4a8234..cc00330 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -495,17 +495,17 @@ def _try_reindex_training_data(self): self._reindex_train_items() self._reindex_feedback() - def _try_drop_unseen_test_items(self): + def _try_drop_unseen_test_items(self, mapping='old'): if self.ensure_consistency: itemid = self.fields.itemid - self._filter_unseen_entity(itemid, self._test.testset, 'testset') - self._filter_unseen_entity(itemid, self._test.holdout, 'holdout') + self._filter_unseen_entity(itemid, self._test.testset, 'testset', mapping) + self._filter_unseen_entity(itemid, self._test.holdout, 'holdout', mapping) - def _try_drop_unseen_test_users(self): + def _try_drop_unseen_test_users(self, mapping='old'): if self.ensure_consistency and not self._warm_start: # even in state 3 there could be unseen users userid = self.fields.userid - self._filter_unseen_entity(userid, self._test.holdout, 'holdout') + self._filter_unseen_entity(userid, self._test.holdout, 'holdout', mapping) def _try_drop_invalid_test_users(self): if self.holdout_size >= 1: @@ -613,7 +613,7 @@ def _map_entity(self, entity, dataset): entity_index_map = seen_entities_index.set_index('old').new dataset.loc[:, entity] = dataset.loc[:, entity].map(entity_index_map) - def _filter_unseen_entity(self, entity, dataset, label): + def _filter_unseen_entity(self, entity, dataset, label, mapping): if dataset is None: return @@ -625,9 +625,9 @@ def _filter_unseen_entity(self, entity, dataset, label): raise NotImplementedError try: - seen_entities = index_data.training['old'] + seen_entities = index_data.training[mapping] except AttributeError: - seen_entities = index_data['old'] + seen_entities = index_data[mapping] seen_data = dataset[entity].isin(seen_entities) if not seen_data.all(): @@ -883,54 +883,15 @@ def set_test_data(self, testset=None, holdout=None, warm_start=False, test_users if (testset is None) and (holdout is None): return # allows to cleanup data if ensure_consistency: # allows to disable self.ensure_consistency without actually changing it - self._try_drop_unseen_test_items() # unseen = not present in training data - self._try_drop_unseen_test_users() # unseen = not present in training data + index_mapping = 'old' if reindex else 'new' + self._try_drop_unseen_test_items(mapping=index_mapping) # unseen = not present in training data + self._try_drop_unseen_test_users(mapping=index_mapping) # unseen = not present in training data self._try_drop_invalid_test_users() # inconsistent between testset and holdout if reindex: self._try_reindex_test_data() # either assign known index, or reindex (if warm_start) self._try_sort_test_data() - -class BinaryDataMixin(object): - def __init__(self, *args, **kwargs): - raise NotImplementedError - self.binary_threshold = kwargs.pop('binary_threshold', None) - super(BinaryDataMixin, self).__init__(*args, **kwargs) - - def _binarize(self, data, return_filtered_users=False): - feedback = self.fields.feedback - data = data[data[feedback] >= self.binary_threshold].copy() - data[feedback] = np.ones_like(data[feedback]) - return data - - def _split_test_data(self): - super(BinaryDataMixin, self)._split_test_data() - if self.binary_threshold is not None: - self._training = self._binarize(self._training) - - def _split_eval_data(self): - super(BinaryDataMixin, self)._split_eval_data() - if self.binary_threshold is not None: - userid = self.fields.userid - testset = self._binarize(self.test.testset) - test_users = testset[userid].unique() - user_sel = self.test.holdout[userid].isin(test_users) - holdout = self.test.holdout[user_sel].copy() - self._test = namedtuple('TestData', 'testset holdout')._make([testset, holdout]) - if len(test_users) != (testset[userid].max()+1): - # remove gaps in test user indices - self._update_test_user_index() - - def _update_test_user_index(self): - testset, holdout = self._test - userid = self.fields.userid - new_test_idx = self.reindex(testset, userid, sort=False, inplace=True) - holdout.loc[:, userid] = holdout[userid].map(new_test_idx.set_index('old').new) - new_test_idx.old = new_test_idx.old.map(self.index.userid.test.set_index('new').old) - self.index = self.index._replace(userid=self.index.userid._replace(test=new_test_idx)) - - class LongTailMixin(object): def __init__(self, *args, **kwargs): raise NotImplementedError From e112f76270609e066fd6668735427f9771ecf878 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Fri, 17 Aug 2018 20:10:54 +0300 Subject: [PATCH 11/13] improve status message for holdout filtering --- polara/recommender/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polara/recommender/data.py b/polara/recommender/data.py index cc00330..98791f2 100644 --- a/polara/recommender/data.py +++ b/polara/recommender/data.py @@ -549,7 +549,7 @@ def _filter_short_sessions(self, group_id=None): invalid_session_index = invalid_sessions.index[invalid_sessions] holdout.query('{} not in @invalid_session_index'.format(group_id), inplace=True) if self.verbose: - msg = '{} of {} {}\'s were filtered out from holdout. Reason: not enough items.' + msg = '{} of {} {}\'s were filtered out from holdout. Reason: incompatible number of items.' print(msg.format(n_invalid_sessions, len(invalid_sessions), group_id)) def _align_test_users(self): From d760d595cdfe9a0ade9c46d1233a11865c75bfde Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Tue, 21 Aug 2018 14:18:35 +0300 Subject: [PATCH 12/13] new example of Polara usage based on the EigenRec paper --- examples/Reproducing EIGENREC results.ipynb | 1118 +++++++++++++++++++ 1 file changed, 1118 insertions(+) create mode 100644 examples/Reproducing EIGENREC results.ipynb diff --git a/examples/Reproducing EIGENREC results.ipynb b/examples/Reproducing EIGENREC results.ipynb new file mode 100644 index 0000000..e0a96fa --- /dev/null +++ b/examples/Reproducing EIGENREC results.ipynb @@ -0,0 +1,1118 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The main goal of this tutorial is to demonstrate how one can use Polara to conduct custom experiments with very specific requirements. Needless to say, it can be useful for reproducing someone's research as well. Below you'll find a particular example based on one of my favorite papers called *\"[EIGENREC: generalizing PureSVD for effective and efficient top-N recommendations](https://arxiv.org/abs/1511.06033)\"*, which helped me to see standard SVD-based models in a differrent light and even led me to [my own discoveries](https://arxiv.org/abs/1807.10634). Even though it's not necessary for understanding the material below, I strongly recommend to read the original paper as it builds on top of clear ideas and contains a very thorough analysis.\n", + "\n", + "The key take home message from this paper for me personally is that **SVD can be viewed as a particular case of a more general eigendecomposition problem of the scaled similarity matrix**. Based on that insight, the authors of the *EigenRec* model propose several modifications, which involve tuning the scaling factor as well as the similarity measure in order to improve the model's performance.\n", + "\n", + "
In this tutorial we are not going to reproduce the full work and will focus only on some of its easy-to-implement parts. Basically, we will alter only the scaling factor to see how it affects the quality of recommendations.
\n", + "\n", + "Nevertheless, the tutorial allows to verify validity of the proposed ideas and creates a convenient playground for further exploration." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data preparation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Getting Movielens-1M data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As in the previous tutorials, let's download the Movielens dataset. The task should be already familiar to you. This is one of the datasets used in the paper as well. One could use some other datasets, it wouldn't change anything in later parts of the tutorial. The main requirement is to have it in the form of a Pandas dataframe, similarly to what is returned by the `get_movielens_data` function." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from polara import RecommenderData\n", + "from polara import get_movielens_data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
useridmovieidrating
0111935
116613
219143
3134084
4123555
\n", + "
" + ], + "text/plain": [ + " userid movieid rating\n", + "0 1 1193 5\n", + "1 1 661 3\n", + "2 1 914 3\n", + "3 1 3408 4\n", + "4 1 2355 5" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data = get_movielens_data() # will automatically download it\n", + " # alternatively you can specify a path to the local copy as an argument to the function\n", + "data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "data_model = RecommenderData(data, 'userid', 'movieid', 'rating', seed=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Custom experimental setup with item sampling" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The *EigenRec* paper follows a specific experimentation setup, mainly based on the settings, proposed in my another favorite paper [Performance of recommender algorithms on top-n recommendation tasks](https://dl.acm.org/citation.cfm?id=1864708.1864721), devoted to the *PureSVD* model itself. For evaluation purposes, the authors sample 1.4% of all available ratings and additionally shrink the resulting sample by leaving 5-star ratings only. Quote from the paper (Section 4.2.1): \n", + "
\"...we form a probeset $\\mathcal{P}$ by randomly sampling 1.4% of the ratings of the dataset, and we use each item $v_j$,rated with 5-star by user $u_i$ in $\\mathcal{P}$ to create the test set $\\mathcal{T}$...\"
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This setup can be easily implemented in Polara with the help of `test_ratio` and `holdout_size` parameters of the `RecommendeData` instance. **It requires a two-step preparation procedure.**\n", + "\n", + "**The first step** is to sample data without filtering top-rated items. The following configuration does the thing:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Preparing data...\n", + "2 unique movieid's within 2 holdout interactions were filtered. Reason: not in the training data.\n", + "Done.\n", + "There are 986206 events in the training and 14001 events in the holdout.\n" + ] + } + ], + "source": [ + "data_model.test_ratio = 0 # do not split dataset into folds, use entire dataset for sampling\n", + "data_model.holdout_size = 0.014 # sample this fraction of ratings from data\n", + "data_model.random_holdout = True # sample ratings randomly (not just 5-star)\n", + "data_model.warm_start = False # allow test users to be part of the training (excluding holdout items)\n", + "\n", + "data_model.prepare() # perform sampling" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Mind the `test_ratio` parameter setting. Together with the `test_fold` parameter it controls, which fraction of the dataset to sample from; 0 means the whole dataset and turns off data splitting mechanism used by Polara for cross-validation. The value of `test_fold` has no effect in that case. Also note that by default Polara performs some additional manipulations with data like cleaning and reindexing to transform it into a uniform internal representation for further use. Key actions and their results are reported in an output text, which can be turned off by setting `data_model.verbose = False`. Here's how to see the final result of sampling:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
useridmovieidrating
19025904
1409695
1331922
61110303
231210585
\n", + "
" + ], + "text/plain": [ + " userid movieid rating\n", + "19 0 2590 4\n", + "14 0 969 5\n", + "133 1 92 2\n", + "61 1 1030 3\n", + "231 2 1058 5" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_model.test.holdout.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**The second step** is to leave only items with rating 5, as it was done in the original paper. The easiest way in our case would be to simply run:\n", + "```python\n", + "data_model.test.holdout.query('rating==5', inplace=True)\n", + "```\n", + "However, in general, you shouldn't manually change the data after it was processed by Polara, as it may break some internal logic. A more appropriate and a safier way to achieve the same is to use the `set_test_data` method, specifically designed to cover custom configurations: " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "data_model.set_test_data(holdout=data_model.test.holdout.query('rating==5'), # select only 5-star ratings\n", + " warm_start=data_model.warm_start, \n", + " reindex=False, # avoid reindexing users and items second time\n", + " ensure_consistency=False # do not try to filter out unseen entities (they are already excluded)\n", + " # leaving it as True wouldn't change the result but would lead to extra checks\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that we reuse the previously sampled holdout dataset (the $\\mathcal{P}$ dataset in the authors' notation), which is already reindexed by Polara's built-in data pre-processing procedure. In order not to loose the index mapping between internal and external representation of movies and users (stored in the `data_model.index` attribute) it's very important to set `reindex` argument of the `set_test_data` method to `False`. Now the `data_model.test.holdout` dataframe stores the final result, namely the $\\mathcal{T}$ dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
useridmovieidrating
1409695
231210585
326423725
488533015
931911985
\n", + "
" + ], + "text/plain": [ + " userid movieid rating\n", + "14 0 969 5\n", + "231 2 1058 5\n", + "326 4 2372 5\n", + "488 5 3301 5\n", + "931 9 1198 5" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_model.test.holdout.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Scaled SVD-based model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the simplest case of the EigenRec model, when only the scaling factor is changed, we can go with a very straightforward approach. Instead of computing similarity matrices and solving an eigendecomposition problem, it is sufficient to apply standard SVD to a scaled rating matrix $\\tilde R$: \n", + "\n", + "$$\n", + "\\tilde R = R \\, S^{d-1} \\approx U\\Sigma V^T,\n", + "$$ \n", + "\n", + "where $R$ is an $M \\times N$ rating matrix, $S = \\text{diag}\\{\\|r_1\\|_2, \\dots, \\|r_N\\|_2\\}^d$ is a diagonal scaling matrix with its non-zero values depending on a scaling parameter $d$ and $r_i$ denotes an $i$-th column of $R$. Note that due to the orthogonality of columns in the SVD factors the approximation of $\\tilde R$ can be written in an equivalent and more convenient form $\\tilde RVV^T$, which can be used to generate recommendations." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Scaling input data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to calculate the scaled version of the *PureSVD* approach we can reuse the `SVDModel` class implemented in Polara. One of the ways to do that is to redefine the `build` method in an `SVDModel`'s subclass. A simpler solution, however, is to directly modify an output of the `get_training_matrix` method, which is generally available for all models in Polara and is used internally in the `SVDModel` in particular. This method returns the rating matrix in a sparse format, which is then fed into the `scipy`'s truncated SVD implementation within the `build` method (you can run the `SVDModel.build??` command with double question mark to see it). Assuming we already have sparse rating matrix, the following function will help to scale it:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from scipy.sparse import diags\n", + "from scipy.sparse.linalg import norm as spnorm\n", + "\n", + "def sparse_normalize(matrix, scaling, axis):\n", + " '''Function to scale either rows or columns of the sparse rating matrix'''\n", + " if scaling == 1: # no scaling (standard SVD case)\n", + " return matrix\n", + " \n", + " norm = spnorm(matrix, axis=axis, ord=2) # compute Euclidean norm of rows or columns\n", + " scaling_matrix = diags(np.power(norm, scaling-1, where=norm!=0))\n", + " \n", + " if axis == 0: # scale columns\n", + " return matrix.dot(scaling_matrix)\n", + " if axis == 1: # scale rows\n", + " return scaling_matrix.dot(matrix)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Sampling random items for evaluation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Somewhat more involved modifications are required to generate model predictions, as it's based on an additional sampling of items not previously seen by the test users. Quote from the paper (Section 4.2.1): \n", + "
\"For each item in $\\mathcal{T}$, we randomly select another 1000 unrated items of the same user...\"
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This means that we need to generate prediction scores for 1000 randomly selected unseen items in addition to every item from the holdout. Moreover, **every set of 1001 items is treated independently of the user it belongs to**. Normally, Polara performs evaluation on a *per user basis*; however, in this case the logic is different and we have to take care of users with mulltiple items in the holdout. From the line below it can be clearly seen that some test users can have up to 8 items:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "8" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_model.test.holdout.userid.value_counts().max()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to \"flatten\" the holdout dataset and to independently generate prediction scores for every holdout item (and 1000 of additionally sampled items) we will customize the `get_recommendations` method of the `SVDModel` class. Below is the support function, that helps to achieve the necessary result. It iterates over all holdout items, randomly samples a predefined amount of previously unrated items and generates prediction scores for them:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "def sample_scores_flat(useridx, itemidx, seen_data, all_items, user_factors, item_factors, sample_size=1000, random_state=None):\n", + " '''Function to randomly sample unrated items and generate prediction scores for them.'''\n", + " scores = []\n", + " for user, items in itemidx.groupby(useridx): # iterate over every test user and get all user items\n", + " seen_items = seen_data[1][seen_data[0]==user].tolist() # list of the previously rated items of the user\n", + " seen_items.extend(items.tolist()) # take holdout items into account as well\n", + " item_pool = all_items[~all_items.isin(seen_items)] # exclude seen items from all available items\n", + " for item in items:\n", + " sampled_items = item_pool.sample(n=sample_size, random_state=random_state) \n", + " scores.append(item_factors[sampled_items.values, :].dot(user_factors[user, :]))\n", + " return scores" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "
Prediction scores are generated similarly to the standard *PureSVD* model by an orthogonal projection of a vector $r$ of user ratings onto the latent feature space, defined by the formula $VV^Tr$. Note that unlike the model computation phase, no scaling is used in the prediction.
\n", + "\n", + "The code above complies with this definition by expecting `user_factors` to be the product $V^Tr$ for a set of test users and `item_factors` to be $V$ itself. Below you can find a full implementation of our new model." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Defining the model" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "from polara import SVDModel\n", + "\n", + "class ScaledSVD(SVDModel):\n", + " '''Class that adds scaling functionality to the PureSVD model'''\n", + " \n", + " def __init__(self, *args, **kwargs):\n", + " super().__init__(*args, **kwargs)\n", + " self.col_scaling = 1 # scaling parameted d, initially corresponds to PureSVD\n", + " self.n_rnd_items = 1000 # number of randomly sampled items\n", + " self.seed = 0 # to control randomization\n", + " self.method = 'ScaledSVD'\n", + " \n", + " def get_training_matrix(self, *args, **kwargs):\n", + " svd_matrix = super().get_training_matrix(*args, **kwargs) # get sparse rating matrix\n", + " return sparse_normalize(svd_matrix, self.col_scaling, 0)\n", + " \n", + " def get_recommendations(self):\n", + " holdout = self.data.test.holdout\n", + " itemid = self.data.fields.itemid # \"movieid\" in the case of Movielense dataset\n", + " userid = self.data.fields.userid # \"userid\" in the case of Movielense dataset\n", + " \n", + " itemidx = holdout[itemid] # holdout items of the test users\n", + " useridx = pd.factorize(holdout[userid])[0] # have to \"rebase\" user index;\n", + " # necessary for indexing rows of the matrix with test user ratings\n", + " \n", + " # prediction scores for holdout items\n", + " test_matrix, seen_data = self.get_test_matrix() \n", + " item_factors = self.factors[itemid] # right singular vectors, matrix V\n", + " user_factors = test_matrix.dot(item_factors) # according to product V^T r for every test user\n", + " holdout_scores = (user_factors[useridx, :] * item_factors[itemidx.values, :]).sum(axis=1).squeeze()\n", + " \n", + " # scores for randomly sampled unseen items\n", + " all_items = self.data.index.itemid.new # all unique (reindexed) items\n", + " rs = np.random.RandomState(self.seed) # fixing random state to control random output\n", + " sampled_scores = sample_scores_flat(useridx, itemidx,\n", + " seen_data, all_items,\n", + " user_factors, item_factors,\n", + " self.n_rnd_items, random_state=rs)\n", + " \n", + " # combine all scores and rank selected items \n", + " scores = np.concatenate((holdout_scores[:, None], sampled_scores), axis=1) # stack into array with 1001 columns\n", + " rankings = np.apply_along_axis(np.argsort, 1, -scores)\n", + " return rankings" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The model is ready and can be used in a standard way:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ScaledSVD training time: 0.5471807377243749s\n" + ] + } + ], + "source": [ + "svd = ScaledSVD(data_model) # create model\n", + "svd.rank = 50\n", + "svd.col_scaling = 0.5\n", + "svd.build() # fit model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, when we have our model computed, its time to evaluate it. However, **we cannot use the built-in evaluation routine**. Normally, the number of test users is equal to the number of rows in recommendations array and that's the logic Polara relies on. In our case the number of test users is lower than the number of rows in recommendations array and actually corresponds to the total number of ratings in the holdout:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# of test users: 2009\n", + "# of rows and columns in recommendations array: (3140, 1001)\n", + "# of ratinhgs in the holdout: 3140\n" + ] + } + ], + "source": [ + "# if you run the cell for the first time you'll notice a short delay before print output due to calculation of recommendations\n", + "print('# of test users:', data_model.test.holdout.userid.nunique())\n", + "print('# of rows and columns in recommendations array:', svd.recommendations.shape)\n", + "print('# of ratinhgs in the holdout:', data_model.test.holdout.shape[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will fix this inconsistency in the next section." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Worth noting here that Polara implements a unified system of callbacks, which reset the `svd.recommendations` property whenever either the `data_model` or the model itself are changed in a way that affects the models' output (try, for example, call `svd.recommendations`, then set the rank of the model to some higher value and call `svd.recommendations` again). This mechanism helps to ensure predictable and consistent state and to prevent accidental reuse of the cached results during experiments. \n", + "It can also be extended with user-defined triggers, which is probably the topic for another tutorial." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model evaluation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Simple approach" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When you try to evaluate your model, it calls for the `model.recommendations` property which is automatically filled with the result of the `get_recommendations` method. The simplest way to evaluate the result in accordance with the new structure of the recommendations array is to define a small function as shown below:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def evaluate_mrr(model):\n", + " '''Function to calculate MRR score.'''\n", + " is_holdout = model.recommendations==0 # holdout items are always in the first column before sorting\n", + " pos = np.where(is_holdout)[1] + 1.0 # position of holdout items (indexing starts from 0, so adding 1) \n", + " mrr = np.reciprocal(pos).mean() # mean reciprocal rank\n", + " return mrr" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, to compute the MRR score, as it is done in the original paper, simply run:" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.3130822345824591" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "evaluate_mrr(svd)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### More functional approach" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "While the previously described approach is fully working and easy, in some cases you may want to use the built-in `model.evaluate` method, as it provides additional functionality. It is also useful to see how Polara can be customized to serve specific needs. The key ingredient here is the control of the type of entities that are recommended. By default, Polara expects items to be recommended to users and looks for the corresponding fields in the test data. These fields are defined via `data_model.fields.userid` and `data_model.fields.itemid` attributes respectively. The default behavior, however, can be redefined at the model level be setting `model._prediction_key` (users by default) and `model._prediction_target` (items by default) attributes to custom values. This scheme, for example, can be utilized in cold start experiments, where the task is to find users potentially interested in a \"cold\" item instead of recommending items to users (see `polara.recommender.coldstart` for implementation details). The following lines show how to change the default settings for our needs:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "svd._prediction_key = 'xuser'\n", + "svd._prediction_target = 'xitem'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, we need to specify the corresponding fields in the holdout data. Recall that our goal is to treat every item in the holdout independently of the user or, in other words, to assign every item to a unique \"virtual\" user (`'xuser'`). Furthermore, by construction, prediction scores for holdout items are located in the first column of the recommendations array. This means that every holdout item (`'xitem'`) should have index 0. Here's the necessary modification:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "data_model.test.holdout['xuser'] = np.arange(data_model.test.holdout.shape[0]) # number of rated items defines the range\n", + "data_model.test.holdout['xitem'] = 0" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's check that the result is the same (up to a small rounding error due to different calculation schemes):" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Ranking(mrr=0.31308223458245904)" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "svd.evaluate('ranking', simple_rates=True) # `simple_rates` is used to enforce calculation of MRR" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
If you'll do the math you'll see that the whole experiment took under 100 lines of code to program, and the most part of it was pretty standard (i.e., declaring variables and methods).
\n", + "\n", + "Less lines of code typically means less risks for having bugs or inconsistencies. By following a certain protocol, Polara provides a high-level interface that abstracts many technical aspects allowing to focus on the most important parts of research." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Reproducing the results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The next task is to repeat experiments from the EigenRec paper, where the authors compute\n", + "
\"...MRR scores as a function of the parameter $d$ for every case, using the number of latent factors that produces the best possible performance for each matrix.\"
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Grid search" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The beauty of SVD-based models is that it is much easier to perform grid-search for finding optimal values of hyper-parameters. Once you have computed a model for a certain set of hyper parameters with some rank value $k$, you can quickly find all other models of rank \"k' < k\" without recomputing SVD.\n", + "
Going from larger values of rank to smaller ones is performed by a simple truncation of the latent factor matrix.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This not only allows to perform experiments faster, but also simplifies the code for it. Moreover, `SVDModel` already has the necessary rank-check procedures, which allow to avoid rebuilding the model when user sets a smaller value of rank. No special actions are required here. Below is the code that implements the grid search experiment, taking that feature into account (note that on a moderate hardware the code will run for approximately half an hour):" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm import tqdm_notebook\n", + "%matplotlib inline\n", + "\n", + "svd_mrr_flat = {} # will stor results here\n", + "svd.verbose = False\n", + "\n", + "max_rank = 150\n", + "scaling_params = np.arange(-20, 21, 2) / 10 # values of d from -2 to 2 with step 0.2\n", + "svd_ranks = range(10, max_rank+1, 10) # ranks from 10 to max_ranks with step 10\n", + "\n", + "for scaling in tqdm_notebook(scaling_params):\n", + " svd.col_scaling = scaling\n", + " svd.rank = max_rank\n", + " svd.build()\n", + " \n", + " for rank in list(reversed(svd_ranks)): # iterating over rank values in a descending order\n", + " svd.rank = rank # allows to truncate factor matrices without recomputing SVD\n", + " svd_mrr_flat[(scaling, rank)] = svd.evaluate('ranking', simple_rates=True).mrr" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we have the results of the grid search stored in the `svd_mrr_flat` dictionary. There's one catch that wasn't clear for me at first:\n", + "
in order to show the effect of parameter $d$ the authors have fixed the value of rank corresponding to the best result achieved with EigenRec.
\n", + "\n", + "This means that the curve on Figure 1 in the original paper is obtained with a fixed value of rank, corresponding to the optimal point at the top of the curve, and all other points are obtained by only changing the scaling factor. Here's one way to draw it:" + ] + }, + { + "cell_type": "code", + "execution_count": 167, + "metadata": {}, + "outputs": [], + "source": [ + "result_flat = pd.Series(svd_mrr_flat)\n", + "best_d, best_rank = result_flat.idxmax()\n", + "best_d, best_rank" + ] + }, + { + "cell_type": "code", + "execution_count": 199, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAARgAAACcCAYAAACk2LT2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAHDlJREFUeJzt3Xl8VfWd//HXJztkI4SwBkhACBAWgYDgBigiVgEr4oZaK8pIdX52rOMyrcV2xplabR0toGhVRFFaxLZo5YcLiVQFgciiEMIStrBlgaxkz2f+SKAQEnKBe3Nu7v08H488vOeeb875nHh537N9v0dUFWOM8YQApwswxvguCxhjjMdYwBhjPMYCxhjjMRYwxhiPsYAxxniMBYwxxmMsYIxLRGSPiFSKSIcG728UERWRBBFZUN+mRESOisinItLvlLb3iEhN/fwiEdkkIje0/NaYlmIBY87FbuD2ExMiMgho06DNb1U1AugGHABebzB/df38dsA8YLGItPNcycZJFjDmXLwN3H3K9I+AhY01VNUy4M/AxU3Mr61fXjjQx71lGm9hAWPOxRogSkT6i0ggcCvwTmMNRSScur2dnU3MDwR+DFQBez1TrnFakNMFmFbnxF7MF8A26g6DTvWoiDwERFEXHFMazB8lIgXU7blUA3eqao5nSzZOsT0Yc67eBu4A7qHxw6PnVbUdkACUAUkN5q+pnx8DLAOu8FilxnEWMOacqOpe6k72/gD44Czt9gEPAy+KSMMTwahqCfAT4C4RGeqhco3DLGDM+ZgBXKWqpWdrpKqfAgeBmU3Mzwf+CPzS7RUar2ABY86Zqu5S1fUuNn8OeExEQpuY/7/AD0RksHuqM95EbMApY4yn2B6MMcZjLGCMMR5jAWOM8RgLGGOMx1jAGGM8xrGuAu3atdOLLrrIqdW3uNLSUsLDw50uo8XY9vq29PT0PFWNa66dYwHTqVMn1q939VaK1i8tLY2xY8c6XUaLse31bSLiUgdVO0QyxniMBYwxxmNsuAbTapRX1ZBxqIjvDhQSFBDAVf060jk6zOmyzFl4VcBUVVWRnZ1NeXm506W4XXR0NBkZGS2+3rCwMOLj4wkODm7xdV+I6ppaduSUsDm7gE3ZhWzOLiDzcDFVNad3bRnULZrx/TtxzYBO9O8SiYg4VLFpjFcFTHZ2NpGRkSQkJPjcB6W4uJjIyMgWXaeqkp+fT3Z2NomJiS267nOhquzJP14XJvvrwmTLwSLKqmoAiAwLYnB8NPdd0Ysh8dEMim/H8YpqPs04wmdbj/C/n2/nhc+2061dG8b378j4AZ24JDGWkCA7A+A0rwqY8vJynwwXp4gIsbGx5ObmOl1Ko1SV1MwcfrN8G9uPlAAQFhxActdobhvZnSHx7RgcH01CbDgBAWd+Jvp0iuQnYy8it7iC1G05fJpxhD+t389bq/cSGRrEmKQ4rhnQibF9OxLdtnXtwfkKrwoYwMLFzbz17/n9gUKe+XsGq7PySYhty3/dOJBhPWLo2ymCoMBz2/OIiwzllhHduWVEd8qravhyRx6fZRzhs4wcPtp8iKAAYeqweP77pkEENhJUxnNcChgRmQi8CAQCf1TV3zSY/wDwIFADlAAzVXWrm2ttES+99BIvv/wyw4YN49Zbb2Xr1q088cQTF7zcLl26UFJS4oYKT/f0008TERHBo48+6vZle0L2seP87pPt/GXDAdqHh/CrycnccUkPgs8xVJoSFhzI+AGdGD+gE7W1yqbsAj749gBvr9lLYKDwzI0DvTZ0fVGzAVM/+vtc4BogG1gnIssaBMi7qvpKffvJwO+BiR6o1+PmzZvH8uXLT56zmDx5cousV1VRVQICfPO8QWFZFfPSdvLmV3sQYNbY3swa25uoMM8dugQECEN7xDC0RwwRYUG8nLaLDhGhPHJNX4+t05zOlU/zSGCnqmapaiWwmAYjxatq0SmT4UCrHMXqgQceICsri8mTJ/PCCy+wYMECHnroIQCmTJnCwoV1Y1zPnz+f6dOnA7Br1y4mTpzI8OHDueKKK9i2bRsAu3fvZvTo0YwYMYKnnnqq0fXt2bOH/v3785Of/IRhw4axf/9+Zs2aRUpKCsnJycyePftk24SEBGbPns2wYcMYNGjQyfWc6rXXXuO6666jrKzMrX+XC1FZXcsbX+5m7HOpvLoqixsGdyH10bE8PrGfR8OloceuTeKWlHhe+nwHb6/e02Lr9Xsnvjmb+gFupu6w6MT0XcCcRto9COwC9gN9mltu3759taGtW7ee8V5L69mzp+bm5qqq6ptvvqkPPvigqqoePnxYe/furatWrdI+ffpofn6+qqpeddVVun37dlVVXbNmjY4bN05VVSdNmqRvvfWWqqrOmTNHw8PDz1jX7t27VUR09erVJ987sdzq6modM2aMbtq06WRdL730kqqqzp07V2fMmKGqqrNnz9bnnntO//CHP+ikSZO0vLz8jPU48XdduXKlfrTpoF7525Xa8/GP9I7XVut32QUtXsepqqprdMaCdZrwxEf64aYDbl12amqqW5fn7YD12sy/cVV16RxMYwesZ+yhqOpcYK6I3AH8grqn/p2+IJGZ1A8AHRcXR1pa2mnzo6OjKS4uBuDZT3ax7Yh7z1n06xTB4xN6n7WNqlJSUkJoaCjl5eVUVlZSXFxM27ZtefLJJxk3bhzvvvsuwcHBHDp0iK+//pqpU6ee/P2KigqKi4v58ssvWbBgAcXFxdx44408/vjjJ7fthJKSEnr06EFycvLJeQsXLmTBggVUV1dz+PBh0tPTSUxMRFWZMGECxcXF9OvXjyVLllBcXExFRQVLly6la9euvPfee1RWVlJZWXnaesrLy8/4W3vS3qIaFnxXxu7ib4mPEB4ZHsqgDmXk7dhA2o4WK6NRt8Qr+w4H8PB7G9i3I4MBsYFuWW5JSUmL/o1bC1cCJhvofsp0PHUjxTdlMfByYzNU9VXgVYCkpCRt2DksIyPj5L0iwSHBBAa653/+CcEhwc3eiyIiREREEBkZSVhYGCEhISd/Z+fOncTGxnLs2DEiIyNRVdq1a8fmzZsbXU5UVBRBQUEn9vDOWHdERMTJdUHdYdWcOXNYt24dMTEx3HPPPYgIkZGRJy85R0ZGEhUVhaoSGRlJaGgogwcPZuPGjRQWFjZ6v0tYWBhDh7bMk0F255Xy8NyvoDaAZ6cO5Obh3b3uys0lo6u4Zf5q5m46zuKZKQyKj77gZfpbZ0dXuRIw64A+IpJI3VP8bqPuwVsniUgfVT3x3XQ9cMHfU7MnJV/oItxq7dq1LF++nA0bNjBmzBgmTJhAYmIiiYmJLFmyhGnTpqGqbN68mSFDhnDZZZexePFi7rzzThYtWuTSOoqKiggPDyc6OpojR46wfPlylz60Q4cOZdasWUyePJkVK1bQtWvXC9za81NUXsX9C9cTIPDkJWHcMqKHI3U0J7ptMAtnjOSmeV9zz5treX/WpSR28J+hFlpSsyd5VbUaeAhYAWQAf1bVLSLy6/orRgAPicgWEdkIPEIjh0etWUVFBffffz9vvPEGXbt25Xe/+x333nsvqsqiRYt4/fXXGTJkCMnJyfztb38D4MUXX2Tu3LmMGDGCwsJCl9YzZMgQhg4dSnJyMvfeey+XXXaZyzVefvnlPP/881x//fXk5eWd13ZeiJpa5eH3NrAnr5R504fTsa13Xw3rFBXG2zNGosBdr39DTpHvdU/xCq6cqPHEj7ee5PWUoqIix9bdEn/X//77Vu35+Ef69uo9qtp6Tnpu3HdM+z+1XK994QstOF553stpLdvrLrh4kte7v2ZMq7A0PZv5q7K4c1QP7hzV0+lyzsmQ7u2Yf9dwduWWcP/C9ZTX938y7mEBYy7It/uO8eQH3zG6V6zXnTdz1RV94nh+2hDW7j7K/3tvA9U1tU6X5DMsYMx5O1RYxr+8nU7n6DDmTR/mttv9nTDl4m7MnjSAT7Ye4Rd//f7klT9zYbyus6OqWl8RN/LUP5SyyhpmLkzneEU1i+67hJjwEI+spyX9+LJE8koqmJta16Xg0WuTnC6p1fOqgAkLCyM/P5/Y2FgLGTfQ+vFgwsLcO+qbqvLY0s18f7CQ1+5KoW+nlh3nxpMenZBEfkklc1J3EhMewozLvXccndbAqwImPj6e7Oxsrx2/5EKUl5e7/R+6K06MaOdO89J28eGmgzw2MYnxAzq5ddlOExGe+eEgCo5X8Z8fbaVdm2CmDnfv38+feFXABAcHe/XIaxciLS2txe6m9aRPthzmuRWZ3HhxV2aNOXu3i9YqMEB48faLuXfBOh5bupmoNsFc42NB2lJa71k50+K2HS7ip3/ayJD4aH4zdbBPH8aGBgUy/64UBnaN4sF3v2VNVr7TJbVKFjDGJUdLK7nvrfVEhAYx/64UwoLd20/MG0WEBvHmj0fSo31b7n9rPd8fcO2ObPNPFjCmWZXVtcx6J52c4gpevTvFrx4V0j48hLdnjCSqTTA/emMtWbnuH5XQl1nAmGY9/0km3+w+ynM3D+bi7u2cLqfFdYluw8IZIwG46/W1HCr0ngG9vJ0FjDmr4vIq3lmzl5uGdmPKxd2cLscxveMiWPDjkRSWVXH362s5VlrZ/C8ZCxhzdn/deJDjlTXcfWmC06U4blB8NK/dncLeo8f58YJ1lFZUO12S17OAMU1SVRat2Uty1yiGuGFQJl8wuncsc24fyubsAh54J52KausceTYWMKZJG/YXsO1wMdMv6enTl6TP1YTkzjw7dTD/2JHHI3/aRE2t9VtqilfdaGe8y6I1+wgPCWTyxc6MkOfNpqV0p7Csiv/6ewZRbYKZEGMh0xgLGNOowuNVfLT5IDcPjyci1D4mjbnvil4cLa1kXtoujnQPYsSl1fa3asAOkUyjln6bTUV1LdMvaV0DSLW0f782iRmXJ7JyfzVXPZ/G++nZ1Noh00kWMOYMqsqib/Zycfd2DOga5XQ5Xk1EeOqGATw1Koyu7drw6JJN/HDeV6TvPeZ0aV7BAsac4ZvdR9mVW8r0S7zzqQDeqHe7QD6YdSm/v2UIh4vKmfry1/x08Qa/vynPAsac4d1v9hEZFsQNg+3k7rkICBBuGhbPyp+N5aFxF/Hx94e56vkvePGzHZRV+uflbAsYc5q8kgqWf3+IqcPiaRPi+x0aPSE8NIhHr03i80fGMK5fHC98tp3xv/+CDzcd9LuhOC1gzGneT8+mqkbt8MgNurdvy7zpw1k8cxRRbYL51/c2cMv81X7VK9ulgBGRiSKSKSI7ReSJRuY/IiJbRWSziHwuInbpoRWqrVXeW7uPkQnt6eNDw2A6bVSvWD7618v5n5sGkZVbyqQ5X/LInzay40hx87/cyjUbMCISCMwFrgMGALeLyIAGzTYAKao6GHgf+K27CzWe99WuPPbmH2f6KNt7cbfAAOH2kT1I/fex3H9FLz7+/hDXvLCK+95ax/o9R50uz2Nc2YMZCexU1SxVraTu4fZTTm2gqqmqerx+cg1gg5i2Qu9+s4/24SFMHNjZ6VJ8VlRYMP/xg/58/cTVPHx1H9bvPcbNr6xm2itf83nGEZ+7h8aVgOkG7D9lOrv+vabMAJZfSFGm5R0pKueTrUe4eXg8oUF2ctfT2oeH8G/X9OXrJ65i9qQBHCwoZ8Zb65n44iqWpmdT5SMPf3PlvubGerk1GrMicieQAoxpYv5MYCZAXFwcaWlprlXpA0pKSrx6e5ftqqSmVumlh0hLO3LBy/P27XW3C9neROBXI4W1h0P5OKuUny3ZxDMfbubahGDGxAcRFtR6O5pKc5fNRGQ08LSqXls//SSAqv5Pg3bjgT8AY1Q1p7kVJyUlaWZm5vnW3eqkpaUxduxYp8toVE2tcuVvU0nsEM47913ilmV68/Z6gru2V1VJy8zl5bRdrN1zlOg2wfxodE/uvjSBDhGhF16om4hIuqqmNNfOlT2YdUAfEUkEDgC3AXc0WNlQYD4w0ZVwMd7li+05HCgo4+fX93e6FL8nIozr15Fx/TqSvvcYr3yxi5dW7uTlL3Zxdb9O3DIiniv7xBHUSh7T22zAqGq1iDwErAACgTdUdYuI/BpYr6rLgOeACGBJ/bgh+1R1sgfrNm60aM0+4iJD7dk/XmZ4zxheuzuFnTklLF67j79sOMD/33KYjpGh3DQsnmkp8fSOi3C6zLNyqW+5qn4MfNzgvV+e8nq8m+syLeRAQRmpmTnMGtu7VT+83pdd1DGCX9wwgMcm9mPlthzeT9/Pa//I4pUvdpHSM4ZpKfFcP7irVw4V4X0VmRb1p7X7UOC2EXbvi7cLCQpg4sDOTBzYmZyicj7YcIA/r9/P40u/4+llW7l+cBemDY9nZGJ7rxmB0ALGj1XV1LJ43X7G9o2je/u2TpdjzkHHqDAeGNObf7myF9/uK2DJ+v18uOkg76dnkxDblvH9O9G7YwS9OoSTGBdOXESoI6FjAePHPs/IIae4gmdsUKlWS0QY3jOG4T1j+OWkASz/7jBL0vezcM1eKqv/eS9NZGgQiXHhdYHTIYJeceH0igsnsUM4bUM8FwMWMH5s0Td76RIdxrikOKdLMW7QNiSIqcPjmTo8nppa5WBBGVl5pezOLan7b14p6/Yc468bD572e12iw+jXOZKUhPaMTGzPoG7Rbns0sAWMn9qbX8o/duTxb+P7tppLnsZ1gQFC9/Zt6d6+LWP6nv4FUlZZw578usDJyi0hK7eUzQcKSa2/Ly0kMIDB8dGMSGzPiIQYhvdsT3Sb4POqwwLGT723dj+BAcKtI7o7XYppYW1CAunfJYr+XU4fDvVoaSXr9xxl/d5jrN19lNdWZfFymiICSZ0iSUmIYURCe0YktHd5XRYwfqiiuoYl6/dzdb+OfvUge3N27cNDmJDcmQnJdZ1dyypr2Li/gHV7jrJuz1H+8u0B3lmz75yWaQHjh1ZsOUJ+aSXTR9nJXdO0NiGBjO4dy+jesQBU19Sy7XAxa3cfZcazri3DAsYPLU3PJj6mDVdc1MHpUkwrEhQYwMBu0QzsFs0MF3/Hzu75meOV1azOymdicmcCArzjZizjuyxg/MzqXflUVtcyrl9Hp0sxfsACxs+s3JZDeEggKQkxTpdi/IAFjB85MdbIZRd1sFHrTIuwgPEjO3JKOFBQZodHpsVYwPiR1G11Y4GNta4BpoVYwPiR1Mwc+nWOpEt0G6dLMX7CAsZPFJVXsX7PMTs8Mi3KAsZPfLUjj+paZVySBYxpORYwfiI1M4eosCCG9WjndCnGj1jA+AFVJTUzlyv7tp7R6I1vsE+bH9hysIjc4go7PDItzgLGD6Rl1l2eHmOXp00Ls4DxAyu35TAkPtqrngxo/INLASMiE0UkU0R2isgTjcy/UkS+FZFqEbnZ/WWa83W0tJIN+wsYa4dHxgHNBoyIBAJzgeuAAcDtIjKgQbN9wD3Au+4u0FyYf+zIRRW7/8U4wpUBp0YCO1U1C0BEFgNTgK0nGqjqnvp5tY0twDgndVsOseEhDO4W7XQpxg+5cojUDdh/ynR2/XvGy9XUKl9sz2VM3zgbXMo4wpU9mMY+mXo+KxORmcBMgLi4ONLS0s5nMa1SSUlJi2/vzoIajh2vomNtXouv24ntdZK/ba+rXAmYbODUZ1vEAwebaHtWqvoq8CpAUlKSjh079nwW0yqlpaXR0tv77SeZBMhOHphyJe3ahrToup3YXif52/a6ypVDpHVAHxFJFJEQ4DZgmWfLMu6QmpnL8J4xLR4uxpzQbMCoajXwELACyAD+rKpbROTXIjIZQERGiEg2MA2YLyJbPFm0aV5OcTnfHSi0y9PGUS49tkRVPwY+bvDeL095vY66QyfjJb7IzAWw7gHGUXYnr49Ky8ylU1Qo/btEOl2K8WMWMD6oqqaWVdtzGZfUERG7PG2cYwHjg9L3HqO4otrOvxjHWcD4oNTMHIIDhcsuinW6FOPnLGB8UNq2XEYktCcyLNjpUoyfs4DxMQcKysg8UmxXj4xXsIDxMScGlxrXzwaXMs6zgPExqdtyiY9pQ++4CKdLMcYCxpdUVNfw1c48rupnl6eNd7CA8SFrdx+lrKrGzr8Yr2EB40NSt+USGhTAqF52edp4BwsYH5KWmcPo3rG0CQl0uhRjAAsYn7Enr5SsvFI7PDJexQLGR6SeuDxtAWO8iAWMj0jNzKVXXDg9Yts6XYoxJ1nA+IDjldWsycq3vRfjdSxgfMDqXflUVtdawBivYwHTypVWVLPom320DQlkRGKM0+UYcxqXhsw03umTLYd5etkWDhaW88g1fQkNssvTxrtYwLRCBwrKeHrZFj7deoR+nSP5wx3DGN7T9l6M97GAaUWqa2p586s9vPDZdlThyev6ce/liQQH2pGu8U4WMK3Ehn3H+I+/fE/GoSKu7teRX01JJj7GLkkb72YB4+UKy6p4bsU2Fn2zj06RYbxy5zCuTe5svaVNq+BSwIjIROBFIBD4o6r+psH8UGAhMBzIB25V1T3uLdW/qCofbj7Ef360lfySCu65NIGfTUgiItS+E0zr0eynVUQCgbnANdQ9p3qdiCxT1a2nNJsBHFPVi0TkNuBZ4FZPFOyLqmpqKSqrorD+p+B4FW98tZt/7MhjcHw0b94zgoHdop0u05hz5srX4Uhgp6pmAYjIYmAKcGrATAGern/9PjBHRERVtamFllYpf9t44LyKdkXDNSt69vl6ot2J6X820DNeQK0qNarU1io1tUqNUvda66ZPvK6tf39HVgUf5W46GSJF9T+FZVWUVtacUX9EaBC/mpzMnaN6Ehhgh0OmdXIlYLoB+0+ZzgYuaaqNqlaLSCEQC+Q1tdDcMuXhxRvPrdpWKkAgOADaF+QR3SaYqDbBdG/flqiwYKLbnPgJIrrtP6d7x0XYQ+tNq+dKwDT29dlwz8SVNojITGBm/WTF3mdv+N6F9fuKDpwlcH2Qba9vS3KlkSsBkw10P2U6HjjYRJtsEQkCooGjDRekqq8CrwKIyHpVTXGlSF9g2+vb/HF7XWnnyh1a64A+IpIoIiHAbcCyBm2WAT+qf30zsPJs51+MMf6h2T2Y+nMqDwErqLtM/YaqbhGRXwPrVXUZ8DrwtojspG7P5TZPFm2MaR1cuqlCVT8GPm7w3i9PeV0OTDvHdb96ju1bO9te32bb2wixIxljjKdYLzljjMc4GjAi8pyIbBORzSLyFxFp52Q9niYi00Rki4jUiojPXnEQkYkikikiO0XkCafr8SQReUNEckTEL265EJHuIpIqIhn1n+WHz9be6T2YT4GBqjoY2A486XA9nvY9cBOwyulCPOWUriXXAQOA20VkgLNVedQCYKLTRbSgauBnqtofGAU8eLb/v44GjKp+oqrV9ZNrqLvHxmepaoaqZjpdh4ed7FqiqpXAia4lPklVV9HIPV++SlUPqeq39a+LgQzq7uRvlNN7MKe6F1judBHmgjXWtaTJD6BpvUQkARgKfNNUG4/3/ReRz4DOjcz6uar+rb7Nz6nb9Vrk6Xo8zZXt9XEudRsxrZuIRABLgZ+qalFT7TweMKo6/mzzReRHwA3A1b5w929z2+sHXOlaYloxEQmmLlwWqeoHZ2vr9FWkicDjwGRVPe5kLcZtXOlaYlopqRtK8XUgQ1V/31x7p8/BzAEigU9FZKOIvOJwPR4lIj8UkWxgNPB3EVnhdE3uVn/S/kTXkgzgz6q6xdmqPEdE3gNWA0kiki0iM5yuycMuA+4Crqr/N7tRRH7QVGO7k9cY4zFO78EYY3yYBYwxxmMsYIwxHmMBY4zxGAsYY4zHWMAYYzzGAsYY4zEWMMYYj/k/VvcswVEIIkkAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "result_flat.xs(best_rank, axis=0, level=1).plot(label='fixed rank', legend=True, title='MRR',\n", + " figsize=(4.3, 2), ylim=(0, None), xlim=(-2, 2), grid=True);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Comparing this picture to the bottom left graph of Figure 1 in the original paper leads to a satisfactory conclusion that the curves on the graphs are very close. Of course, there are slight differences; however, there are many factors that may affect it, like data sampling and unrated items randomization. It would be a good idea to repeat the experiment with different `seed` values and draw a confidence region around the curve. However, there are no drammatic differences in the general behavior of the curves, which is a very nice result that didn't take too much efforts. Here are some top-score configurations from the experiment:" + ] + }, + { + "cell_type": "code", + "execution_count": 170, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.4 120 0.323893\n", + " 110 0.322767\n", + " 140 0.321977\n", + "0.6 60 0.321719\n", + "0.4 130 0.321421\n", + "dtype: float64" + ] + }, + "execution_count": 170, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result_flat.sort_values(ascending=False).head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## A bit of exploration" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The difference between the best result achieved with the EigenRec approach and the standard PureSVD result (that corresponds to the point with scaling parameter equal to 1) is quite large. However, such a comparison is a bit unfair as the restriction on having a fixed value of rank is artificial. We can draw another curve that corresponds to optimal values of both scaling parameter and rank of the decomposition:" + ] + }, + { + "cell_type": "code", + "execution_count": 200, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAARgAAACcCAYAAACk2LT2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJzt3XlcVXX++PHXm8u+r26gYoKkKLnglpaamVvZZFk6aplOTpnfb9+a7691rKaZmmmyxSbbpswy08otv6NmadKUmYKGKBIuKIILIMgu++f3x0GGCPWK93Lg8nk+Hvch557PPed9WN5+zud8FlFKoWmaZg9OZgegaZrj0glG0zS70QlG0zS70QlG0zS70QlG0zS70QlG0zS70QlG0zS70QlGs4qIHBORChEJbvB+oogoEQkXkaW1ZYpFJE9EvhaRq+uVnSUi1bX7C0Vkr4jc3PxXozUXnWC0y3EUmHZ+Q0T6AB4NyvxdKeUNhAIngPcb7N9Ru98feBNYKSL+9gtZM5NOMNrlWAbcXW/7HuCjxgoqpc4BnwF9L7C/pvZ4XkCkbcPUWgqdYLTL8SPgKyI9RcQC3AV83FhBEfHCqO0cvsB+C3AvUAmk2ydczWzOZgegtTrnazHfAj9j3AbV978iMh/wxUgctzbYP0RE8jFqLlXADKVUtn1D1syiazDa5VoG/BaYReO3RwuVUv5AOHAOiGqw/8fa/QHAeuA6u0WqmU4nGO2yKKXSMRp7JwBrLlLuOPAQsEhEGjYEo5QqBuYBM0Wkn53C1UymE4zWFHOAG5RSJRcrpJT6GjgJzL3A/lzgPeBpm0eotQg6wWiXTSl1RCmVYGXxl4BHRcTtAvtfAyaISIxtotNaEtETTmmaZi+6BqNpmt3oBKNpmt3oBKNpmt3oBKNpmt3oBKNpmt2YNlTA399fRUREmHX6ZldSUoKXl5fZYTQbfb2Obffu3WeUUiGXKmdagmnfvj0JCdZ2pWj94uLiGDlypNlhNBt9vY5NRKwaoKpvkTRNsxudYDRNsxs9XYPWolXXKPJKKsgpKudMQSFVJ5JwzkqkUlnI6TACi38oPu4u+Lo74+3ujI+7Cz7uzvi4O+PmbDE7/DZPJxjNdPtPFJBwLI/sonJyisrJKS4nt7AUn6LDdClLpQ9HiHE6whDJwFWq//PBwy+QVNONLdUD+LpmACmqCyB1u12dnfCtTTpXBXsxrncHburVAT9Pl+a/yDZKJxjNND+fLuTVrw+yOfk04XKafpajDHI9xjVOaXSvTsNNlYEzVDj7UBjQm9wO43EOG4D3VQNxri6l6sAGog5+SZ/Tq3mEVZzz7MTJ9iM5Eng9Rzz7kl8BRWVVFJ6r5Kfj+Wz9OZsnLfsYFhHMhD4dGauTjd3pBKM1u7ScYl7bcoj/SzrBeNd9JAR+RnBpmrHTyQM6xkCneyG0P3Tqj2vgVQQ7/bq50KVDL7jh/0FxNhzcjEfqJrofWUv3o5+Amy9E3AhREyDyRpS7P3szC9i47xQb953i0VVJPOlkJJuJfTpyU3R7/D1dm/k74fh0gtGaTUZeKa9vPcTqPZn0c04nLmQVXQt3g/tVMOoV6DwIQnqC5TJ/Lb3bQf+ZxqvyHKTFQepGSP0SkteAkzNyzTT63rKIvp39eWL81ew7UcCG88lmdRJPrhWujQhmYh/jNirASycbW7DqJyki44BFgAV4Tyn1twb77wceBKqBYmCuUuqAjWPVWqnTBWW8se0Qn8ZnECpnWNdxAzF5X0JVEIx/CWLvBYuNblVcPCBqvPGqqYGTe2DvCoh/D5yc4eZXERFiwvyJCfPn8XFXs/9EYV2yeWz1Pp5au5+h3YMY17sDY3q1p52Pu21ia4MumWBqZ39fDIwBMoF4EVnfIIF8opR6u7b8JOAVYJwd4tVakTPF5bwVd4RlP6bjVVPM+6HfcF3eaqRQYPjDxsvdz34BODlBWKzxcvOB7181ajujnqwrIiL0CfOjT5gfj42LIvmkkWw27TvFU2v388d1+4ntGsDY6A6M692BsABP+8XrgKypwQwCDiul0gBEZCXGTPF1CUYpVVivvBegZ7FqwwrOVbLqYAXzvtlGdWU5f+8azy35H2PJyYdrpsINfwS/sOYNavQzUJID374IXiEw6L5fFREReof60TvUj0fHRnEwq5gv95/my+TT/GVDCn/ZkEKfUD/G9e7A2OgORLTzbt5raIWsSTChQEa97UxgcMNCIvIg8AjgCtxgk+i0VudsSQVT3tnB4ewKFlx1iLuLl+Jy+hh0GwE3/Rk6XmNOYCJw8yIozYON/w88g6D35IsUF6I6+BDVwYeHbowkPbeEzcmn+XL/aV7anMpLm1OJaOfNuNqajZ4ZsnGXnDJTRKYAY5VSv6vdngkMUkr91wXK/7a2/D2N7JtL7QTQISEhAz777LMrDL/1KC4uxtvbsf/Hq6hW/D2+DM+iNN72+YBOZYco9upK2lX3kBfY3/gjN5lTdTkxSc/iW3iQpJinyQ+4/IR3tqyG3VnV7M6qIvVsDTUKfFwUEQHOXOXnRHd/C938nPBwNv967WXUqFG7lVKxlypnTYIZCjyrlBpbu/0EgFLqrxco7wScVUpd9OY6KipKpaamXio+h+Hog+Gqqmu4/+M9HElNZLPXcyhxwm3cc9B3Oji1sB61587CBxMg/zjM+hd0avqqKXklFWxJyWLdDwc4XeFG2hljoQURiAjxpm9nf/p28advZ3+i2vvgbHGM0TkiYlWCseYWKR6IFJFuGKv4TcVYeKv+ySKVUodqNycCh9DaDKUUC75IZmfKUb4LWoxrjYUfY/7KkP5TzQ6tcR4BMGMNvH8TfHwHzPkKgro36VCBXq7cGduZdsVHGDlyJAWllSRm5pN4PJ/EjLNsScni892ZxmldLPQJ9aNvF38iQrzx83TBz8MF/9p//Txc8HCxIC2gpmcrl0wwSqmq2qVAN2M8pl6ilEoWkeeABKXUemC+iNyIsc7wWYxF0bU2YtHWQ3y66xhfd1iCf+FxmLmOsvTqS3/QTL4dYeZaWHITLPsNzPkafDpc8WH9PF0Y0SOEET2MqVKUUhzPKyUxI5+fjueTmJHP0u3HqKiuafTzrhYnfD1c8PNwxt/TtS7xdA70ZEDXAPp18cfXvfX0PraqH4xSaiOwscF7T9f7+iEbx6W1Eit2Hee1LYf4IHQD3XN/gImvQLfrID3O7NAuLTgCpn8OS2+Bj2+HWRvAw9+mpxARugZ50TXIi1v7hgJQXlVNdmE5Becq6175pZX1tivq3ssqLCP1dBFfJJ6gRhm3XlHtfRjQNYDY8ABiuwYSFuDRYms9uiev1mRfH8jiqbX7eDI0kVG5KyB2DgycY3ZYlyd0AEz9GJbfCSt/CzNWG5317MjN2ULnQE86X8ZnisurSDyeT0J6HrvTz/JF4kmW7zwOQIiPG7FdAxhQ+4ru5Ierc8to69EJRmuS3el5zP9kD7e3O8V9+a9B+HUw/kWzw2qa7jfAbW/D6jmw+ncw5cPLH65gZ95uzgyPDGZ4ZDBgTGORerqI3el5JKSfZXf6WTbtPw2Am7MTg7oFcnNMR9OHPbSs76LWKhzOLmLOhwnE+BTzYtWLiG8nuPMj23X3N0OfO6DkDHz5GGx4GG55vUU8Vr8Qi5PQq5MvvTr5MnNoOABZhWUkHDtLQnoeW1Oy64Y9DIsIZmKMOaPHdYLRLktWYRn3LInHSyr52Od1nPJL4Z714BlodmhXbsj9UJIN370MXu1g9AKzI7os7X3dmRjTkYkxHXn65l51Y6w27DvJo6uSeMqyj+ERwUyM6cSYXu3x87B/stEJRrNawblK7lmyi/zScrZHrsTtyD6YtgLa9TQ7NNu5YYExpOC7hUZv36HzzI6oSRqOsdp3ooANSaf4V9Iptn2+FxeLcH1kCBNjOnJjr/Z2ezKlE4xmlbLKauZ+lMCRnGK+jo3Hf+96Y3xP1HizQ7MtEbj5NaMz3uYnjD4zfaeZHdUV+cXo8fFXszezgA1JJ9mQdIqtP2fjanFiWEQQI6PaMTIqhK5Btlt+RScY7ZJqahR/+GwvO4/m8emIXMJ3vgJ97jRGQzsiJwvc/j4snwJfPGiM+L56gtlR2YSIGL2LO/vzxPieJGbmG4kmJYttqckAhAd5MjKqHSN6hDDkqiA8XJveE1snGO2SXttykA37TrHwemcG73kcOvWHSS27EfSKObvB1OXw4ST4fBbMXAPhw82OyqacnIT+XQLo3yWABTf34tiZEr49mENcajYr44+z9IdjuDo7MeSqIEb0CGFkVAhXBXtdVp8bnWC0iyqtqOKD7ce4s5cndxz8b2NelanL7d5XpEVw84Hpq+CD8bBimjFuyazR4M0gPNiL8GAv7rk2nLLKanYdzSMuNYdvD2bz538d4M//gs6BHnW9lK2hE4x2URuSTlFWXsYfS16Goiy4dxP4djI7rObjFVQ7pGAsLJsMszcbPYAdnLuLhet7hHB9jxCgFxl5pbW1mxzW7Dlh9XFaRnc/rcVaGZ/BX3zW4Ju1E25dDGEDzA6p+fmFGkkGYNltUGD9H5ij6BzoyYwhXXnvnlh+enqM1Z/TCUa7oINZRaSmn2ByzWaImQoxU8wOyTzBkTBjlfF06ePJxsRVbdTlLGinE4x2QSt2Hed2lx9wqT4Hg+aaHY75OvUz+v3kHTWeMJUXmx1Ri6cTjNaosspq1u7J5D6POOgQY6xRpBkjxad8YKxW8OkMqCo3O6IWTScYrVGbk0/TrSyFsIo0iJ3t2I+kL9fVE2HSG5C2DdbMhZoWPveNifRTJK1RK3YdZ65nHMrijfS5w+xwWp5+0432mK+egg3+4P0bsyNqkXSC0X4lLaeYA2nHGeP5AxIz3egPov3atfOhNBe+f4XITllwbaz+XjWgb5G0X/k0PoMpzt/jXFNu3B5pFzb6aRjyIKEnN8E/YiHxE2NFSQ3QCUZroKKqhlUJGfzOIw5CY6FDH7NDatlEYNwL7O7/d2MxuXUPwHujIWOX2ZG1CDrBaL+wJSWLiHNJdKw8rmsvl6HIN8qYOPy2d6DoFLw/Blbf1yY75dWnE4z2Cyt2HWeORxzKzReibzM7nNbFyclYGnd+Alz3v3DgC3gjFuJehIpSs6MzhU4wWp2MvFJSDh9htNqB9P0tuOqF3pvEzduYDW/+LogcA3EvwOJBsH81tLElZnWC0ep8lpDBHZZ/Y1FVMOBes8Np/QLCjbmKZ20Ad39YNdsYmX0y0ezImo1VCUZExolIqogcFpHHG9n/iIgcEJEkEdkqIl1tH6pmT1XVNXwen8697t9Cl2uh3dVmh+Q4wofD77+FWxbBmUPw7khY83vI/tnsyOzukglGRCzAYmA80AuYJiK9GhT7CYhVSsUAq4C/2zpQzb62peYQUbKH9lUndeOuPThZYMAs+O89Rv+ZA1/Am4Phk6lw/Eezo7Mba2owg4DDSqk0pVQFsBK4tX4BpdQ2pdT5VqwfgTDbhqnZ28pdx7nXbRvKMwh6TTI7HMfl7gc3/QUeToYRj0PGj8ZcM0vGQeqXDteHxpoEEwpk1NvOrH3vQuYAm64kKK15nSo4R3JqKqNUvNG46+xmdkiOzysIRj1hJJpxL0JBJqy4C966FhJXQHWl2RHahDVDBRob5dZoU7iIzABigREX2D8XmAsQEhJCXFycdVE6gOLi4hZ7vV8cruAOpzicqGZnVS/O2SDOlny99nBl13s1cs1rtMv+ns4Za/Bedz9lm/5IZtitnOo4hmrnVjw9qVLqoi9gKLC53vYTwBONlLsRSAHaXeqYSil69Oih2pJt27aZHUKjqqpr1PAXvlI5z0Uo9eEkmx23pV6vvdjsemtqlErdrNT745R6xlepv3ZRautflCrKts3xbQRIUFb8nVtzixQPRIpINxFxBaYC6+sXEJF+wDvAJKVUto1yn9YMvjuUQ0TRToKrs/Wj6ZZABHrcBLM3GT2Duw6Df/8dXulpzD9zcDNUV5kdpdUueYuklKoSkfnAZsACLFFKJYvIcxhZbD3wEuANfF67pMFxpZRuKWwFVu7K4F7Xb1De7ZGrJ5odjlZf50Ew7RPIOQh7PoS9KyHl/8C7g9FjuN8MYyrPFsyq6RqUUhuBjQ3ee7re1zfaOC6tGeQUlZOcksxw15+Qfg+37sXrHVlIDxj7vLGS5qHN8NNy+OEfsP016DzEmJsm+rYWOVWEng+mDVu1O5M7nL5BUND/HrPD0S7F2RV63mK8ik4bNZqfPob1/wWbHjOSTN/p0PXaFjMDoU4wbZRSilW70vjc9Vuk+xgI0J2vWxWfDjD8f2DYQ5AZDz8tg/1rIHE5BF4FUROM26egSAiKAO92piQdnWDaqB1puUTkbyfQNU837rZmIkZbTedBMO5vcGC9kWR2/ROq601I7uYLQd3/k3CCI2q/7g6utlvsviGdYNqolbsyuNv1G5RPJyTyJrPD0WzB1Qv6TjNeNdVG573cQ5B7xBgDlXsYju+AfZ/98nO+odA+GroMMcahdeoHLu42CUknmDYor6SC5P2JDHPZCwOeBIv+NXA4ThbjtjegK0Q0eAZTUQp5aUbCyT0EZw7DyZ/g0FfGfosrdOoPXYdCl6HQeTB4+DcpDP2b1Qat2ZPJHbIVJRak/0yzw9Gam6sndOhtvOoryTXGRh3fAek7jCdV378KCLTrZdRwul5r/GslnWDaoI2J6Sxx+TcSNb5tLWSvXZxXkLHm0/n+UBWlcGK3kXCO74CkTyHh/cs6pE4wbUxWYRmhp7bg71oAsbpxV7sIV09jJctu1xnb1VWQtd9INn+aZ9Uh9Ix2bcw3P2dzu+U7Knw6w1U3mB2O1ppYnKFTXxjygNUf0Qmmjfku+RhDLQdwiZ5kTFKtaXakf8PakHMV1VSn/Rs3KpEe+tG0Zn86wbQh2w+f4Tq1hypnL+Pxo6bZmU4wbcjWlNPcYNmLU/eRetY6rVnop0htRE2N4mjKbjrJGWO+EU1rBroG00bsP1lAzLna9ZIjxpgbjNZm6ATTRmxJyWaUZS9VIdHgd7E52zXNdnSCaSN2JKcx0CkV5yh9e6Q1H51g2oBTBecIyt6BM9WgR05rzUgnmDZga0o2o5wSqXb1g7BBZoejtSE6wbQBWw+c5kbnvThFjtZTM2jNSicYB1daUUVe2m6COKsnltKanf7vzMF9f+gMw9VPxkbDiYdasMrKSjIzMykrKzM7FKv4+fmRkpJidhg25+7uTlhYGC4uTVtxQicYB7c1JZtpzonUdOqPk3eI2eFYLTMzEx8fH8LDw5EWMkP+xRQVFeHj0/KWDbkSSilyc3PJzMykW7duTTqGVbdIIjJORFJF5LCIPN7I/utFZI+IVInIHU2KRLO5mhpFQsphYuQQTq3s9qisrIygoKBWkVwclYgQFBR0RbXISyYYEbEAi4HxQC9gmoj0alDsODAL+KTJkWg2l3SigOhzCTihWuXjaZ1czHelPwNrajCDgMNKqTSlVAWwEri1fgGl1DGlVBJQc0XRaDa1NSWLGyyJ1HgGGzPFa3b12muvUVpaWrc9YcIE8vPzr/i4cXFx3HzzzVd8nMaMHDmShIQEuxwbrEswoUBGve3M2ve0Fm5r8klucN6HU8SNenKpZtAwwWzcuBF//6bNxn+5qqqqmuU8l8uaRt7G6kiqKScTkbnAXICQkBDi4uKacphWqbi4uFmv98y5Gtyyk/B1K+RAZSjZzfy9vtLr9fPzo6ioyHYBNcEbb7zBsmXLALj77rt58MEHSU9PZ/LkycTGxpKUlERERATvvPMOS5cu5eTJk4wYMYKgoCA2bNhA7969+fbbbykuLmby5MkMHTqU+Ph4evfuzYwZM3jhhRfIycnhvffeIzY2loSEBB5//HHKyspwd3fnrbfeIjIyktLSUqqqqn71/Vi+fDmbN2+mrKyM0tJSVq5cybRp08jPz6eyspIFCxYwceJE0tPTuf322xk6dCg7d+6kY8eOrFy5Eg8PD6qrqykpKaGgoIAHHniA0NBQnn766V+cp6ysrMk/S2sSTCbQud52GHCyKSdTSr0LvAsQFRWlRo4c2ZTDtEpxcXE05/V+tOMYIy0focSJXrfMp5dnYLOdG678elNSUuqeyvzp/5I5cLLQRpEZenXy5Zlboi+4f/fu3XzyySfEx8ejlGLw4MGMHTuWgIAADh06xAcffMCwYcOYPXs2y5YtY968ebz11lt8++23BAcHA0b7hbe3NwBpaWmsXr2a6OhoBg4cyLp169ixYwfr169n0aJFrFu3jgEDBrB9+3acnZ3ZsmULzz//PKtXr8bT0xNnZ+dfPaVyd3cnPj6epKQkAgMDqaqqYv369fj6+nLmzBmGDBnCXXfdhbe3N0eOHOHTTz+lb9++3HnnnXz11VfMmDEDi8WCm5sb999/P7179+app5761ffC3d2dfv2adottTb05HogUkW4i4gpMBdY36Wxas9mSks041yRj0axmTi6O4Pvvv+e2227Dy8sLb29vJk+ezHfffQdA586dGTZsGAAzZszg+++/v+TxunXrRp8+fXByciI6OprRo0cjIvTp04djx44BUFBQwJQpU+jduzcPP/wwycnJlzzumDFjCAw0fr5KKZ588kliYmK48cYbOXHiBFlZWXXn79u3LwADBgyoOyfA73//+wsmlyt1yRqMUqpKROYDmwELsEQplSwizwEJSqn1IjIQWAsEALeIyJ+UUhf+70Gzq+LyKg4fOUKUyxGInG52OFfsYjUNe1Hqwq0ADZ+sWPOkxc3tPzMIOjk51W07OTnVtZ8sWLCAUaNGsXbtWo4dO2ZVDdDL6z/rSi9fvpycnBx2796Ni4sL4eHhdY+Y65/fYrFw7ty5uu1rr72Wbdu28Yc//AF3d9ssGXueVS1/SqmNSqkeSqnuSqnna997Wim1vvbreKVUmFLKSykVpJOLub4/lMO11PbebYWPp1uC66+/nnXr1lFaWkpJSQlr167luuuM9YGOHz/Ojh07AFixYgXDhw8HwMfH54rajQoKCggNNZ6fLF26tEmfb9euHS4uLmzbto309HSrPjdnzhwmTJjAlClTbN5YrB8tOKAtKdmMcdmL8ukI7Xtf+gPar/Tv359Zs2YxaNAgBg8ezO9+97u6doiePXvy4YcfEhMTQ15eHg88YKwTNHfuXMaPH8+oUaOadM5HH32UJ554gmHDhlFdXX3Zn58+fToJCQnExsayfPlyrr76aqs/+8gjj9C/f39mzpxJTY0Ne5sopUx59ejRQ7Ul27Zta5bzVFXXqEF/2qhK/9RRqS/mN8s5G3Ol13vgwAHbBGJjR48eVdHR0b96v7Cw0IRomkdjPwuM5pFL/p3rGoyDSczIJ/xcMh41Jfr2SDOdTjAOZmtKFjc4J6KcXKDbCLPDcTjh4eHs37/f7DBaDZ1gHMzWlGzGu+1Dug4Fd1+zw9HaOJ1gHEhGXilFWUfpUnVM3x5pLYJOMA5ka0oWIy17jQ2dYLQWQCcYB7L152wmeuwD/y4Q3MPscDRNJxhHUVRWyZ600wysSYLIsaDnUrlir7/+Oj179mT69OmsX7+ev/3tbzY57vnxSbb27LPPsnDhQrscu6n0lJkO4rtDZ+inUnCtKdO3Rzby5ptvsmnTprrpIidNmtQs563rQ+IAU2y0/ivQANhyIIuxrkkoZ3cIH252OK3e/fffT1paGpMmTeLVV19l6dKlzJ8/H4Bbb72Vjz76CIB33nmH6dON8V5Hjhxh3LhxDBgwgOuuu46ff/4ZgKNHjzJ06FAGDhzIggULGj3fsWPH6NmzJ/PmzaN///5kZGTwwAMPEBsbS3R0NM8880xd2fDwcJ555hn69+9Pnz596s5T3z//+U/Gjx//izFHZtA1GAdQXaPYlprN465JSJfrwNXT7JBsa9PjcHqfbY/ZoQ+Mv/Atz9tvv82XX37Jtm3bCA4O/sXYoHfffZdhw4bRrVs3Xn75ZX788UfAGCrw9ttvExkZyc6dO5k3bx7ffPMNDz30EA888AB33303ixcvvuA5U1NT+eCDD3jzzTcBeP755wkMDKS6uprRo0eTlJRETEwMAMHBwezZs4c333yThQsX8t5779Ud54033uCrr75i3bp1vxjkaAadYBzAnuNn8T2XQTu3DIj8b7PDcXjt27fnueeeqxv5HBgYyKlTp/jhhx+YMmVKXbny8nIAtm/fzurVqwGYOXMmjz32WKPH7dq1K0OGDKnb/uyzz3j33Xepqqri1KlTHDhwoC7BTJ48GTCmXlizZk3dZ5YtW0ZYWBjr1q1r8lIjtqQTjAPYtO80o+seT48xNxh7uEhNwyz79u0jKCiIkyeNuddqamrw9/cnMTGx0fLWTOlQf+qFo0ePsnDhQuLj4wkICGDWrFm/mN3/fM3EYrH8YgR07969SUxMvKKlRmxJt8G0cu99l8aS7Ue5w/cABEVCoPm/VI5u165dbNq0iZ9++omFCxdy9OhRfH196datG59//jlgNNTu3Wsk/WHDhrFy5UrAmLPFGoWFhXh5eeHn50dWVhabNm2y6nP9+vXjnXfeYdKkSXXJz0w6wbRSSin+ujGFv2xI4TfRfvQsT9JPj5pBeXk59913H0uWLKFTp068/PLLzJ49G6UUy5cv5/333+eaa64hOjqaL774AoBFixaxePFiBg4cSEFBgVXnueaaa+jXrx/R0dHMnj27bgY9awwfPpyFCxcyceJEzpw506TrtBVRF5m5y56ioqJUamqqKec2gy3n5K2sruGx1Ums2XOCmUO68uzVGVhWToWZ66B70+YisTVbzMnbs2dP2wVkZ464suN5jf0sRGS3Uir2Up/VbTCtTGlFFQ8u38O21BweGdOD/xreAVn1HLh4QddrzQ5P035BJ5hW5GxJBfcujScpM58XbuvDb/32weLfQGEmjHoKnM19JKlpDekE00qcyD/H3e/vJOPsOZbc1pGRRx6H1A3QLhqmfACdB5kdoqb9ik4wrUDq6SLuWbKLsopyvh68j65fLwIUjHkOhswDi/n9HexBKaXXpzbZlbbR6gTTwsUfy2PO0ngGWI7wVtAy3HcfgB7jYMJLxqhpB+Xu7k5ubi5BQUE6yZhEKUVubu4VLWUNP/fvAAAEdUlEQVSiE0wL9vWBLJ745Dv+7L6KSVVfIhUd4c5l0PMWhx8tHRYWRmZmJjk5OWaHYpXzy706Gnd3d8LCwpr8easSjIiMAxZhLLz2nlLqbw32uwEfAQOAXOAupdSxJkel8emudLZ/8S5fuS4noLoAGXw/3PAUuDnmo9CGXFxcWkRPVGvFxcU1eXlVR3bJBCMiFmAxMAZjnep4EVmvlDpQr9gc4KxSKkJEpgIvAnfZI2BHoJSiqLyKM0Xl5JZUkFtQTGFBLqUFZygrzKWs4Az9T63kdZd9VLfvi0xaBJ36mh22pl02a2owg4DDSqk0ABFZCdwK1E8wtwLP1n69CnhDRERdpIWo8lwRm1f8o0lBW+eXp5YG26hG9gtII+Xr3hP1n+0ahaqpoqamGmqqUbUvaqpRqv57NaCqKSs6y5Yd/8C1shD36iJ8VAm+UkJPSvCWMhoqd/GkasyLOA++D5wsV/B90DTzWJNgQoGMetuZwOALlVHGWtYFQBBwwX7KgVVZjE394+VF20pV40Q5Lpyz+FLu4kOVlx81bu2p8vAj3zOAUu8gPHwD8fANwtkrENz9cQuO1IvWa62eNQmmsdbEhjUTa8ogInOBubWb5fKnwra0wEww5Jg7MKR5BXOR/2AcUFu73ihrClmTYDKBzvW2w4CGwzTPl8kUEWfAD8hreCCl1LvAuwAikmDNWAZHoa/XsbXF67WmnDWjqeOBSBHpJiKuwFRgfYMy64F7ar++A/jmYu0vmqa1DZeswdS2qcwHNmM8pl6ilEoWkecwFsBeD7wPLBORwxg1l6n2DFrTtNbBqn4wSqmNwMYG7z1d7+syYErDz13Cu5dZvrXT1+vY9PU2wrT5YDRNc3x6RjtN0+zG1AQjIi+JyM8ikiQia0XE38x47E1EpohIsojUiIjDPnEQkXEikioih0XkcbPjsScRWSIi2SLSJrpciEhnEdkmIim1v8sPXay82TWYr4HeSqkY4CDwhMnx2Nt+YDLwb7MDsZd6Q0vGA72AaSLSy9yo7GopMM7sIJpRFfAHpVRPYAjw4MV+vqYmGKXUV0qp82su/IjRx8ZhKaVSlFKOPhFx3dASpVQFcH5oiUNSSv2bRvp8OSql1Cml1J7ar4uAFIye/I0yuwZT32zAurUZtJassaElF/wF1FovEQkH+gE7L1TG7vPBiMgWoEMju55SSn1RW+YpjKqXdYvGtGDWXK+Ds2rYiNa6iYg3sBr4H6VU4YXK2T3BKKVuvNh+EbkHuBkY7Qi9fy91vW2ANUNLtFZMRFwwkstypdSai5U1+ynSOOAxYJJSqtTMWDSbsWZoidZKiTF/6ftAilLqlUuVN7sN5g3AB/haRBJF5G2T47ErEblNRDKBocAGEdlsdky2Vttof35oSQrwmVIq2dyo7EdEVgA7gCgRyRSROWbHZGfDgJnADbV/s4kiMuFChXVPXk3T7MbsGoymaQ5MJxhN0+xGJxhN0+xGJxhN0+xGJxhN0+xGJxhN0+xGJxhN0+xGJxhN0+zm/wOTcZ5FWBAhIwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "result_flat.groupby(level=0).max().plot(label='optimal rank', legend=True)\n", + "result_flat.xs(best_rank, axis=0, level=1).plot(label='fixed rank', legend=True, title='MRR',\n", + " figsize=(4.3, 2), ylim=(0, None), xlim=(-2, 2), grid=True);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now the difference is less pronounced. Anyway, the EigenRec approach still performs better. Moreover, the difference vary significantly from dataset to dataset and in some cases that difference can be much more noticeable. Another degree of freedom here, which may increase the top score, is the maximum value of rank used in the grid search. We have manually set it to be 150. Let's look which values of rank were used at each point of the curve:" + ] + }, + { + "cell_type": "code", + "execution_count": 202, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAACcCAYAAAAu9y23AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJztnXd4VNX2v9896Y1A6BAkdAghBEioIiAtdPWKwkUMShEV9apXwQpfvD8uKqKAWEARRRQQQbkCIiiIAgqhhV4CAQIhlJBeZ2b9/pjJkJ4hyWRSzvs888zsc/bZe50pa3ZZ+7OViKChoaFR2dDZ2wANDQ2NkqA5Lw0NjUqJ5rw0NDQqJZrz0tDQqJRozktDQ6NSojkvDQ2NSonmvDQ0NColmvPSsAqlVJRSKlMpVSfP8UNKKVFK+SmllpvzJCul4pRSW5VSbXPknaCUMpjPJyqlDiulhpf/3WhUBTTnpXEnnAfGZieUUh0Atzx53hERT6AxcBn4PM/5PebzNYGPgFVKqZq2M1mjqqI5L407YQXwaI50GPBVQRlFJA1YAwQVct5oLs8DaFW2ZmpUBzTnpXEn/AXUUEq1U0o5AA8DXxeUUSnlgamVdraQ8w7AY0AWcME25mpUZRztbYBGpSO79fU7cBJT1zAn/1ZKTQNqYHJKo/Kc766UisfU4tIDj4jINduarFEV0VpeGnfKCuCfwAQK7jLOE5GagB+QBrTJc/4v8/lawAagt80s1ajSaM5L444QkQuYBu6HAuuKyHcReA5YoJTKO6iPiCQDTwHjlVKdbGSuRhVGc14aJWEicK+IpBSVSUS2AleAKYWcvwl8BrxZ5hZqVHk056Vxx4hIpIiEW5n9XeBlpZRLIec/AIYqpQLLxjqN6oLSxAg1NDQqI1rLS0NDo1KiOS8NDY1Kiea8NDQ0KiWa89LQ0KiUaM5LQ0OjUmK35UE6nU7c3PLFLlZZjEYjOl31+a/Q7rdqk5qaKiJi1xu2m/NydnYmJaXIGMcqxY4dO+jbt6+9zSg3tPut2iil0oo5vwwYDlwTkQDzsVnAZOC6OdurIrLJfO4VTMHPBuBZEdlSnA3V569CQ0OjPFkOhBZw/H0RCTI/sh2XPzAGaG++5iOz6kiRaM5LQ0OjzBGRnUCcldlHAatEJENEzmOSUepa3EWa8yoHvjnyDS9FvER8ejwA3x//nkErBpGSaeo2rz66mkErBpGhzwBgZcRKBq0YhMFoAGD5oeWEfn37T2zp/qWM+HaEJf3Rvo94YPUDlvTCvxfy0HcPWdLv7X6PcevGWdJz/5zLhB8mWNL/2fkfJm2YZEnP3D6TJ3960pJ+7dfXeGbTM5b0y1tf5oUtL1jSz//8PNO3Trekn9n0DJ+d/8ySnvrTVGbtmGVJT/xxIv9v5/+zpMN+CGPx3sX53zjAYBSuJ2Vw/EoiO09E89u2TexcOYdfv36bVdv+4rvwS/x89Cq7z94gIjqe8zdSuJGcQYbeUGB5GnZnmlIqQim1TClVy3ysMXApR55o87Ei0fS8ygEXBxfiMuMwihGALGMWyZnJCKalWZmGTJIzky3586Yz9BmkZKVYfT5f2pBhcZQFnU/Xp5OalZo7rb+dTtOnkaa/PcSRlpWG3qjPdd4gt51FalYqmcbMXOl0ffrttD53+nDMSbLS7iIl7iTXkzK4npzBzcRUvJLOclf6KToQSaAuku7qEs4qh1M6O4cIYzO2Gbqw1diFE3IXoCynnR111HB1xMvVieZ1PAgNaMAg/wZ4uzuhUWoclVI517cuEZElxVzzMfAWIObn94DHyfmh3abYdYt2W9vo6uoq6enpxWesIlS3AV1r7vfk1UTe33qaLceu4qeu0snhPF2do+ioO0cLwzlcxPT9yHT0IrFWAFkNgnD07YJn8xAcDanoj29Ed/pnnK/uRyGkuTfiSv2+RPrcQ6R7EPGZkJSuJzEti4MX47kcn4aTg6JXyzoM7dCQwWXoyKrb56uUShURj2Ly+AE/ZQ/YF3bOPFiPiPzXfG4LMEtE9hRVvtbysjFGMRKXFoe2AP42564n88G2M/wv4jJDnI8Q7rOGOqnn+Bk9TsoZ/4Yh0OgxaNwZGnXG2ac5dQoIQ3Bq4A/3vgTJ1+D0FtxObaZF5HpanP8GXGpAywHQZii0GoC41uRwdAKbjsSw6UgML6+N4FXdEXq1rMOwDg0Z1L4+Nd2d7fBuVB+UUg1FJMacvB84an69AfhGKTUfaIRpT4O9xZantbxsy4X4C/gt8OPfrf/Nu2Pftbc55UZBLZFLcaks/PUM3x+IppPjBebXWkvTxP3g0xxD96fosGcuTWu3YvMjP5e84qw0OLcDTm2CUz9DyjXQOULHsTBiAegcEBGOXE5go9mRXYpLw1Gn6NmyDsM6mLqWtTzuzJFpLa98578F+gJ1gFhgpjkdhKlLGAU8ke3MlFKvYepC6oF/icjmYm2wxnkppUKBBYAD8JmIzM1zfirwNKYYjWRgiogcL6rM6uK8bqbeZEXECnzifHh06KPFX1BFyPljvpqQzofbz7B63yUaqxssrLeRwLifwb029JkBwY+BgxORcZE08W6Cs0MZtYCMRrhyAA5/C/s+gy6PwfD3Qd0eYhERjl5OtDiyi3GpOOoUPVrUJjSgAQP961PPy/WO7rc6YE230eY2FOe8zPEWp4GBmGYB9gFjczonpVQNEUk0vx4JPCUiBcV4WKguziub6vbl3rFjBwHBPfh4RyQr/rqAhzGZBY1/o3fc9yiloPuTcPfz4Oqd71q9UU+GPgMP5zL8bWybBX++D32mQ79XC8wiIhy7YnJkm4/EEHUzFaUguGktBrdvQGhAA3xruRd6v9Xp860IzsuaMa+uwFkROQeglFqFKS7D4ryyHZcZD6yYKaguxCbH4uRQvWa3EtKyWHs6k6d+244hK4N3mu5jRPzXOFyPh45j4N7Xwdu3wGszDZmELA2hT9M+LByysOyM6j8TUq7D72+DR13oOjlfFqUUAY29CWjszcuD23A6Npmfj17l52NX+c/GE/xn4wk6NPYmNKABg9s3oGU9z7KzT+OOscZ5FRSD0S1vJqXU08ALgDNwb5lYVwV4/bfX+d/p/7EqeJW9TSkXbqVkMvrTPZy9lskbzc/waPJynK5GQbM+MOgtaNixyOudHZx5yP8hAurlm6AqHUrB8AWQGgebXjJ1WQMeKCK7ok0DL9o08OK5Aa24cDOFLceu8vPRq7y75RTvbjlFy3qehJpbZNqETPljTbdxNDBYRCaZ0+OBriLyTCH5/2nOH1bAuSmYN2NwdHTssnXr1lKaX/E5knCE2PRYunt0x9Ozav9TZxqEd/al4550jk+8vqBR+hmSPZpyrnkYcT6dc4012QudIYPAiFnUSDxNROCbxNcq2pkWxK10I/tjDeyP1XPqlhGjgJeT0LKWI829dbSo6UAzbx1ujva/X1vRr18/u3cbrXFePTDFXAw2p3PFZBSQXwfcEpH8gxk50Ma8qhZ6g5GpXx8g8tQhtnjMRpQOl9DZEDQOdMUuU8tfnlHP0v1LaVOnDfc2K+OGfNot+GIoxF+ECT9Bo5LvvBaXksm2E7H8sPs4VzNdOHfDFPyrFLSs60lQk5oE3VWToCY1aVPfC0eHqrGopbKMee0DWimlmmHaHXkMpk1HLSilWonIGXNyGHAGDUSEQ1cP0dKnpb1NsSkiwhs/HuPvE+f5o/ZinI0O/BX4X7p3HlPiMg1GA/P2zGNAswFl77zcasEj6+DzQfD1gzDxF6jdokRF+Xg481BwE+olR9K3b18SUrM4FB3PoYvxHLp0i20nYvluf7SpWicHOjT2JuiumrSs64m3uxPebk7UND97uznh5uRgmtDQKJZinZeI6M3bt2/BFCqxTESOKaVmA+EisgHTeqUBQBZwC8jXZayOXEu5RuclnVk0ZBEBlPEYTgViwa9nWL03iq0NllEz8SKM/4H0C6VbW+ji6MKux3dR36N+GVmZhxoNYfx6WDYIVtwHE7eCV4NSF+vt7kSf1nXp07ouYHLsF+NSOXQpnoMX4zl0KZ7lu6LINBgLvN7ZQUcNNye83Ryp6e5scWpNfNzp0rQWne6qSQ3X6jUBVBhakKoNSclM4eezPxPUIIhLEZeqZLfx270XeWXdEb5o/D/63fwWhs2HkIll2k1OSE9Ap3R4uXiVSXm5uLwflo8An2YwYSO41SxRMXdyvxl6A9cSM0hIy7I84lOzcqQzcx2LT80iJiENo5i6o23qe9GlaS2C/WoR3NQH31pu5d5aqyzdRo0S4uHswT/8/wHApVwTtlWDrcdjeW39EV5tfMjkuIInQsjEMq0jIT2B1h+2JqxjGO8MfKdMywagcRcY8zWsfAhW/RMe+R6cbKvw6+LoQBMfd5rcwTXJGXoOXYwn/EIc+y/c4sdDV1j590UA6nq5ENy0Fl3Mj/aNvHF2rBpja0WhOS8bEhkXSVJmEh3r3/mMVkVn/4U4pn1zgH/Ui2Fy/Afg1xuGvF3m9Xi7evNyz5fp16xfmZdtocW9cP8n8P1E+H4SjP4SHCrWT8PTxZG7W9Xh7lZ1AJNU0KmrSey/EEf4hVvsv3CLzUevAuDiqKNrMx+GBzYs0VKnykLF+oSqGO//9T5fR3xN/Ix4e5tSppy9lsTEL8MJ9Ermbf3bqBqN4KGvwEbBuC/2fNEm5eaiw4OQcgN+ng4bn4cRCytEaEdhOOgU/o1q4N+oBuN7+AEQm5hOeNQtwi/E8euJa0z//givrT9qWnweWLYqGhUBzXnZkGe7PcvINiPtbUaZEpuYTtiyfXioLL72WoguPhXCNoC7j03rTc5MZvbvsxnRegS9m/a2TSXdp5oWcv/xHnjUg/5v2KYeG1G/hivDAhsyLLAhbw73t6zZ3HjkCi+vjeA1hyPc3bIOwwIbMdC/Pt5ulduRac7LhrSu3ZrWtVvb24wyIyEti7Ble4lPzWBXq1W4RB6Bsd9CvXY2r1undKw5toZarrVs57wA7n3DtIzoj3mmKPweT9muLhuilKKDrzcdfL2ZHtrGpKIREcNPETFs/+4wTg6Ke1rVZVhgQwb416+UM5ia87IhP53+iaAGQfjWKHgdX2UiPcvAlK/CibyezNbgfdQ8vMG0XrDNkHKp393JnaNPHcXT2carFJSC4R+YAlm3vGKKCQsaa9s6bYxSikDfmgT61mTGkLYcjk5gY8QVNkbE8OvJazg76OjVsjZ929Sjb5u6NK1t10lEq9Gcl42IT49nxLcjmDdwXvmM2dgQo1F4cc1h/j4fx+o+N/H7ez50eMikClGOZDuuc7fOUc+jnu0cmc4B/vE5rBwNPz5tUr5oO9Q2dZUzSilT1H+TmrwypB2HouNNTuxELNtPHQPAr7Y7fdvUo0/runRvXhs35ztfIVEeaM7LRng4efD3pL9p5NXI3qaUmg+2nWbjkRjm3eNItwMzoFFnGGmfAe3oxGj8F/szvdd0/q/f/9muIkcXGLMSvhwJ302A8evA727b1WcHdDpF57tq0fmuWrwx3J+oGyn8fvo6O05dY9W+iyzfHYWzo47uzWvTp3Vd+rapS/M6HhVmBYDmvGyEk4MTXRsXu3tThSc1U88Xu6J4yN+dB08/Cy5eph+1jWOhCsO3hi9f3vclHep3sH1lLl4wbi18MQS+HWtaB1mMKkZlxq+OB351PAjr6Ud6loG95+PYceo6v5++xls/Heetn6CJj5tl9YC90ZyXjYiIjeBC/AWGthqKQwkWJlcUNkbEkJ6Rzusp70FSLDy2GWrYtzX5cMDD5VeZR23zMqLBsOIBeHwL1Knaa1UBXJ0cuKd1Xe5pXRfw51JcqrlVdp11By7b2zxA27fRZnx1+CseXvswOlW53+JV+y7xH6911Ij9G0YtBt8u9jYJgB1RO9h+fnv5VObd2OTAAFbcDwkV48dbnjTxceeR7k35LCyYg28OtLc5gOa8bMYrd7/Cnol7Ksz4QEk4HZvEqQuXecC4BQLHQOBoe5tk4fktz/PWzrfKr8I6reCRtaZZyK8fMIkaVlNcHCtGT6JCLczOysoiOjqaqrhgOz09HVfX4jdyqEjEp2ZBZhI1SQbPBuBo/TITW9/vucRz1Herj4eTbab1XV1d8fX1xckpT/zT+T/g639Agw7w6I/gYprxrOp6bXnRFmbnITo6Gi8vL/z8/Cp1iwXgswOf0bVxVwLrBwKQlJSEl5cNVBFshNEonLiaSCt1GWeH2lC3zR3NLtr6ftthu8BYEeHmzZtER0fTrFmz3Ceb9YbRX8DqR0yPf642zUxqlDsVqtuYnp5O7dq1K73jSs1KZfL/JvO/U/+ztyklJjE9CxdjOs6SAR51KuQ6v68Of8XH+z4u83KVUtSuXbvwHkDbYTDyQzi3HdZNAWPptMs0SkaFankBld5xAbg5uhHzYgxOusq35CKbmymZ1NUlIUqHcqtlb3MK5MdTP3It5RpPhjxZ5mUX+z3sNM40/vXLa7CxJnjeV+Y2aBRNhXNeVQGlFA08S6/KaS8ysgykZ2RSQ5eMcqtdIg368uDL+7602ZiXVfScBqk34c/5tGoUCz2DTbFhGuVCheo2VkY++OADUlNTLemhQ4ey7eQ2Ptr3ERn6jBKXu2PHDoYPH14WJuajb9++hIeHF3o+LjWTWioZhZjinMqgzDvh4MGDTJo0qcBz4eHhPPvss4BpuVBBLaSoqCgCAgqW3V6+fDlXrlyxpMeMGcOZM6XYcqH/m9D9aRpf2QyLguHQN6adujVsjua8Skle57Vp0ya2X97Ocz8/Z/PgVL1eX+ZlGkW4lZJJHV0SOLmbHuXMnDlzeOaZ/Dvr6fV6goODWbjw9ma08/fMZ/KG/BvIFkZe5/Xkk0/yzjulUGhVCkLnsL/zO6aNdH94Ej7rD5f2lrzMKoBSaplS6ppS6miOYz5Kqa1KqTPm51rm40optVApdVYpFaGU6mxNHZrzysP8+fMJCAggICCADz74ADD9k7dt25awsDACAwN58MEHSU1NZeHChVy5coV+/frRr59J6dPPz4/nAp9j94O7CfAPYNKkSQQEBDBx4kS2bdtGr169aNWqFXv3mr7ce/fupWfPnnTq1ImePXty6tSpIu1bvnw5o0ePZsSIEQwaNIjk5GT69+9P586d6dChAz/++KPF5nbt2jF58mTat2/PoEGDSEtLy1WW0WgkLCyM119/3XIsKS2Lv379iUemPG8aqMfUChwxYgRg+rEHBwfTvn17Zs6cWaCNOfenXLt2LRMmTADg+vXr/OMf/yAkJISQkBB27dqV79qkpCQiIiLo2NG0DGfWrFlMmTKFQYMG8eijj+ZqkV6/fp0PP/uQNZvWMOWJKTRt2pQbN24AYDAY8t372rVrCQ8PZ9y4cQQFBZGWlkbv3r3Ztm1bqf8Ikmq0MW3icf+nkBQDnw+E7ydXy4BWM8uB0DzHZgC/ikgr4FdzGmAI0Mr8mAJYNwsjInZ5uLi4SF6OHz+eK93niz7yxcEvREQkU58pfb7oIysOrxARkZTMFOnzRR9ZdWSViIjEp8VLny/6yPfHvxcRkesp16XPF31kw8kNIiISkxSTr768hIeHS0BAgCQnJ0tSUpL4+/vLgQMH5Pz58wLIn3/+KSIijz32mLz77rsiItK0aVO5fv26pYzs9Pnz58XBwUEiIiLEYDBIUFCQPPbYY2I0GuWHH36QUaNGiYhIQkKCZGVliYjI1q1b5YEHHhARke3bt8uwYcPy2fjFF19I48aN5ebNmyIikpWVJQkJCaZ7vn5dWrRoIUaj0VL/wYMHRURk9OjRsmKF6b3r06eP7NmzR8aMGSP/+c9/cpUfeS1Jblw4Lk0aNZDkRFO5U6dOtVybXa9er5c+ffrI4cOHLWXu27dPREQ8PDwkMTFRRES+++47CQsLExGRsWPHyh9//CEiIhcuXJC2bdvmu7/ffvvN8h6IiMycOVM6d+4sqamp+d6Xp59+WubMmSMiIps3bxYg13tf2L1n25nNgAEDJDw8PJ8teb+PRbF9+/bbifQkkW2zRWbXFflPA5Htc0UyUqwuqzIApEgxv3HADziaI30KaGh+3RA4ZX79KTC2oHxFPbSWVw7+/PNP7r//fjw8PPD09OSBBx7gjz/+AKBJkyb06tULgEceeYQ///yz0HI+2P8Bu2N206xZMzp06IBOp6Nt27b079/fJBLXoQNRUVEAJCQkMHr0aAICAnj++ec5duxYsXYOHDgQHx+TcqmI8OqrrxIYGMiAAQO4fPkysbGxADRr1oygoCAAunTpYqkT4IknniAgIIDXXnvNcixTbyA9IwMfxwxCB97L/zZuQq/Xs3HjRkaNGgXAmjVr6Ny5M506deLYsWMcP37cyncXtm3bxrRp0wgKCmLkyJEkJiaSlJSUK09MTAx16+Ze+Dty5Ejc3PIvBP/zzz8ZM8a0N+SgwYOoVev2rGhR956XevXq5epKlhoXT5MK67S90Gog7JgDi7vC0e/BTkHhFYT6IhIDYH6uZz7eGHLtUBNtPlYkFXq2cceEHZbXTg5OudLuTu650t6u3rnSddzr5EpbM/snRXyx8g4MFzaVLkp4b997TG4/GReX28GLOp3OktbpdJZuyhtvvEG/fv1Yv349UVFRVkVpe3jcnmFbuXIl169fZ//+/Tg5OeHn52eJT8pZv4ODQ65uY8+ePdm+fTsvvviiJRI+LiXLMlD/8NhxLP70M3x8fAgJCcHLy4vz588zb9489u3bR61atZgwYUKBsVA535uc541GI3v27CnQEWXj5uaWr8yc95uT7M/rrd/fYvWx1Qi3P7+i7j0v6enpRdpUYmr5mbT9o/6EzTNg7eOwdymEzoVGQWVfX/niqJTKOUOzRESWlLCsgn5MxXp5q1peSqlQpdQp84DajALOv6CUOm4ebPtVKdXUmnIrGvfccw8//PADqamppKSksH79enr3NkkOX7x4kT179gDw7bffcvfdJm0nLy+vXK0HJYqoKVE81cE6+eCEhAQaNzb9ySxfvvyObU5ISKBevXo4OTmxfft2Lly4YNV1EydOZOjQoYwePRq9Xo+IcCs1gzoqCZw96DtgMAcOHGDp0qU8/LBJxSExMREPDw+8vb2JjY1l8+bNBZZdv359Tp06hdFoZP369ZbjgwYN4sMPP7SkDx06lO/adu3acfbsWavu4e6772bNmjW0qdOGVg6tiE8sfqOTvJ8XwOnTp2nfvr1VdZYIv7vhid9hxAK4cQaW9IV1T8C1k7ar0/boRSQ4x8MaxxWrlGoIYH6+Zj4eDbl2gvMFim0KF+u8lFIOwGJMg2r+wFillH+ebAeBYBEJBNYCNthgz/Z07tyZCRMm0LVrV7p168akSZPo1KkTYPpRffnllwQGBhIXF8eTT5oCI6dMmcKQIUMsA/YADjoHXB2tW9f38ssv88orr9CrVy8MhjuP1B43bhzh4eEEBwezcuVK2rZta/W1L7zwAp07d2b8+PEkpGbiYkjFiSxwr4ODgwPDhw9n8+bNlgHyjh070qlTJ9q3b8/jjz9u6UbnZe7cuYwePZp7772Xhg0bWo4vXLiQ8PBwAgMD8ff355NPPsl3bdu2bUlISMjnYApi5syZ/PLLL8wdPxe/k340rNew2CVJEyZMYOrUqZYB+9jYWNzc3HLZaRN0DtBlAjx7wBQfdvxH+KgbfDMGLv5l27orDhuAMPPrMODHHMcfNc86dgcSsruXRWLFoFsPYEuO9CvAK0Xk7wTsKq5cawbsKwrnz5+X9u3bW5V3Z9ROmbF1hiSmJ+Y6nj2AXVE5fz1ZEi6fEmNMhIjRUOrySnO/8+fPl6VLlxabLz093TLZsWvXLvHv5l+iuj777LMCz5V4wN4akm+I/DZHZG5TkZk1RD4fLHJys4ih9O99eUAxA/bAt0AMkIWpZTURqI1plvGM+dnHnFdhaiBFAkcwNYSK9U3WdBvvdDBtIlBwf6IacCDmAPP/mo+TjfYwtAWZeiNp6el4kYJy8wE7a5A9+eSTucasCuPixYuEhITQsWNHRn84mshhkaRlFT62VRA1a9YkLCys+IxljUdt6PcKPH8MQt+GhGj49mH4uCcc+hYMWeVvUxkiImNFpKGIOImIr4h8LiI3RaS/iLQyP8eZ84qIPC0iLUSkg4hYFe1crCSOUmo0MFhEJpnT44GuIpIvilAp9QgwDegjIvnCy5VSUzDFceDo6Nhl69atuc57e3vTsmXlV6k0ijGfCKHBYMDBoWIus4lPN+KUeYsG6hbJHnchutLvsFze93sw9iDhMeGM9R9bZhtznD17loSEBKvyJicn54pvu1OUUU+9a3/S5NI6PFMukO5Sh2jfUcQ0HIjB0T6S20XRr18/u0viWOO8egCzRGSwOf0KgIj8N0++AcAiTI7rWr6C8lCQnteJEydo1872ewDag4oqiSNi2ja+hVzAydmtzCSOK+r93gl38n0sMz0vETizFf58Hy7uBtea0HWK6eFZMbTjoWLoeVnTP9gHtFJKNVNKOQNjMA2wWVBKdcIUaDbSGsdVlXlhywusOrrK3mZYTXKGHhdDMk7orV7HWFFJyUxha+TW4jNWZJSC1oPg8c2miP2mvWDnOzC/nUk/7PQWMJT9srDKSLHOS0T0mLqCW4ATwBoROaaUmq2Uyt7L/l3AE/hOKXVIKbWhkOKqNCLC5rObORJ7xN6mWE2ceR2j6BxN+xNWYpYdXMagrwcRGRdpb1PKhiZdYew38PQ+6PYEXNgD3zwE77eHrTNNYRfVmAolA611G8uXLIORyJg42uguoTzrl+muQPa435ikGI5fP07vpr1xdij9uJ1duo1Foc+EM1vg4Eo48wuIAZp0N2mLtb+/XOV4Kku3UaMICpLEiY8vPliyOMpDEudWaia1lDmeyt2+XcacUjclpaFXQ/o371+o4zp06BCbNm0qVR12xdEZ2o2Af66CF47DgP8z6YlteAbmtYYfnoKoXdVmCZLmvEpJTuf12/nf8JnkQ6ZTZrnUXRolBDFL39RWSSiXGnbVYS9I6qakXEq4xPw980nX51+2VOmdV068GsDd/4Jp+0xjYx0eNAW+Lh8KizrDltdg/3KTM0uKrZIOTXNeeSiNJM7V5Kus2r22sk0aAAAaX0lEQVSKlIQUyzUVVRInLdOAc2YiE597jdffyR/pvn//fvr06UOXLl0YPHgwMTEx6PV6QkJC2LFjBwCvvPKKZWG3n58f06dPp2vXrnTt2pXISNO4U2EyOEVJ3cyaNYuwsDAGDRqEn58f69at4+WXX6ZDhw6EhoaSlZVVqI0RsRG8+MuLTHxjIl27dqV169b88ccfZGZm8uabb7J69WqCgoJYvXp1ib4fFQ6lTGNjIxfBv0/DfZ9AjcamNZT/e87kzN5rDXPvMi1L+n4y7HjbtEg8JgIyU+x9ByXHmkhWWzyKi7CfteGoPPTJ7jJ9zNpwtJB4YRPVSRJn/c/b5cFRQ+Wt6c+IGI256sjMzJQePXrItWvXRERk1apV8thjj4mIyNGjR6Vt27byyy+/SFBQkGRkZFjuO1te58svv5TBgweLSOEyOEVJ3cycOVN69eolmZmZcujQIXFzc5NNmzaJiMh9990n69evL9TGtKw06Taom7zwwgsiIrJx40bp37+/5b17+umni/wO5MSmEfa2xqAXiYsSObNV5K9PRH56UeTLUSLz25si+nM+3msn8vWDIjvniUTtFslMK7Z4rJDEsfWjQqtKlDc5JXEAiyTOyJEj80niLFy4kH//+99FlpctiQMUKYkTFhbGmTNnUEpZWhVFUZAkzs6dO9HpdFZJ4ojAjOenMX5EH1577ZV8OwOdOnWKo0ePMnCgaWdkg8FgWfvXvn17xo8fz4gRI9izZw/OzrfHl8aOHWt5/te//gWYZHByyubklMEpTOoGYMiQITg5OdGhQwcMBgOhoSZdu+z3rjAbXR1dcc1w5YEHHsh339UKnQPUamp6tByQ+1xmKsSdg5tn4eYZuHEWrhw0TQIAODhDo87QtAfc1QOadAO3muV/D8VQYZ3XzBE2XOVfCFJKSZwJP0wg0T/Rkq6okjh6o5HuwR3ZvjucF3Ue5F1CLiK0b9/eoqKRlyNHjlCzZk2Lk8wm53uS/booGZzCpG5y2q7T6XBycrKUl/3eFWVjqlsq751+jyYBTXB3cLeJXHalxtkdGgSYHjlJuQmX/oKLe0xhGbsXmYJlUVDPH+7qDk17mp4rANqYVw5KK4lzLeUaRqc723zBHpI4eoMwdexwhg4ewOgx4/L9uNu0acP169ct95uVlWURSVy3bh03b95k586dPPvss7lmVrPHkVavXk3Xrl0B62RwSkJRNooSfrnyCyeun8h1TUFyOBo58Kht2pNy0H9g8q8w4xKE/QT9XgWv+hCxGr6faIozqwBozisHpZXE2TRuEzUP31nzurwlcbIMRnSixxEjL/x7ukUSx5hjxxtnZ2fWrl3L9OnT6dixI0FBQezevZsbN24wY8YMPv/8c1q3bs20adN47rnnLNdlZGTQrVs3FixYwH//a1o9Zo0MTkkozEYA91R3tg3cxuCWg3Nd069fP44fP161BuxtibO7aYfwPi/D+PUw/QJM+d0kplgB0IJUrSAqKorhw4dz9OjR4jMXQkUJUr2ZkoFz/Hk8HAzo6vuX2U7Yfn5+hIeHU6eOadOOinK/paHCBalWILQg1SrEtnPb6Lu8L1HxUfY2pUiS0zLxUOkoN+8yc1wVkTM3z9B3eV/+uPCHvU3RsBEVdsC+IuHn51dsq0tv1GMQQ5nJsdgCo1EgIwmdkjJfx1jRZvTqe9YnOTOZlKxKHMekUSSa8yojQluGEtoy7zZ1FYvkDD2epCLoUM52bfHbnBouNQifUjY7eGtUTLRuYzUiMT0LL9JMC3jtrJZaXogIWZVclVSjYKrHN7gcuH/1/UzfOt3eZhSKiJCRloqz0qNca9jbnHLhStIVGs1vxIqIFfY2RcMGaM6rjGjk2Yi6HhVH6TIvaVkG3MU8/uNSPZxXQ8+GjGg9gua1mtvbFA0boDmvPCxcuJB27doxbtw4NmzYwNy51sW0LB62mH/3LHy5kK221po1axbz5s0rNl9iuh4vUhFHV5O0io0ZMGBA8ZlKQXx8PB999FGReZRSLBmxhL5+fW1qi4Z90Abs8/DRRx+xefNmmjVrBpjW35UH2YtNdTrb/J+kpGVQX6WjXOvbpPxssjfe2LZtm03ryXZeTz1V/Oa+cWlxGIyGCt0y1rhztJZXDqZOncq5c+cYOXIk77//PsuXL2fatGkAjBo1iq+++gqATz/9lHHjxgEQGRlJ8MPBuLzsQufQzpw8adoF+fz58/To0YOQkBDeeOONAuvLlq156qmn6Ny5M5cuXeLJJ58kODiY9u3bM3PmTEtePz8/Zs6caZG+ya4nJ0uXLmXIkCH5tra/HBPLtMfG0nXoI4TcO8IiS/Pss88ye/ZsALZs2cI999yD0Wi0bMzau3dvWrduzU8//QSYHNNLL71ESEgIgYGBfPrpp4ApQLNfv37885//tCxEz25p7tixgz59+vDQQw/RunVrZsyYwcqVK+natSsdOnSwSjrn8ccfp2/fvjRv3tyi+TVjxgwiIyMJCgripZdeKvQzTclMoeF7DVnw9wIAzt06x8kbt9+7yLhITt24LUN05uYZTt88XWh5GhUIe8lZFLvp7KbpIsuGlu1j0/RipT5yStzklFC5evWqtGjRQnbu3CmtWrWySNLce++98t3u72Ts2rGy8feN0q9fPxERGTFihHz55ZciIvLhhx+Kh4dHvrrOnz8vSinZs2eP5Vh2uXq9Xvr06SOHDx+22LVw4UIREVm8eLFMnDhRREzyMe+++64sWrRIRowYIenp6fnqeWD0Q7Jx3bdivHJYLkRFWWRpUlJSxN/fX3777Tdp3bq1nD17VkREwsLCZPDgwWIwGOT06dPSuHFjSUtLk08//VTeeustETFt+NqlSxc5d+6cbN++Xdzd3eXcuXOWOrPvd/v27eLt7S1XrlyR9PR0adSokbz55psiIvLBBx/Ic889JyJFS+f06NFD0tPT5fr16+Lj4yOZmZl3tBHwkvAlcijmkIiIjPp2lHT8uKPl3JCvh0jIkhBL+t4v75W7l90tIpVcEsfGoEniVB7q16/P7NmzLQoQPj4+JCcns3v3bm4+eROAV3mVjAzTdpW7du3i+++/B2D8+PFMn17wTGTTpk3p3v32Kv01a9awZMkS9Ho9MTExHD9+nMDAQIBcMi/r1q2zXLNixQp8fX354YcfcHLKv9nt79t/4+yxw7yq04GDs0WWxsvLi6VLl3LPPffw/vvv06JFC8s1Dz30EDqdjlatWtG8eXNOnjzJL7/8QkREBGvXrgVMi8LPnDmDs7MzXbt2tXS18xISEmJpibVo0YJBgwYBJnmb7du3A0VL5wwbNgwXFxdcXFyoV69ePjWL4pjcZbLl9St3v0Jq1m3Z7jfueYNMw23l29l9Z2OUO1tcr2EfKq7zGlIxFn/m5MiRI9SuXZsrV64AJrmXmjVrcvDgwQIlcgo6lpecsjDnz59n3rx57Nu3j1q1ajFhwgRyrv/MlolxcHDIpQQREBDAoUOHiI6OzudADEbBaDDy14YvcGvYOp9Wfd57Ksx2pRQiwqJFixg8OPeC5x07dlglbwOFSwMVJZ2TV9qnNBI33Xy75Ur3aNIjV7rXXSbNtpM3TvLsrmdZ7bcaHzefEtenYTu0MS8r2bt3L5s3b+bgwYPMmzeP8+fPU6NGDZPg4LwOjFs3DhHh8OHDAPTq1YtVq0z7N65cudKqOhITE/Hw8MDb25vY2Fg2b95s1XWdOnXi008/ZeTIkfmcUHKGnr733M2Hy1dbQiSyZWkuXLjAe++9x8GDB9m8eTN///235brvvvsOo9FIZGQk586do02bNgwePJiPP/7YIph4+vRpUlLKZvnNnUrn2FreJsuQxcEbBzl27ZjN6tAoHZrzsoKMjAwmT57MsmXLaNSoEe+99x6PP/44IsLKlSvJOpHF71/9Tvv27S0a8gsWLGDx4sWEhIRYvWV8x44d6dSpE+3bt+fxxx+3KLdaw9133828efMYNmwYN27csBxPSsvig7deJvzIaQI7dbHI0ogIEydOZN68eTRq1IjPP/+cSZMmWVp6bdq0oU+fPgwZMoRPPvkEV1dXJk2ahL+/P507dyYgIIAnnniizIT+7lQ6p3bt2vTq1YuAgIAiB+xLSof6Hfh1+K/0btq7zMuuLiilopRSR8x7uYabj/kopbYqpc6Yn2uVuAJrBsaAUOAUcBaYUcD5e4ADgB540Joyix2wr2IkJiaWe51Go1FOXo4T4+UDIglXrL4uLCxMvvvuu1LVbY/7LWuOHz8uRqNRdl/cXWxebcC+QL8RBdTJc+ydbB8CzADeLq6cwh7FtryUUg7AYmAI4A+MVUr558l2EZgAfFNiL1pJMRgNFXaANy3LgJsxBQVQTZYElTUrj6yk57Ke/B71u71NqSqMAr40v/4SuK+kBVkzYN8VOCsi5wCUUqvMBlimhkQkynyuYv6KbcjOCzsJXRnK9rDt9GzS097m5CIxTY+XSkN0jignd6uvK4kcdVXlQf8HSdenV7jPtpIgwC9KKQE+FZElQH0RiQEQkRilVL2SFm6N82oMXMqRjga6FZK31IiIVbN0FYWGXg35V7d/0axmwWEC9iQpPZO6Kg3lUrWFB22BuVuDq6MrkzpPsrM1FRLH7HEsM0vMziknvUTkitlBbVVK5Y+sLo0BVuQp6FtfIu1opdQUYAqAo6OjZfPSbDw9PYmOjsbb27vSOLDGLo15vfvrAEXOfhkMhnLd/CHLKKisNBx0BtLECX05bzxR3vdblogICQkJpKSkWL6jEfERfB71Of8N+C/ujvlbscnJyfm+z1UcvYgEF5VBRK6Yn68ppdZj6sXFKqUamltdDYFrJTXAGucVDTTJkfYFrhSSt0jMnnkJmDTs82p+Z2VlER0dzeXLl0tSvF1IyUrBzdENXTH6WOnp6bi65t1kzHYkZ+gxpsWTRRrUcAaddTOeZUV5329Z4+rqSseOHS1Bvx6XPfjk8if4dfTDv27eId/qp2FfHEopD0AnIknm14OA2cAGIAyYa37+saR1WOO89gGtlFLNgMvAGOCfJa2wKJycnAqN0q6o9P+qP1mGLHY+trPIfDt27LDsRFQePLpsL69FP0dr33qox38ut3qzKe/7tTUhjUM4+tTRYv+kNCzUB9abe1COwDci8rNSah+wRik1EdNE3+iSVlCs8xIRvVJqGrAFcACWicgxpdRsIFxENiilQoD1QC1ghFLq/0SkYmzuZmMeD3rc3ibkIzlDz9nISNo4RUKrcfY2p8qgUzoyDZlsjdzKsNbD7G1OhcY8wdexgOM3gf5lUYdVy4NEZBOwKc+xN3O83oepO1ntGBdY8ZzDn2eu05ODpkSrQfY1poqxeO9iXvjlBQ5PPUxg/UB7m1OtqbhrGysBGfoM4tPjqedRr0JNMGw7cY2BTocRr4ao+gHFX6BhNVO6TKFtnbZ0qNfB3qZUe7QOfCk4EHOABu81YPNZ69YglgcGo/DHiSvcozuCajVQC5EoYzycPRjSaohlobqG/dCcVym4y/suFg1ZRKcGFWdg+tClePzSjuFmTNG6jDbku2Pf0enTTrnkdTTKF63bWAoa12jMtK7T7G1GLn49Ecu9jocQnROqWR97m1NlaeDZgLoedbmVdgv3O1i9oFF2aM6rFFxMuIibo1uF0kb/9cQ1lrocQTXpoa1ntCG9m/Zm6/it9jajWqN1G0vB1J+mMvjrwcVnLCcuxaWSFHueu/RRWpexnIhLi+ObI9VOj6BCoLW8SsHLvV4mJbNsxPjKgl9PxNLXwSSGqDmv8uH9Pe8zd9dcVoZYJzipUXZozqsUVLT9AH89eY2pbkfA8y6o09re5lQLXur1EqPbjybuRFylExWo7GjdxhKSrk8n/Eo4yZnJ9jYFgKT0LA6cu0qIMQJaDdZCJMqJGi41LMGqi/Yu4oHVD2gzkOWE5rxKyPHrxwlZGsIvkb/Y2xQA/jhzg05yAmdjutZltBMKhVIKN0fTJiJaHJht0ZxXCWleqzk/PPwDvZpYrzNvS7Ydj2WwcwTi6Ap+d9vbnGrJM92eYe3otSiluJl6ky5LurAjaoe9zaqyaGNeJaSma01GtR1lbzMAU1T99lPXmOEcgbqrNzhrcUf2InvM61rKNZRS1HYzbTWnjYeVPZrzKiERsREYxUhQgyB7m8KBi7eokXaJei6XoNWz9jZHA2hXtx3hk8MtDuulrS8hIswbNE9zYmWE1m0sITN3zGTcuoqhKLH5yFX6W0IkBtrXGA0L2U5KRMjQZ5CmT9McVxmiOa8S8t/+/2XpiKX2NoPP/jjHsl3nebDGcajdCnwql5hjdUApxaKhi1g8dDEAp26cwuu/Xvxx4Q8Atp3bhuccT/6ONm36u/H0RjzneHLoqmnj3fUn1uM5x5Pj10173qw+uhrPOZ6cjTsLwIrDK/Cc48nFhIsAfH7gczzneHI1+SoAH+/7GM85nsSlxQGw4K8FeM7xtMyUv7PrHTzneJJlMG0m/P92/j8853ha7J+5fSY+b9/eNXzGthk2eJfuHK3bWELa1mlr1/pFhLmbT/LpznPc196bdlEREKhtFFGRyTke9kSXJ2jk1QiAJjWaMDV4KvU96wPgV9OPqcFTqeNeB4AWPi2YGjyVWq6m/Vlb1W7F1OCpeLt4A9CmThumBk/Fy9kLAP+6/kwNnmpZcxlQL4CpwVNxdTTJcnds0JGpwVNx0pkkrjs37MzU4KkWldjgRsFMDZ5qsbubbzfS9emWdEXZSUnZazrX1dVVsndnrmyk69PZfGYz3X2709CroVXXlKXGeZbByPTvI1h34DLjuzdlVttLOKwaA+N/gBb9yqSO0lLdNN2r2/0qpVJFxMOeNmjdxhIQGRfJA2sesMs0eGqmnilfhbPuwGVeGNia2UOa4rD/C3DygKYV4x9RQ6M80LqNJaCFTwvCJ4fjV9OvXOu9lZLJY8v3EREdz5z7O/BP7yOw+D5IjIZ+r4GjS7nao6FhTzTnVQJcHV3p0qhLudZ5OT6NRz//m0u30lh2f0P6Rs6AUxuhXnsY/QU06Vqu9mho2BvNeZWAXRd3cSv9FsNbDy+X+k5dTSJs2V7SMzPY2u0ITbcuAAQGzobuT4GDU7nYoaFRkbDbmFemdyZv/PaGJd3mwzbM/n22Jd1sQTPm/jnXkvad78v8PfNN1xoy8Z3vy6K/FwGQnJmM73xfPg3/FIBbabfwne/LsoPLALiafBXf+b6sjDDJllxKuITvfF/WHFsDwLlb5/Cd78v6E+sBOHH9BL7zfdl0xrRhUkRsBL7zfdkaaRKf++38bzy7uXyCQfdFxTH6k920M5zir9pv0XT/HGjWG57+G3o9pzkujWqL3VpeOr2OdnXbWdIDmg3IFX4wqPkgWte+LesS2jKUlj4tTdcqHaEtQ2leqzkAjjpHQluGWsagnBycCG0ZSlPvpoCpmxfaMhTfGqbd2dyc3AhtGUpjr8YAuDu5E9oy1DJ17eXiRWjLUOp7mKaua7jUILRlKPU86gEwtsPYXLbZiq3HY3nlmz94y3UtI/U/ozIbwkMroN0ITTVCo9pjVaiEUioUWIBp09nPRGRunvMuwFdAF+Am8LCIRBVVZmUOlSgJdzqVvnrvBXb9uIRZziupRQKq6xNw72vg4mU7I8uQ6hY6UN3utyKEShTb8lJKOQCLgYFANLBPKbVBRI7nyDYRuCUiLZVSY4C3gYdtYXBVQERIytBzIymDmymZ3ExIJjHhJqkJN0hPvEl6wg06x6xiodMRDPWDUCMXQCP7r6HU0KhIWNNt7AqcNW/fjVJqFTAKyOm8RgGzzK/XAh8qpZQU0awTo5Et3y4qkdHWkbtqlSeNFHBegSogv+WYkttpoyBGPUajAYwGxPzAaEAk5zEjiIH0pFts27MI56xEXA1JeEkKNVQK7UjBU+VvgWY4uaMf+DaO3SaDzqEU74OGRtXEGufVGLiUIx0NdCssj4jolVIJQG3gRuEVZzH41Ot3Zm0lxYCODJxIc6hBhpMXeg9vjC710bt5E+9ei1TP2rjV8MGtRm0cPXzAtSYudVqBu0/xhWtoVFOscV4FjQznbVFZkwel1BRgiiX9f4nVSS/XEa7r7W1EOeIIaPdbdXGztwHWOK9ooEmOtC9wpZA80UopR8AbiMtbkIgsAZYAKKXCRSS4JEZXRrT7rdpUx/u1tw3WxHntA1oppZoppZyBMcCGPHk2AGHm1w8CvxU13qWhoaFRWopteZnHsKYBWzCFSiwTkWNKqdlAuIhsAD4HViilzmJqcY2xpdEaGhoaVgWpisgmYFOeY2/meJ0OjL7DupfcYf7Kjna/VRvtfssZu+l5aWhoaJQGTc9LQ0OjUmJX56WUelcpdVIpFaGUWq+UqmlPe2yNUmq0UuqYUsqolKqyM1NKqVCl1Cml1FmlVMUQPLcRSqllSqlrSqmj9ralPFBKNVFKbVdKnTB/l5+zly32bnltBQJEJBA4DbxiZ3tszVHgAWCnvQ2xFTmWkw0B/IGxSil/+1plU5YDofY2ohzRAy+KSDugO/C0vT5fuzovEflFRLID+/7CFENWZRGREyJyyt522BjLcjIRyQSyl5NVSURkJwXENFZVRCRGRA6YXycBJzCtsCl37N3yysnjwGZ7G6FRagpaTmaXL7eGbVFK+QGdgL/tUb/N9byUUtuABgWcek1EfjTneQ1Tc3Slre2xNdbcbxXHqqViGpUbpZQn8D3wLxFJtIcNNndeIjKgqPNKqTBgONC/KkTlF3e/1QBrlpNpVGKUUk6YHNdKEVlnLzvsPdsYCkwHRopIdVqkXZWxZjmZRiVFmXbO/Rw4ISLz7WmLvce8PgS8gK1KqUNKqU/sbI9NUUrdr5SKBnoAG5VSW+xtU1ljnoDJXk52AlgjIsfsa5XtUEp9C+wB2iilopVSE+1tk43pBYwH7jX/Zg8ppYbawxAtwl5DQ6NSYu+Wl4aGhkaJ0JyXhoZGpURzXhoaGpUSzXlpaGhUSjTnpaGhUSnRnJeGhkalRHNeGhoalRLNeWloaFRK/j/nHzjxhfYOjgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ax = result_flat.groupby(level=0).idxmax().str[1].plot(label='optimal rank value', ls=\":\", legend=True,\n", + " secondary_y=True, c='g')\n", + "result_flat.groupby(level=0).max().plot(label='optimal rank experiment', legend=True)\n", + "result_flat.xs(best_rank, axis=0, level=1).plot(label='fixed rank experiment', legend=True, title='MRR',\n", + " figsize=(4.3, 2), ylim=(0, None), xlim=(-2, 2), grid=True);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Clearly, the values are capped on the left half of the graph, which leaves the room for further improvement. I have performed experiments with a higher threshold and was able to achieve a higher top score with a bit lower value of the scaling parameter. If you want to see this, simply rerun the grid search experiment with a higher value of the `max_rank` variable. Be prepared that it will take a longer time (hint: reduce the search space for the scaling parameter). Anyway, the key conclusion doesn't change - **even a simple scaling factor can be advantageous and allows to outperform the standard model**. This conclusion is supported by many other experiments in the original paper, which we haven't run here." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Bonus: Double-scaled SVD" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " The main change in the model is basically enclosed in the following line\n", + "```python\n", + "scaled_matrix = sparse_normalize(svd_matrix, self.col_scaling, 0)\n", + "```\n", + "which scales columns of the rating matrix. However, there's nothing really special about this particular type of scaling and we could also scale rows instead of or in addition to that. Scaling rows would help to control the contribution of users with either too high or too low number of rated items. The entire code for defining the double-scaled model is listed below:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```python\n", + "class DScaledSVD(ScaledSVD):\n", + " def __init__(self, *args, **kwargs):\n", + " super().__init__(*args, **kwargs)\n", + " self.row_scaling = 1 # PureSVD config\n", + " self.method = 'DScaledSVD'\n", + " \n", + " def get_training_matrix(self, *args, **kwargs):\n", + " svd_matrix = super().get_training_matrix(*args, **kwargs)\n", + " return sparse_normalize(svd_matrix, self.row_scaling, 1)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that running the grid search experiment with two scaling parameters, `row_scaling` and `col_scaling`, will take more time. In my experiments with Movielense data and the value of rank limited by 150 from the above there was a very weak improvement, which wasn't worth the efforts. This however, may change with higher values of ranks or at least with another data. I'll leave verifying this for the reader." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Instead of conclusion" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Being a researcher myself, I'm often involved in some sort of \"redoing\" the work that was already done by someone else. Leaving the reproducibility aspect aside, there are many other reasons why it can be useful, e.g., it may help to understand presented ideas better or to see if there're any subtle moments in an original work that are not evident at first glance. It helps to create a playground ready for further exploration and may even lead to new ideas. I hope that Polara will help you with this as it helps me in my own research." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + }, + "toc": { + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From f305e37d3bf327ac5064f9b3a9fc36ebd4002d27 Mon Sep 17 00:00:00 2001 From: Evgeny Frolov Date: Tue, 21 Aug 2018 14:31:24 +0300 Subject: [PATCH 13/13] update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bc2b0f8..27d4438 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ svd = SVDModel(data_model) svd.build() svd.evaluate() ``` +Several different scenarios and use cases, which cover many practical aspects, can also be found in the [examples directory](/examples). ## Creating new recommender models Basic models can be extended by subclassing `RecommenderModel` class and defining two required methods: `self.build()` and `self.get_recommendations()`. Here's an example of a simple item-to-item recommender model: