Skip to content

Commit

Permalink
Adding include_paths to the docs
Browse files Browse the repository at this point in the history
  • Loading branch information
seperman committed Aug 26, 2022
1 parent ca8e58e commit b3a4947
Show file tree
Hide file tree
Showing 13 changed files with 298 additions and 60 deletions.
22 changes: 22 additions & 0 deletions conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,28 @@ def nested_a_result():
return json.load(the_file)


@pytest.fixture(scope='class')
def nested_a_affected_paths():
return {
'root[0][0][2][0][1]', 'root[0][1][1][1][5]', 'root[0][2][1]',
'root[1][1][2][0][1]', 'root[1][2][0]', 'root[1][2][0][1][5]',
'root[1][0][2][2][3]', 'root[0][0][1][0][0]', 'root[0][1][0][2][3]',
'root[0][3][0][2][3]', 'root[0][3][1][0][2]', 'root[1][1][1][0][0]',
'root[1][0][1][2][1]', 'root[1][0][2][1][2]', 'root[1][3][0][2][3]',
'root[1][3][1][0][2]', 'root[1][2][0][2]', 'root[1][0][2][0][1]',
'root[0][3][2][0][1]', 'root[0][3][2][1][0]', 'root[1][3][1][1]',
'root[1][2][1][1][0]', 'root[1][2][1][0]', 'root[1][0][0][0][2]',
'root[1][3][2][1][0]', 'root[1][0][0][1][1]', 'root[0][1][2][0]',
'root[0][1][2][1][0]', 'root[0][2][0][1][2]', 'root[1][3][0][1]',
'root[0][3][1][1]', 'root[1][2][0][0][2]', 'root[1][3][2][0][1]',
'root[1][0][1][0]', 'root[1][2][0][0][0]', 'root[1][0][0][0][1]',
'root[1][3][2][2][2]', 'root[0][1][1][2][1]', 'root[0][1][1][2][2]',
'root[0][2][0][0][2]', 'root[0][2][0][0][3]', 'root[0][3][1][2][1]',
'root[0][3][1][2][2]', 'root[1][2][1][2][3]', 'root[1][0][0][1][2]',
'root[1][0][0][2][1]', 'root[1][3][1][2][1]', 'root[1][3][1][2][2]'
}


@pytest.fixture(scope='class')
def nested_b_t1():
with open(os.path.join(FIXTURES_DIR, 'nested_b_t1.json')) as the_file:
Expand Down
10 changes: 7 additions & 3 deletions deepdiff/deephash.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
convert_item_or_items_into_compiled_regexes_else_none,
get_id, type_is_subclass_of_type_group, type_in_type_group,
number_to_string, datetime_normalize, KEY_TO_VAL_STR, short_repr,
get_truncate_datetime, dict_)
get_truncate_datetime, dict_, add_root_to_paths)
from deepdiff.base import Base
logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -123,6 +123,7 @@ def __init__(self,
hashes=None,
exclude_types=None,
exclude_paths=None,
include_paths=None,
exclude_regex_paths=None,
hasher=None,
ignore_repetition=True,
Expand All @@ -146,7 +147,7 @@ def __init__(self,
raise ValueError(
("The following parameter(s) are not valid: %s\n"
"The valid parameters are obj, hashes, exclude_types, significant_digits, truncate_datetime,"
"exclude_paths, exclude_regex_paths, hasher, ignore_repetition, "
"exclude_paths, include_paths, exclude_regex_paths, hasher, ignore_repetition, "
"number_format_notation, apply_hash, ignore_type_in_groups, ignore_string_type_changes, "
"ignore_numeric_type_changes, ignore_type_subclasses, ignore_string_case "
"number_to_string_func, ignore_private_variables, parent "
Expand All @@ -160,7 +161,8 @@ def __init__(self,
exclude_types = set() if exclude_types is None else set(exclude_types)
self.exclude_types_tuple = tuple(exclude_types) # we need tuple for checking isinstance
self.ignore_repetition = ignore_repetition
self.exclude_paths = convert_item_or_items_into_set_else_none(exclude_paths)
self.exclude_paths = add_root_to_paths(convert_item_or_items_into_set_else_none(exclude_paths))
self.include_paths = add_root_to_paths(convert_item_or_items_into_set_else_none(include_paths))
self.exclude_regex_paths = convert_item_or_items_into_compiled_regexes_else_none(exclude_regex_paths)
self.hasher = default_hasher if hasher is None else hasher
self.hashes[UNPROCESSED_KEY] = []
Expand Down Expand Up @@ -327,6 +329,8 @@ def _skip_this(self, obj, parent):
skip = False
if self.exclude_paths and parent in self.exclude_paths:
skip = True
if self.include_paths and parent not in self.include_paths:
skip = True
elif self.exclude_regex_paths and any(
[exclude_regex_path.search(parent) for exclude_regex_path in self.exclude_regex_paths]):
skip = True
Expand Down
9 changes: 5 additions & 4 deletions deepdiff/delta.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
from deepdiff.helper import (
strings, short_repr, numbers,
np_ndarray, np_array_factory, numpy_dtypes, get_doc,
not_found, numpy_dtype_string_to_type, dict_)
not_found, numpy_dtype_string_to_type, dict_,
)
from deepdiff.path import _path_to_elements, _get_nested_obj, GET, GETATTR
from deepdiff.anyset import AnySet

Expand Down Expand Up @@ -70,11 +71,11 @@ def __init__(
serializer=pickle_dump,
verify_symmetry=False,
):
if 'safe_to_import' not in set(deserializer.__code__.co_varnames):
if hasattr(deserializer, '__code__') and 'safe_to_import' in set(deserializer.__code__.co_varnames):
_deserializer = deserializer
else:
def _deserializer(obj, safe_to_import=None):
return deserializer(obj)
else:
_deserializer = deserializer

if diff is not None:
if isinstance(diff, DeepDiff):
Expand Down
56 changes: 39 additions & 17 deletions deepdiff/diff.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,15 @@
type_is_subclass_of_type_group, type_in_type_group, get_doc,
number_to_string, datetime_normalize, KEY_TO_VAL_STR, booleans,
np_ndarray, get_numpy_ndarray_rows, OrderedSetPlus, RepeatedTimer,
TEXT_VIEW, TREE_VIEW, DELTA_VIEW, detailed__dict__,
TEXT_VIEW, TREE_VIEW, DELTA_VIEW, detailed__dict__, add_root_to_paths,
np, get_truncate_datetime, dict_, CannotCompare, ENUM_IGNORE_KEYS)
from deepdiff.serialization import SerializationMixin
from deepdiff.distance import DistanceMixin
from deepdiff.model import (
RemapDict, ResultDict, TextResult, TreeResult, DiffLevel,
DictRelationship, AttributeRelationship,
DictRelationship, AttributeRelationship, REPORT_KEYS,
SubscriptableIterableRelationship, NonSubscriptableIterableRelationship,
SetRelationship, NumpyArrayRelationship, CUSTOM_FIELD)
SetRelationship, NumpyArrayRelationship, CUSTOM_FIELD, PrettyOrderedSet, )
from deepdiff.deephash import DeepHash, combine_hashes_lists
from deepdiff.base import Base
from deepdiff.lfucache import LFUCache, DummyLFU
Expand Down Expand Up @@ -85,6 +85,7 @@ def _report_progress(_stats, progress_logger, duration):
DEEPHASH_PARAM_KEYS = (
'exclude_types',
'exclude_paths',
'include_paths',
'exclude_regex_paths',
'hasher',
'significant_digits',
Expand Down Expand Up @@ -119,6 +120,7 @@ def __init__(self,
exclude_obj_callback=None,
exclude_obj_callback_strict=None,
exclude_paths=None,
include_paths=None,
exclude_regex_paths=None,
exclude_types=None,
get_deep_distance=False,
Expand Down Expand Up @@ -157,7 +159,7 @@ def __init__(self,
raise ValueError((
"The following parameter(s) are not valid: %s\n"
"The valid parameters are ignore_order, report_repetition, significant_digits, "
"number_format_notation, exclude_paths, exclude_types, exclude_regex_paths, ignore_type_in_groups, "
"number_format_notation, exclude_paths, include_paths, exclude_types, exclude_regex_paths, ignore_type_in_groups, "
"ignore_string_type_changes, ignore_numeric_type_changes, ignore_type_subclasses, truncate_datetime, "
"ignore_private_variables, ignore_nan_inequality, number_to_string_func, verbose_level, "
"view, hasher, hashes, max_passes, max_diffs, "
Expand Down Expand Up @@ -188,7 +190,8 @@ def __init__(self,
ignore_numeric_type_changes=ignore_numeric_type_changes,
ignore_type_subclasses=ignore_type_subclasses)
self.report_repetition = report_repetition
self.exclude_paths = convert_item_or_items_into_set_else_none(exclude_paths)
self.exclude_paths = add_root_to_paths(convert_item_or_items_into_set_else_none(exclude_paths))
self.include_paths = add_root_to_paths(convert_item_or_items_into_set_else_none(include_paths))
self.exclude_regex_paths = convert_item_or_items_into_compiled_regexes_else_none(exclude_regex_paths)
self.exclude_types = set(exclude_types) if exclude_types else None
self.exclude_types_tuple = tuple(exclude_types) if exclude_types else None # we need tuple for checking isinstance
Expand Down Expand Up @@ -431,21 +434,24 @@ def _skip_this(self, level):
Check whether this comparison should be skipped because one of the objects to compare meets exclusion criteria.
:rtype: bool
"""
level_path = level.path()
skip = False
if self.exclude_paths and level.path() in self.exclude_paths:
if self.exclude_paths and level_path in self.exclude_paths:
skip = True
if self.include_paths and level_path not in self.include_paths:
skip = True
elif self.exclude_regex_paths and any(
[exclude_regex_path.search(level.path()) for exclude_regex_path in self.exclude_regex_paths]):
[exclude_regex_path.search(level_path) for exclude_regex_path in self.exclude_regex_paths]):
skip = True
elif self.exclude_types_tuple and \
(isinstance(level.t1, self.exclude_types_tuple) or isinstance(level.t2, self.exclude_types_tuple)):
skip = True
elif self.exclude_obj_callback and \
(self.exclude_obj_callback(level.t1, level.path()) or self.exclude_obj_callback(level.t2, level.path())):
(self.exclude_obj_callback(level.t1, level_path) or self.exclude_obj_callback(level.t2, level_path)):
skip = True
elif self.exclude_obj_callback_strict and \
(self.exclude_obj_callback_strict(level.t1, level.path()) and
self.exclude_obj_callback_strict(level.t2, level.path())):
(self.exclude_obj_callback_strict(level.t1, level_path) and
self.exclude_obj_callback_strict(level.t2, level_path)):
skip = True

return skip
Expand Down Expand Up @@ -477,12 +483,12 @@ def _get_clean_to_keys_mapping(self, keys, level):
return result

def _diff_dict(self,
level,
parents_ids=frozenset([]),
print_as_attribute=False,
override=False,
override_t1=None,
override_t2=None):
level,
parents_ids=frozenset([]),
print_as_attribute=False,
override=False,
override_t1=None,
override_t2=None):
"""Difference of 2 dictionaries"""
if override:
# for special stuff like custom objects and named tuples we receive preprocessed t1 and t2
Expand Down Expand Up @@ -1097,7 +1103,7 @@ def get_other_pair(hash_value, in_t1=True):
old_indexes=t1_indexes,
new_indexes=t2_indexes)
self._report_result('repetition_change',
repetition_change_level)
repetition_change_level)

else:
for hash_value in hashes_added:
Expand Down Expand Up @@ -1423,6 +1429,22 @@ def get_stats(self):
"""
return self._stats

@property
def affected_paths(self):
"""
Get the list of paths that were affected.
Whether a value was changed or they were added or removed.
"""
result = OrderedSet()
for key in REPORT_KEYS:
value = self.get(key)
if value:
if isinstance(value, PrettyOrderedSet):
result |= value
else:
result |= OrderedSet(value.keys())
return result


if __name__ == "__main__": # pragma: no cover
import doctest
Expand Down
44 changes: 21 additions & 23 deletions deepdiff/helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import time
from ast import literal_eval
from decimal import Decimal, localcontext
from collections import namedtuple, OrderedDict
from collections import namedtuple
from itertools import repeat
from ordered_set import OrderedSet
from threading import Timer
Expand Down Expand Up @@ -220,28 +220,6 @@ class indexed_set(set):
"""


JSON_CONVERTOR = {
Decimal: float,
OrderedSet: list,
type: lambda x: x.__name__,
bytes: lambda x: x.decode('utf-8')
}


def json_convertor_default(default_mapping=None):
_convertor_mapping = JSON_CONVERTOR.copy()
if default_mapping:
_convertor_mapping.update(default_mapping)

def _convertor(obj):
for original_type, convert_to in _convertor_mapping.items():
if isinstance(obj, original_type):
return convert_to(obj)
raise TypeError('We do not know how to convert {} of type {} for json serialization. Please pass the default_mapping parameter with proper mapping of the object to a basic python type.'.format(obj, type(obj)))

return _convertor


def add_to_frozen_set(parents_ids, item_id):
return parents_ids | {item_id}

Expand All @@ -257,6 +235,26 @@ def convert_item_or_items_into_set_else_none(items):
return items


def add_root_to_paths(paths):
"""
Sometimes the users want to just pass
[key] instead of root[key] for example.
Here we automatically add all sorts of variations that might match
the path they were supposed to pass.
"""
if paths is None:
return
result = OrderedSet()
for path in paths:
if path.startswith('root'):
result.add(path)
else:
result.add(f"root.{path}")
result.add(f"root[{path}]")
result.add(f"root['{path}']")
return result


RE_COMPILED_TYPE = type(re.compile(''))


Expand Down
Loading

0 comments on commit b3a4947

Please sign in to comment.