Add support for callable group_by

deepdiff/diff.py

@@ -157,7 +157,7 @@ def __init__(self,
                  exclude_regex_paths: Union[str, List[str], Pattern[str], List[Pattern[str]], None]=None,
                  exclude_types: Optional[List[type]]=None,
                  get_deep_distance: bool=False,
-                 group_by: Union[str, Tuple[str, str], None]=None,
+                 group_by: Union[str, Tuple[str, str], Callable, None]=None,
                  group_by_sort_key: Union[str, Callable, None]=None,
                  hasher: Optional[Callable]=None,
                  hashes: Optional[Dict[Any, Any]]=None,
@@ -943,7 +943,7 @@ def _diff_by_forming_pairs_and_comparing_one_by_one(
         t2_from_index=None, t2_to_index=None,
     ):
         for (i, j), (x, y) in self._get_matching_pairs(
-            level, 
+            level,
             t1_from_index=t1_from_index, t1_to_index=t1_to_index,
             t2_from_index=t2_from_index, t2_to_index=t2_to_index
         ):
@@ -1834,7 +1834,32 @@ def _get_view_results(self, view):
 
     @staticmethod
     def _get_key_for_group_by(row, group_by, item_name):
+        """
+        Get the key value to group a row by, using the specified group_by parameter.
+
+        Example
+            >>> row = {'first': 'John', 'middle': 'Joe', 'last': 'Smith'}
+            >>> DeepDiff._get_key_for_group_by(row, 'first', 't1')
+            'John'
+            >>> nested_row = {'id': 123, 'demographics': {'names': {'first': 'John', 'middle': 'Joe', 'last': 'Smith'}}}
+            >>> group_by = lambda x: x['demographics']['names']['first']
+            >>> DeepDiff._get_key_for_group_by(nested_row, group_by, 't1')
+            'John'
+
+        Args:
+            row (dict): The dictionary (row) to extract the group by key from.
+            group_by (str or callable): The key name or function to call to get to the key value to group by.
+            item_name (str): The name of the item, used for error messages.
+
+        Returns:
+            str: The key value to group by.
+
+        Raises:
+            KeyError: If the specified key is not found in the row.
+        """
         try:
+            if callable(group_by):
+                return group_by(row)
             return row.pop(group_by)
         except KeyError:
             logger.error("Unable to group {} by {}. The key is missing in {}".format(item_name, group_by, row))
@@ -1914,13 +1939,13 @@ def affected_paths(self):
         Whether a value was changed or they were added or removed.
 
         Example
+            >>> from pprint import pprint
             >>> t1 = {1: 1, 2: 2, 3: [3], 4: 4}
             >>> t2 = {1: 1, 2: 4, 3: [3, 4], 5: 5, 6: 6}
             >>> ddiff = DeepDiff(t1, t2)
-            >>> ddiff
             >>> pprint(ddiff, indent=4)
-            {   'dictionary_item_added': [root[5], root[6]],
-                'dictionary_item_removed': [root[4]],
+            {   'dictionary_item_added': ['root[5]', 'root[6]'],
+                'dictionary_item_removed': ['root[4]'],
                 'iterable_item_added': {'root[3][1]': 4},
                 'values_changed': {'root[2]': {'new_value': 4, 'old_value': 2}}}
             >>> ddiff.affected_paths
@@ -1946,13 +1971,13 @@ def affected_root_keys(self):
         Whether a value was changed or they were added or removed.
 
         Example
+            >>> from pprint import pprint
             >>> t1 = {1: 1, 2: 2, 3: [3], 4: 4}
             >>> t2 = {1: 1, 2: 4, 3: [3, 4], 5: 5, 6: 6}
             >>> ddiff = DeepDiff(t1, t2)
-            >>> ddiff
             >>> pprint(ddiff, indent=4)
-            {   'dictionary_item_added': [root[5], root[6]],
-                'dictionary_item_removed': [root[4]],
+            {   'dictionary_item_added': ['root[5]', 'root[6]'],
+                'dictionary_item_removed': ['root[4]'],
                 'iterable_item_added': {'root[3][1]': 4},
                 'values_changed': {'root[2]': {'new_value': 4, 'old_value': 2}}}
             >>> ddiff.affected_paths

docs/basics.rst

@@ -89,8 +89,8 @@ String difference 2
 
     >>>
     >>> print (ddiff['values_changed']["root[4]['b']"]["diff"])
-    --- 
-    +++ 
+    ---
+    +++
     @@ -1,5 +1,4 @@
     -world!
     -Goodbye!
@@ -172,7 +172,7 @@ Datetime
 Group By
 --------
 
-group_by can be used when dealing with the list of dictionaries. It converts them from lists to a single dictionary with the key defined by group_by. The common use case is when reading data from a flat CSV, and the primary key is one of the columns in the CSV. We want to use the primary key instead of the CSV row number to group the rows. The group_by can do 2D group_by by passing a list of 2 keys.
+group_by can be used when dealing with the list of dictionaries. It converts them from lists to a single dictionary with the key defined by group_by. The common use case is when reading data from a flat CSV, and the primary key is one of the columns in the CSV. We want to use the primary key instead of the CSV row number to group the rows. The group_by can do 2D group_by by passing a list of 2 keys. It is also possible to have a callable group_by, which can be used to access keys in more nested data structures.
 
 For example:
     >>> [
@@ -249,14 +249,36 @@ Now we use group_by='id':
      'values_changed': {"root['BB']['James']['last_name']": {'new_value': 'Brown',
                                                              'old_value': 'Blue'}}}
 
+Callable group_by Example:
+    >>> from deepdiff import DeepDiff
+    >>>
+    >>> t1 = [
+    ...     {'id': 'AA', 'demographics': {'names': {'first': 'Joe', 'middle': 'John', 'last': 'Nobody'}}},
+    ...     {'id': 'BB', 'demographics': {'names': {'first': 'James', 'middle': 'Joyce', 'last': 'Blue'}}},
+    ...     {'id': 'CC', 'demographics': {'names': {'first': 'Mike', 'middle': 'Mark', 'last': 'Apple'}}},
+    ... ]
+    >>>
+    >>> t2 = [
+    ...     {'id': 'AA', 'demographics': {'names': {'first': 'Joe', 'middle': 'John', 'last': 'Nobody'}}},
+    ...     {'id': 'BB', 'demographics': {'names': {'first': 'James', 'middle': 'Joyce', 'last': 'Brown'}}},
+    ...     {'id': 'CC', 'demographics': {'names': {'first': 'Mike', 'middle': 'Charles', 'last': 'Apple'}}},
+    ... ]
+    >>>
+    >>> diff = DeepDiff(t1, t2, group_by=lambda x: x['demographics']['names']['first'])
+    >>> pprint(diff)
+    {'values_changed': {"root['James']['demographics']['names']['last']": {'new_value': 'Brown',
+                                                                           'old_value': 'Blue'},
+                        "root['Mike']['demographics']['names']['middle']": {'new_value': 'Charles',
+                                                                            'old_value': 'Mark'}}}
+
 .. _group_by_sort_key_label:
 
 Group By - Sort Key
 -------------------
 
 group_by_sort_key is used to define how dictionaries are sorted if multiple ones fall under one group. When this parameter is used, group_by converts the lists of dictionaries into a dictionary of keys to lists of dictionaries. Then, group_by_sort_key is used to sort between the list.
 
-For example, there are duplicate id values. If we only use group_by='id', one of the dictionaries with id of 'BB' will overwrite the other. However, if we also set group_by_sort_key='name', we keep both dictionaries with the id of 'BB'. 
+For example, there are duplicate id values. If we only use group_by='id', one of the dictionaries with id of 'BB' will overwrite the other. However, if we also set group_by_sort_key='name', we keep both dictionaries with the id of 'BB'.
 
 Example:
     >>> [{'id': 'AA', 'int_id': 2, 'last_name': 'Nobody', 'name': 'Joe'},

tests/test_diff_group_by.py

@@ -0,0 +1,59 @@
+"""Tests for the group_by parameter of Deepdiff"""
+
+import pytest
+
+from deepdiff import DeepDiff
+
+
+class TestGetKeyForGroupBy:
+    def test_group_by_string(self):
+        """Test where group_by is a single key (string)."""
+        row = {'first': 'John', 'middle': 'Joe', 'last': 'Smith'}
+        group_by = 'first'
+        item_name = 't1'
+        actual = DeepDiff._get_key_for_group_by(row, group_by, item_name)
+        expected = 'John'
+
+        assert actual == expected
+
+    def test_group_by_callable(self):
+        """Test where group_by is callable."""
+        row = {'id': 123, 'demographics': {'names': {'first': 'John', 'middle': 'Joe', 'last': 'Smith'}}}
+        group_by = lambda x: x['demographics']['names']['first']
+        item_name = 't1'
+        actual = DeepDiff._get_key_for_group_by(row, group_by, item_name)
+        expected = 'John'
+        assert actual == expected
+
+    def test_group_by_key_error(self):
+        """Test where group_by is a key that is not in the row."""
+        row = {'id': 123, 'demographics': {'names': {'first': 'John', 'middle': 'Joe', 'last': 'Smith'}}}
+        group_by = 'someotherkey'
+        item_name = 't1'
+        with pytest.raises(KeyError):
+            DeepDiff._get_key_for_group_by(row, group_by, item_name)
+
+
+class TestGroupBy:
+    def test_group_by_callable(self):
+        """Test where group_by is a callable."""
+        t1 = [
+            {'id': 'AA', 'demographics': {'names': {'first': 'Joe', 'middle': 'John', 'last': 'Nobody'}}},
+            {'id': 'BB', 'demographics': {'names': {'first': 'James', 'middle': 'Joyce', 'last': 'Blue'}}},
+            {'id': 'CC', 'demographics': {'names': {'first': 'Mike', 'middle': 'Mark', 'last': 'Apple'}}},
+        ]
+
+        t2 = [
+            {'id': 'AA', 'demographics': {'names': {'first': 'Joe', 'middle': 'John', 'last': 'Nobody'}}},
+            {'id': 'BB', 'demographics': {'names': {'first': 'James', 'middle': 'Joyce', 'last': 'Brown'}}},
+            {'id': 'CC', 'demographics': {'names': {'first': 'Mike', 'middle': 'Charles', 'last': 'Apple'}}},
+        ]
+
+        actual = DeepDiff(t1, t2, group_by=lambda x: x['demographics']['names']['first'])
+        expected = {
+            'values_changed': {
+                "root['James']['demographics']['names']['last']": {'new_value': 'Brown', 'old_value': 'Blue'},
+                "root['Mike']['demographics']['names']['middle']": {'new_value': 'Charles', 'old_value': 'Mark'},
+            },
+        }
+        assert actual == expected