Skip to content

Commit f799a93

Browse files
authored
Merge pull request #1 from jokiefer/feature/bulk-support
Feature/bulk support
2 parents 2249f3e + f9c47cf commit f799a93

File tree

10 files changed

+297
-24
lines changed

10 files changed

+297
-24
lines changed

CHANGELOG.rst

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
77
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
88

99

10+
[0.3.0] - 2023-07-10
11+
--------------------
12+
13+
Added
14+
~~~~~
15+
16+
* bulk operating for `add` and `delete` operations
17+
18+
Fixed
19+
~~~~~
20+
21+
* adds `check_resource_identifier_object` check on parser to check update operation correctly
22+
23+
1024
[0.2.0] - 2023-07-06
1125
--------------------
1226

README.rst

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ See the `usage <https://drf-json-api-atomic-operations.readthedocs.io/en/latest/
1919
Implemented Features
2020
~~~~~~~~~~~~~~~~~~~~
2121

22-
* creating, updating, removing multiple resources in a single request (sequential db calls)
22+
* creating, updating, removing multiple resources in a single request (sequential db calls optional bulk db calls for create and delete)
2323
* `Updating To-One Relationships <https://jsonapi.org/ext/atomic/#auto-id-updating-to-one-relationships>`_
2424
* `Updating To-Many Relationships <https://jsonapi.org/ext/atomic/#auto-id-updating-to-many-relationships>`_
2525
* error reporting with json pointer to the concrete operation and the wrong attributes
@@ -29,5 +29,4 @@ ToDo
2929
~~~~
3030

3131
* permission handling
32-
* use django bulk operations to optimize db execution time
3332
* `local identity (lid) <https://jsonapi.org/ext/atomic/#operation-objects>`_ handling

atomic_operations/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
__version__ = "0.2.0"
1+
__version__ = "0.3.0"
22
VERSION = __version__ # synonym

atomic_operations/parsers.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ def check_update_operation(self, idx, operation):
103103
raise MissingPrimaryData(idx)
104104
elif not isinstance(data, dict):
105105
raise InvalidPrimaryDataType(idx, "object")
106+
self.check_resource_identifier_object(idx, data, operation["op"])
106107

107108
def check_remove_operation(self, idx, ref):
108109
if not ref:

atomic_operations/views.py

Lines changed: 83 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ class AtomicOperationView(APIView):
2424
#
2525
serializer_classes: Dict = {}
2626

27+
sequential = True
28+
response_data: List[Dict] = []
29+
2730
# TODO: proof how to check permissions for all operations
2831
# permission_classes = TODO
2932
# call def check_permissions for `add` operation
@@ -89,30 +92,92 @@ def get_serializer_context(self):
8992
def post(self, request, *args, **kwargs):
9093
return self.perform_operations(request.data)
9194

95+
def handle_sequential(self, serializer, operation_code):
96+
if operation_code in ["add", "update", "update-relationship"]:
97+
serializer.is_valid(raise_exception=True)
98+
serializer.save()
99+
if operation_code != "update-relationship":
100+
self.response_data.append(serializer.data)
101+
else:
102+
# remove
103+
serializer.instance.delete()
104+
105+
def perform_bulk_create(self, bulk_operation_data):
106+
objs = []
107+
model_class = bulk_operation_data["serializer_collection"][0].Meta.model
108+
for _serializer in bulk_operation_data["serializer_collection"]:
109+
_serializer.is_valid(raise_exception=True)
110+
instance = model_class(**_serializer.validated_data)
111+
objs.append(instance)
112+
self.response_data.append(
113+
_serializer.__class__(instance=instance).data)
114+
model_class.objects.bulk_create(
115+
objs)
116+
117+
def perform_bulk_delete(self, bulk_operation_data):
118+
obj_ids = []
119+
for _serializer in bulk_operation_data["serializer_collection"]:
120+
obj_ids.append(_serializer.instance.pk)
121+
self.response_data.append(_serializer.data)
122+
bulk_operation_data["serializer_collection"][0].Meta.model.objects.filter(
123+
pk__in=obj_ids).delete()
124+
125+
def handle_bulk(self, serializer, current_operation_code, bulk_operation_data):
126+
bulk_operation_data["serializer_collection"].append(serializer)
127+
if bulk_operation_data["next_operation_code"] != current_operation_code or bulk_operation_data["next_resource_type"] != serializer.initial_data["type"]:
128+
if current_operation_code == "add":
129+
self.perform_bulk_create(bulk_operation_data)
130+
elif current_operation_code == "delete":
131+
self.perform_bulk_delete(bulk_operation_data)
132+
else:
133+
# TODO: update in bulk requires more logic cause it could be a partial update and every field differs pers instance.
134+
# Then we can't do a bulk operation. This is only possible for instances which changes the same field(s).
135+
# Maybe the anylsis of this takes longer than simple handling updates in sequential mode.
136+
# For now we handle updates always in sequential mode
137+
self.handle_sequential(
138+
bulk_operation_data["serializer_collection"][0], current_operation_code)
139+
bulk_operation_data["serializer_collection"] = []
140+
92141
def perform_operations(self, parsed_operations: List[Dict]):
93-
response_data: List[Dict] = []
142+
self.response_data = [] # reset local response data storage
143+
144+
bulk_operation_data = {
145+
"serializer_collection": [],
146+
"next_operation_code": "",
147+
"next_resource_type": ""
148+
}
149+
94150
with atomic():
151+
95152
for idx, operation in enumerate(parsed_operations):
96-
op_code = next(iter(operation))
97-
obj = operation[op_code]
98-
# TODO: collect operations of same op_code and resource type to support bulk_create | bulk_update | filter(id__in=[1,2,3]).delete()
153+
operation_code = next(iter(operation))
154+
obj = operation[operation_code]
155+
99156
serializer = self.get_serializer(
100157
idx=idx,
101158
data=obj,
102-
operation_code="update" if op_code == "update-relationship" else op_code,
159+
operation_code="update" if operation_code == "update-relationship" else operation_code,
103160
resource_type=obj["type"],
104-
partial=True if "update" in op_code else False
161+
partial=True if "update" in operation_code else False
105162
)
106-
if op_code in ["add", "update", "update-relationship"]:
107-
serializer.is_valid(raise_exception=True)
108-
serializer.save()
109-
# FIXME: check if it is just a relationship update
110-
if op_code == "update-relationship":
111-
# relation update. No response data
112-
continue
113-
response_data.append(serializer.data)
114-
else:
115-
# remove
116-
serializer.instance.delete()
117163

118-
return Response(response_data, status=status.HTTP_200_OK if response_data else status.HTTP_204_NO_CONTENT)
164+
if self.sequential:
165+
self.handle_sequential(serializer, operation_code)
166+
else:
167+
is_last_iter = parsed_operations.__len__() == idx + 1
168+
if is_last_iter:
169+
bulk_operation_data["next_operation_code"] = ""
170+
bulk_operation_data["next_resource_type"] = ""
171+
else:
172+
next_operation = parsed_operations[idx + 1]
173+
bulk_operation_data["next_operation_code"] = next(
174+
iter(next_operation))
175+
bulk_operation_data["next_resource_type"] = next_operation[bulk_operation_data["next_operation_code"]]["type"]
176+
177+
self.handle_bulk(
178+
serializer=serializer,
179+
current_operation_code=operation_code,
180+
bulk_operation_data=bulk_operation_data
181+
)
182+
183+
return Response(self.response_data, status=status.HTTP_200_OK if self.response_data else status.HTTP_204_NO_CONTENT)

docs/source/usage.rst

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,4 +73,19 @@ Now you can call the api like below.
7373
}
7474
}
7575
}]
76-
}
76+
}
77+
78+
79+
Bulk operating
80+
==============
81+
82+
By default all operations are sequential db calls. This package provides also bulk operating for creating and deleting resources. To activate it you need to configure the following.
83+
84+
85+
.. code-block:: python
86+
87+
from atomic_operations.views import AtomicOperationView
88+
89+
class ConcretAtomicOperationView(AtomicOperationView):
90+
91+
sequential = False

tests/test_parsers.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,27 @@ def test_primary_data_without_id(self):
271271
}
272272
)
273273

274+
data = {
275+
ATOMIC_OPERATIONS: [
276+
{
277+
"op": "update",
278+
"data": {
279+
"type": "articles",
280+
}
281+
}
282+
]
283+
}
284+
stream = BytesIO(json.dumps(data).encode("utf-8"))
285+
self.assertRaisesRegex(
286+
JsonApiParseError,
287+
"The resource identifier object must contain an `id` member",
288+
self.parser.parse,
289+
**{
290+
"stream": stream,
291+
"parser_context": self.parser_context
292+
}
293+
)
294+
274295
def test_primary_data(self):
275296
data = {
276297
ATOMIC_OPERATIONS: [

tests/test_views.py

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,155 @@ def test_view_processing_with_valid_request(self):
198198
self.assertQuerysetEqual(RelatedModelTwo.objects.filter(pk__in=[1, 2]),
199199
BasicModel.objects.get(pk=2).to_many.all())
200200

201+
def test_bulk_view_processing_with_valid_request(self):
202+
operations = [
203+
{
204+
"op": "add",
205+
"data": {
206+
"type": "BasicModel",
207+
"attributes": {
208+
"text": "JSON API paints my bikeshed!"
209+
}
210+
}
211+
}, {
212+
"op": "add",
213+
"data": {
214+
"type": "BasicModel",
215+
"attributes": {
216+
"text": "JSON API paints my bikeshed!"
217+
}
218+
}
219+
}, {
220+
"op": "add",
221+
"data": {
222+
"type": "BasicModel",
223+
"attributes": {
224+
"text": "JSON API paints my bikeshed!"
225+
}
226+
}
227+
}, {
228+
"op": "add",
229+
"data": {
230+
"type": "BasicModel",
231+
"attributes": {
232+
"text": "JSON API paints my bikeshed!"
233+
}
234+
}
235+
}, {
236+
"op": "add",
237+
"data": {
238+
"type": "RelatedModel",
239+
"attributes": {
240+
"text": "JSON API paints my bikeshed!"
241+
}
242+
}
243+
}, {
244+
"op": "update",
245+
"data": {
246+
"id": "1",
247+
"type": "RelatedModel",
248+
"attributes": {
249+
"text": "JSON API paints my bikeshed!2"
250+
}
251+
}
252+
}
253+
]
254+
255+
data = {
256+
ATOMIC_OPERATIONS: operations
257+
}
258+
259+
response = self.client.post(
260+
path="/bulk",
261+
data=data,
262+
content_type=ATOMIC_CONTENT_TYPE,
263+
264+
**{"HTTP_ACCEPT": ATOMIC_CONTENT_TYPE}
265+
)
266+
267+
# check response
268+
self.assertEqual(200, response.status_code)
269+
270+
expected_result = {
271+
ATOMIC_RESULTS: [
272+
{
273+
"data": {
274+
"id": "1",
275+
"type": "BasicModel",
276+
"attributes": {
277+
"text": "JSON API paints my bikeshed!"
278+
},
279+
"relationships": {
280+
"to_many": {'data': [], 'meta': {'count': 0}},
281+
"to_one": {'data': None},
282+
}
283+
}
284+
},
285+
{
286+
"data": {
287+
"id": "2",
288+
"type": "BasicModel",
289+
"attributes": {
290+
"text": "JSON API paints my bikeshed!"
291+
},
292+
"relationships": {
293+
"to_many": {'data': [], 'meta': {'count': 0}},
294+
"to_one": {'data': None},
295+
}
296+
}
297+
}, {
298+
"data": {
299+
"id": "3",
300+
"type": "BasicModel",
301+
"attributes": {
302+
"text": "JSON API paints my bikeshed!"
303+
},
304+
"relationships": {
305+
"to_many": {'data': [], 'meta': {'count': 0}},
306+
"to_one": {'data': None},
307+
}
308+
}
309+
},
310+
{
311+
"data": {
312+
"id": "4",
313+
"type": "BasicModel",
314+
"attributes": {
315+
"text": "JSON API paints my bikeshed!"
316+
},
317+
"relationships": {
318+
"to_many": {'data': [], 'meta': {'count': 0}},
319+
"to_one": {'data': None},
320+
}
321+
}
322+
},
323+
{
324+
"data": {
325+
"id": "1",
326+
"type": "RelatedModel",
327+
"attributes": {
328+
"text": "JSON API paints my bikeshed!"
329+
}
330+
}
331+
},
332+
{
333+
"data": {
334+
"id": "1",
335+
"type": "RelatedModel",
336+
"attributes": {
337+
"text": "JSON API paints my bikeshed!2"
338+
}
339+
}
340+
}
341+
]
342+
}
343+
344+
self.assertDictEqual(expected_result,
345+
json.loads(response.content))
346+
347+
# check db content
348+
self.assertEqual(4, BasicModel.objects.count())
349+
201350
def test_parser_exception_with_pointer(self):
202351
operations = [
203352
{

tests/urls.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
from django.urls import path
22

3-
from tests.views import ConcretAtomicOperationView
3+
from tests.views import BulkAtomicOperationView, ConcretAtomicOperationView
4+
45

56
urlpatterns = [
6-
path("", ConcretAtomicOperationView.as_view())
7+
path("", ConcretAtomicOperationView.as_view()),
8+
path("bulk", BulkAtomicOperationView.as_view())
9+
710
]

tests/views.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,11 @@ class ConcretAtomicOperationView(AtomicOperationView):
1212
"update:BasicModel": BasicModelSerializer,
1313
"remove:BasicModel": BasicModelSerializer,
1414
"add:RelatedModel": RelatedModelSerializer,
15+
"update:RelatedModel": RelatedModelSerializer,
1516
"add:RelatedModelTwo": RelatedModelTwoSerializer,
17+
1618
}
19+
20+
21+
class BulkAtomicOperationView(ConcretAtomicOperationView):
22+
sequential = False

0 commit comments

Comments
 (0)