From e1acdd704fcada776a7c9aa3bf2f5496e043dcdf Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Fri, 18 Oct 2019 09:09:52 +0200 Subject: [PATCH 1/2] Fix load parameter *unknow* propagation When deserializing a data structure with the load method, the *unknown* was not propagated to the loading of nested data structures. As result, if a unknown field was present into a nested data structure a ValidationError was raised even if the load methd was called with *unknown=EXCLUDE*. This commit ensures that this parameter is now propagated also to the loading of nested data structures. fixes #1428 --- AUTHORS.rst | 1 + docs/quickstart.rst | 49 +++++++++++++++++++++++++++++++++++++++ src/marshmallow/fields.py | 15 ++++++++---- src/marshmallow/schema.py | 8 +++++-- tests/test_schema.py | 17 ++++++++++++++ 5 files changed, 84 insertions(+), 6 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 00d91f286..0d8f28aa6 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -140,3 +140,4 @@ Contributors (chronological) - `@phrfpeixoto `_ - `@jceresini `_ - Nikolay Shebanov `@killthekitten `_ +- Laurent Mignon `@lmignon `_ \ No newline at end of file diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 5bd26d68d..e952710aa 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -427,6 +427,55 @@ or when calling `load `. The ``unknown`` option value set in :meth:`load ` will override the value applied at instantiation time, which itself will override the value defined in the *class Meta*. +The ``unknown`` option value set in :meth:`load ` is propagated to nested Schema. It's not the case when the value is applied at instantiation time nor when the value is defined in the *class Meta*. + +.. code-block:: python + + from marshmallow import fields, Schema, EXCLUDE, INCLUDE, RAISE + + + class UserSchema(Schema): + class Meta: + unknown = RAISE + + name = fields.String() + + + class BlogSchema(Schema): + class Meta: + unknown = EXCLUDE + + title = fields.String() + author = fields.Nested(UserSchema) + + + BlogSchema().load( + { + "title": "Some title", + "unknown_field": "value", + "author": {"name": "Author name", "unknown_field": "value"}, + } + ) # => ValidationError: {'author': {'unknown_field': ['Unknown field.']}} + + BlogSchema().load( + { + "title": "Some title", + "unknown_field": "value", + "author": {"name": "Author name", "unknown_field": "value"}, + }, + unknown=EXCLUDE, + ) # => {'author': {'name': 'Author name'}, 'title': 'Some title'} + + + BogSchema.load( + { + "title": "Some title", + "unknown_field": "value", + "author": {"name": "Author name", "unknown_field": "value"}, + }, + unknown=INCLUDE, + ) # => {'author': {'name': 'Author name', 'unknown_field': 'value'}, 'title': 'Some title', 'unknown_field': 'value'} + This order of precedence allows you to change the behavior of a schema for different contexts. diff --git a/src/marshmallow/fields.py b/src/marshmallow/fields.py index 003c33ae8..1e7651c43 100644 --- a/src/marshmallow/fields.py +++ b/src/marshmallow/fields.py @@ -567,11 +567,12 @@ def _test_collection(self, value, many=False): if many and not utils.is_collection(value): raise self.make_error("type", input=value, type=value.__class__.__name__) - def _load(self, value, data, partial=None, many=False): + def _load(self, value, data, partial=None, many=False, unknown=None): many = self.schema.many or self.many or many + unknown = unknown or self.unknown try: valid_data = self.schema.load( - value, unknown=self.unknown, partial=partial, many=many + value, unknown=unknown, partial=partial, many=many ) except ValidationError as error: raise ValidationError( @@ -579,17 +580,23 @@ def _load(self, value, data, partial=None, many=False): ) from error return valid_data - def _deserialize(self, value, attr, data, partial=None, many=False, **kwargs): + def _deserialize( + self, value, attr, data, partial=None, many=False, unknown=None, **kwargs + ): """Same as :meth:`Field._deserialize` with additional ``partial`` argument. :param bool|tuple partial: For nested schemas, the ``partial`` parameter passed to `Schema.load`. + :param unknown: For nested schemas, the ``unknown`` + parameter passed to `Schema.load`.. .. versionchanged:: 3.0.0 Add ``partial`` parameter. + .. versionchanged:: 3.2.2 + Add ``unknown`` parameter. """ self._test_collection(value, many=many) - return self._load(value, data, partial=partial, many=many) + return self._load(value, data, partial=partial, many=many, unknown=unknown) class Pluck(Nested): diff --git a/src/marshmallow/schema.py b/src/marshmallow/schema.py index 7265514fe..88c02bc67 100644 --- a/src/marshmallow/schema.py +++ b/src/marshmallow/schema.py @@ -652,6 +652,7 @@ def _deserialize( d_kwargs["partial"] = sub_partial else: d_kwargs["partial"] = partial + d_kwargs["unknown"] = unknown getter = lambda val: field_obj.deserialize( val, field_name, data, **d_kwargs ) @@ -665,6 +666,7 @@ def _deserialize( if value is not missing: key = field_obj.attribute or attr_name set_value(typing.cast(typing.Dict, ret), key, value) + unknown = unknown or self.unknown if unknown != EXCLUDE: fields = { field_obj.data_key if field_obj.data_key is not None else field_name @@ -701,7 +703,8 @@ def load( will be ignored. Use dot delimiters to specify nested fields. :param unknown: Whether to exclude, include, or raise an error for unknown fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`. - If `None`, the value for `self.unknown` is used. + If `None`, the value for `self.unknown` is used. When used, provided + value is propagated to the loading of nested schemas. :return: Deserialized data .. versionadded:: 1.0.0 @@ -709,6 +712,8 @@ def load( This method returns the deserialized data rather than a ``(data, errors)`` duple. A :exc:`ValidationError ` is raised if invalid data are passed. + .. versionchanged:: 3.2.2 + The ``unknown`` parameter value is propagated to the loading of nested schemas. """ return self._do_load( data, many=many, partial=partial, unknown=unknown, postprocess=True @@ -823,7 +828,6 @@ def _do_load( error_store = ErrorStore() errors = {} # type: typing.Dict[str, typing.List[str]] many = self.many if many is None else bool(many) - unknown = unknown or self.unknown if partial is None: partial = self.partial # Run preprocessors diff --git a/tests/test_schema.py b/tests/test_schema.py index 2be35b775..ec167fc5a 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -289,6 +289,23 @@ class Outer(Schema): assert Outer().load({"list1": val, "list2": val}) == {"list1": [], "list2": []} +@pytest.mark.parametrize( + "val", + ( + {"inner": {"name": "name"}, "unknown": 1}, + {"inner": {"name": "name", "unknown_nested": 1}, "unknown": 1}, + ), +) +def test_load_unknown(val): + class Inner(Schema): + name = fields.String() + + class Outer(Schema): + inner = fields.Nested(Inner) + + assert Outer().load(val, unknown=EXCLUDE) == {"inner": {"name": "name"}} + + def test_loads_returns_a_user(): s = UserSchema() result = s.loads(json.dumps({"name": "Monty"})) From 996fbe5b502abd0addd770e93b527cb857bdb40f Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Mon, 25 Nov 2019 23:27:40 +0100 Subject: [PATCH 2/2] test_load_schema: Defines initial unknown parameter into Meta definition --- tests/test_schema.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_schema.py b/tests/test_schema.py index ec167fc5a..9531666b5 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -300,9 +300,15 @@ def test_load_unknown(val): class Inner(Schema): name = fields.String() + class Meta: + unknown = RAISE + class Outer(Schema): inner = fields.Nested(Inner) + class Meta: + unknown = RAISE + assert Outer().load(val, unknown=EXCLUDE) == {"inner": {"name": "name"}}