Add support for JSON serialization. (#857)

Author: jpinner-lyft

docs/release_notes.rst

@@ -4,6 +4,7 @@ Release Notes
 Unreleased
 ----------
 * The ``IndexMeta`` class has been removed. Now ``type(Index) == type``.
+* JSON serialization support (``Model.to_json`` and ``Model.from_json``) has been added.
 
 
 v5.1.0

pynamodb/attributes.py

@@ -17,11 +17,20 @@
 from typing import TYPE_CHECKING
 
 from pynamodb._compat import GenericMeta
-from pynamodb.constants import (
-    BINARY, BINARY_SET, BOOLEAN, DATETIME_FORMAT, DEFAULT_ENCODING,
-    LIST, MAP, NULL, NUMBER, NUMBER_SET, STRING, STRING_SET
-)
-from pynamodb.exceptions import AttributeDeserializationError, AttributeNullError
+from pynamodb.constants import BINARY
+from pynamodb.constants import BINARY_SET
+from pynamodb.constants import BOOLEAN
+from pynamodb.constants import DATETIME_FORMAT
+from pynamodb.constants import DEFAULT_ENCODING
+from pynamodb.constants import LIST
+from pynamodb.constants import MAP
+from pynamodb.constants import NULL
+from pynamodb.constants import NUMBER
+from pynamodb.constants import NUMBER_SET
+from pynamodb.constants import STRING
+from pynamodb.constants import STRING_SET
+from pynamodb.exceptions import AttributeDeserializationError
+from pynamodb.exceptions import AttributeNullError
 from pynamodb.expressions.operand import Path
 
 
@@ -369,6 +378,36 @@ def _container_deserialize(self, attribute_values: Dict[str, Dict[str, Any]]) ->
                 value = attr.deserialize(attr.get_value(attribute_value))
                 setattr(self, name, value)
 
+    @classmethod
+    def _update_attribute_types(cls, attribute_values: Dict[str, Dict[str, Any]]):
+        """
+        Update the attribute types in the attribute values dictionary to disambiguate json string and array types
+        """
+        for attr in cls.get_attributes().values():
+            attribute_value = attribute_values.get(attr.attr_name)
+            if attribute_value:
+                AttributeContainer._coerce_attribute_type(attr.attr_type, attribute_value)
+                if isinstance(attr, ListAttribute) and attr.element_type and LIST in attribute_value:
+                    if issubclass(attr.element_type, AttributeContainer):
+                        for element in attribute_value[LIST]:
+                            if MAP in element:
+                                attr.element_type._update_attribute_types(element[MAP])
+                    else:
+                        for element in attribute_value[LIST]:
+                            AttributeContainer._coerce_attribute_type(attr.element_type.attr_type, element)
+                if isinstance(attr, AttributeContainer) and MAP in attribute_value:
+                    attr._update_attribute_types(attribute_value[MAP])
+
+    @staticmethod
+    def _coerce_attribute_type(attr_type: str, attribute_value: Dict[str, Any]):
+        # coerce attribute types to disambiguate json string and array types
+        if attr_type == BINARY and STRING in attribute_value:
+            attribute_value[BINARY] = attribute_value.pop(STRING)
+        if attr_type in {BINARY_SET, NUMBER_SET, STRING_SET} and LIST in attribute_value:
+            json_type = NUMBER if attr_type == NUMBER_SET else STRING
+            if all(next(iter(v)) == json_type for v in attribute_value[LIST]):
+                attribute_value[attr_type] = [value[json_type] for value in attribute_value.pop(LIST)]
+
     @classmethod
     def _get_discriminator_class(cls, attribute_values: Dict[str, Dict[str, Any]]) -> Optional[Type]:
         discriminator_attr = cls._get_discriminator_attribute()
@@ -1174,4 +1213,5 @@ def _get_serialize_class(self, value):
     float: NumberAttribute(),
     int: NumberAttribute(),
     str: UnicodeAttribute(),
+    bytes: BinaryAttribute(),
 }

pynamodb/models.py

@@ -1,6 +1,7 @@
 """
 DynamoDB Models for PynamoDB
 """
+import json
 import random
 import time
 import logging
@@ -54,6 +55,8 @@
     COUNT, ITEM_COUNT, KEY, UNPROCESSED_ITEMS, STREAM_VIEW_TYPE,
     STREAM_SPECIFICATION, STREAM_ENABLED, BILLING_MODE, PAY_PER_REQUEST_BILLING_MODE, TAGS
 )
+from pynamodb.util import attribute_value_to_json
+from pynamodb.util import json_to_attribute_value
 
 _T = TypeVar('_T', bound='Model')
 _KeyType = Any
@@ -846,7 +849,6 @@ def update_ttl(cls, ignore_update_ttl_errors: bool) -> None:
                     raise
 
     # Private API below
-
     @classmethod
     def _get_schema(cls) -> Dict[str, Any]:
         """
@@ -1110,6 +1112,14 @@ def deserialize(self, attribute_values: Dict[str, Dict[str, Any]]) -> None:
         """
         return self._container_deserialize(attribute_values=attribute_values)
 
+    def to_json(self) -> str:
+        return json.dumps({k: attribute_value_to_json(v) for k, v in self.serialize().items()})
+
+    def from_json(self, s: str) -> None:
+        attribute_values = {k: json_to_attribute_value(v) for k, v in json.loads(s).items()}
+        self._update_attribute_types(attribute_values)
+        self.deserialize(attribute_values)
+
 
 class _ModelFuture(Generic[_T]):
     """

pynamodb/util.py

@@ -0,0 +1,50 @@
+"""
+Utils
+"""
+import json
+from typing import Any
+from typing import Dict
+
+from pynamodb.constants import BINARY
+from pynamodb.constants import BINARY_SET
+from pynamodb.constants import BOOLEAN
+from pynamodb.constants import LIST
+from pynamodb.constants import MAP
+from pynamodb.constants import NULL
+from pynamodb.constants import NUMBER
+from pynamodb.constants import NUMBER_SET
+from pynamodb.constants import STRING
+from pynamodb.constants import STRING_SET
+
+
+def attribute_value_to_json(attribute_value: Dict[str, Any]) -> Any:
+    attr_type, attr_value = next(iter(attribute_value.items()))
+    if attr_type == LIST:
+        return [attribute_value_to_json(v) for v in attr_value]
+    if attr_type == MAP:
+        return {k: attribute_value_to_json(v) for k, v in attr_value.items()}
+    if attr_type == NULL:
+        return None
+    if attr_type in {BINARY, BINARY_SET, BOOLEAN, STRING, STRING_SET}:
+        return attr_value
+    if attr_type == NUMBER:
+        return json.loads(attr_value)
+    if attr_type == NUMBER_SET:
+        return [json.loads(v) for v in attr_value]
+    raise ValueError("Unknown attribute type: {}".format(attr_type))
+
+
+def json_to_attribute_value(value: Any) -> Dict[str, Any]:
+    if value is None:
+        return {NULL: True}
+    if value is True or value is False:
+        return {BOOLEAN: value}
+    if isinstance(value, (int, float)):
+        return {NUMBER: json.dumps(value)}
+    if isinstance(value, str):
+        return {STRING: value}
+    if isinstance(value, list):
+        return {LIST: [json_to_attribute_value(v) for v in value]}
+    if isinstance(value, dict):
+        return {MAP: {k: json_to_attribute_value(v) for k, v in value.items()}}
+    raise ValueError("Unknown value type: {}".format(type(value).__name__))

tests/test_attributes.py

@@ -1,6 +1,7 @@
 """
 pynamodb attributes tests
 """
+import calendar
 import json
 
 from base64 import b64encode
@@ -13,10 +14,10 @@
 
 from pynamodb.attributes import (
     BinarySetAttribute, BinaryAttribute, DynamicMapAttribute, NumberSetAttribute, NumberAttribute,
-    UnicodeAttribute, UnicodeSetAttribute, UTCDateTimeAttribute, BooleanAttribute, MapAttribute,
+    UnicodeAttribute, UnicodeSetAttribute, UTCDateTimeAttribute, BooleanAttribute, MapAttribute, NullAttribute,
     ListAttribute, JSONAttribute, TTLAttribute, VersionAttribute)
 from pynamodb.constants import (
-    DEFAULT_ENCODING, NUMBER, STRING, STRING_SET, NUMBER_SET, BINARY_SET,
+    DATETIME_FORMAT, DEFAULT_ENCODING, NUMBER, STRING, STRING_SET, NUMBER_SET, BINARY_SET,
     BINARY, BOOLEAN,
 )
 from pynamodb.models import Model
@@ -39,6 +40,7 @@ class Meta:
     json_attr = JSONAttribute()
     map_attr = MapAttribute()
     ttl_attr = TTLAttribute()
+    null_attr = NullAttribute(null=True)
 
 
 class CustomAttrMap(MapAttribute):
@@ -1062,3 +1064,119 @@ def test_deserialize(self):
         assert attr.deserialize('1') == 1
         assert attr.deserialize('3.141') == 3
         assert attr.deserialize('12345678909876543211234234324234') == 12345678909876543211234234324234
+
+
+class TestAttributeContainer:
+    def test_to_json(self):
+        now = datetime.now(tz=timezone.utc)
+        now_formatted = now.strftime(DATETIME_FORMAT)
+        now_unix_ts = calendar.timegm(now.utctimetuple())
+        test_model = AttributeTestModel()
+        test_model.binary_attr = b'foo'
+        test_model.binary_set_attr = {b'bar'}
+        test_model.number_attr = 1
+        test_model.number_set_attr = {0, 0.5, 1}
+        test_model.unicode_attr = 'foo'
+        test_model.unicode_set_attr = {'baz'}
+        test_model.datetime_attr = now
+        test_model.bool_attr = True
+        test_model.json_attr = {'foo': 'bar'}
+        test_model.map_attr = {'foo': 'bar'}
+        test_model.ttl_attr = now
+        test_model.null_attr = True
+        assert test_model.to_json() == (
+            '{'
+            '"binary_attr": "Zm9v", '
+            '"binary_set_attr": ["YmFy"], '
+            '"bool_attr": true, '
+            '"datetime_attr": "' + now_formatted + '", '
+            '"json_attr": "{\\"foo\\": \\"bar\\"}", '
+            '"map_attr": {"foo": "bar"}, '
+            '"null_attr": null, '
+            '"number_attr": 1, '
+            '"number_set_attr": [0, 0.5, 1], '
+            '"ttl_attr": ' + str(now_unix_ts) + ', '
+            '"unicode_attr": "foo", '
+            '"unicode_set_attr": ["baz"]'
+            '}')
+
+    def test_from_json(self):
+        now = datetime.now(tz=timezone.utc)
+        now_formatted = now.strftime(DATETIME_FORMAT)
+        now_unix_ts = calendar.timegm(now.utctimetuple())
+        json_string = (
+            '{'
+            '"binary_attr": "Zm9v", '
+            '"binary_set_attr": ["YmFy"], '
+            '"bool_attr": true, '
+            '"datetime_attr": "' + now_formatted + '", '
+            '"json_attr": "{\\"foo\\": \\"bar\\"}", '
+            '"map_attr": {"foo": "bar"}, '
+            '"null_attr": null, '
+            '"number_attr": 1, '
+            '"number_set_attr": [0, 0.5, 1], '
+            '"ttl_attr": ' + str(now_unix_ts) + ', '
+            '"unicode_attr": "foo", '
+            '"unicode_set_attr": ["baz"]'
+            '}')
+        test_model = AttributeTestModel()
+        test_model.from_json(json_string)
+        assert test_model.binary_attr == b'foo'
+        assert test_model.binary_set_attr == {b'bar'}
+        assert test_model.number_attr == 1
+        assert test_model.number_set_attr == {0, 0.5, 1}
+        assert test_model.unicode_attr == 'foo'
+        assert test_model.unicode_set_attr == {'baz'}
+        assert test_model.datetime_attr == now
+        assert test_model.bool_attr is True
+        assert test_model.json_attr == {'foo': 'bar'}
+        assert test_model.map_attr.foo == 'bar'
+        assert test_model.ttl_attr == now.replace(microsecond=0)
+        assert test_model.null_attr is None
+
+    def test_to_json_complex(self):
+        class MyMap(MapAttribute):
+            foo = UnicodeSetAttribute(attr_name='bar')
+
+        class ListTestModel(Model):
+            class Meta:
+                host = 'http://localhost:8000'
+                table_name = 'test'
+            unicode_attr = UnicodeAttribute(hash_key=True)
+            list_attr = ListAttribute(of=NumberSetAttribute)
+            list_map_attr = ListAttribute(of=MyMap)
+
+        list_test_model = ListTestModel()
+        list_test_model.unicode_attr = 'foo'
+        list_test_model.list_attr = [{0, 1, 2}]
+        list_test_model.list_map_attr = [MyMap(foo={'baz'})]
+        assert list_test_model.to_json() == (
+            '{'
+            '"list_attr": [[0, 1, 2]], '
+            '"list_map_attr": [{"bar": ["baz"]}], '
+            '"unicode_attr": "foo"'
+            '}')
+
+    def test_from_json_complex(self):
+        class MyMap(MapAttribute):
+            foo = UnicodeSetAttribute(attr_name='bar')
+
+        class ListTestModel(Model):
+            class Meta:
+                host = 'http://localhost:8000'
+                table_name = 'test'
+            unicode_attr = UnicodeAttribute(hash_key=True)
+            list_attr = ListAttribute(of=NumberSetAttribute)
+            list_map_attr = ListAttribute(of=MyMap)
+
+        json_string = (
+            '{'
+            '"list_attr": [[0, 1, 2]], '
+            '"list_map_attr": [{"bar": ["baz"]}], '
+            '"unicode_attr": "foo"'
+            '}')
+        list_test_model = ListTestModel()
+        list_test_model.from_json(json_string)
+        assert list_test_model.unicode_attr == 'foo'
+        assert list_test_model.list_attr == [{0, 1, 2}]
+        assert list_test_model.list_map_attr[0].foo == {'baz'}

tests/test_model.py

@@ -2528,6 +2528,40 @@ class Meta:
         with self.assertRaises(AttributeError):
             MissingTableNameModel.exists()
 
+    def test_to_json(self):
+        """
+        Model.to_json
+        """
+        user = UserModel()
+        user.custom_user_name = 'foo'
+        user.user_id = 'bar'
+        user.picture = base64.b64decode(BINARY_ATTR_DATA)
+        user.zip_code = 88030
+        json_user = json.loads(user.to_json())
+        self.assertEqual(json_user['user_name'], user.custom_user_name)  # uses custom attribute name
+        self.assertEqual(json_user['user_id'], user.user_id)
+        self.assertEqual(json_user['picture'], BINARY_ATTR_DATA)
+        self.assertEqual(json_user['zip_code'], user.zip_code)
+        self.assertEqual(json_user['email'], 'needs_email')  # set to default value
+
+    def test_from_json(self):
+        """
+        Model.from_json
+        """
+        json_user = {
+            'user_name': 'foo',
+            'user_id': 'bar',
+            'picture': BINARY_ATTR_DATA,
+            'zip_code': 88030,
+        }
+        user = UserModel()
+        user.from_json(json.dumps(json_user))
+        self.assertEqual(user.custom_user_name, json_user['user_name'])  # uses custom attribute name
+        self.assertEqual(user.user_id, json_user['user_id'])
+        self.assertEqual(user.picture, base64.b64decode(json_user['picture']))
+        self.assertEqual(user.zip_code, json_user['zip_code'])
+        self.assertEqual(user.email, 'needs_email')  # set to default value
+
     def _get_office_employee(self):
         justin = Person(
             fname='Justin',