It is easy to change the format of an object which is not JSON serializable eg datetime.datetime.
My requirement, for debugging purposes, is to alter the way some custom objects extended from base ones like dict and list , get serialized in json format . Code :
import datetime import json def json_debug_handler(obj): print("object received:") print type(obj) print("\n\n") if isinstance(obj, datetime.datetime): return obj.isoformat() elif isinstance(obj,mDict): return {'orig':obj , 'attrs': vars(obj)} elif isinstance(obj,mList): return {'orig':obj, 'attrs': vars(obj)} else: return None class mDict(dict): pass class mList(list): pass def test_debug_json(): games = mList(['mario','contra','tetris']) games.src = 'console' scores = mDict({'dp':10,'pk':45}) scores.processed = "unprocessed" test_json = { 'games' : games , 'scores' : scores , 'date': datetime.datetime.now() } print(json.dumps(test_json,default=json_debug_handler)) if __name__ == '__main__': test_debug_json()
Output:
{"date": "2013-05-07T01:03:13.098727", "games": ["mario", "contra", "tetris"], "scores": {"pk": 45, "dp": 10}}
Desired output:
{"date": "2013-05-07T01:03:13.098727", "games": { "orig": ["mario", "contra", "tetris"] ,"attrs" : { "src":"console"}} , "scores": { "orig": {"pk": 45, "dp": 10},"attrs": "processed":"unprocessed }}
Does the default handler not work for serializable objects ? If not, how can I override this, without adding toJSON methods to the extended classes ?
Also, there is this version of JSON encoder which does not work :
class JsonDebugEncoder(json.JSONEncoder): def default(self,obj): if isinstance(obj, datetime.datetime): return obj.isoformat() elif isinstance(obj,mDict): return {'orig':obj , 'attrs': vars(obj)} elif isinstance(obj,mList): return {'orig':obj, 'attrs': vars(obj)} else: return json.JSONEncoder.default(self, obj)
If there is a hack with pickle,__getstate__,__setstate__,and then using json.dumps over pickle.loads object , I am open to that as well, I tried , but that did not work.
12 Answers
Answers 1
It seems that to achieve the behavior you want, with the given restrictions, you'll have to delve into the JSONEncoder class a little. Below I've written out a custom JSONEncoder that overrides the iterencode method to pass a custom isinstance method to _make_iterencode. It isn't the cleanest thing in the world, but seems to be the best given the options and it keeps customization to a minimum.
# customencoder.py from json.encoder import (_make_iterencode, JSONEncoder, encode_basestring_ascii, FLOAT_REPR, INFINITY, c_make_encoder, encode_basestring) class CustomObjectEncoder(JSONEncoder): def iterencode(self, o, _one_shot=False): """ Most of the original method has been left untouched. _one_shot is forced to False to prevent c_make_encoder from being used. c_make_encoder is a funcion defined in C, so it's easier to avoid using it than overriding/redefining it. The keyword argument isinstance for _make_iterencode has been set to self.isinstance. This allows for a custom isinstance function to be defined, which can be used to defer the serialization of custom objects to the default method. """ # Force the use of _make_iterencode instead of c_make_encoder _one_shot = False if self.check_circular: markers = {} else: markers = None if self.ensure_ascii: _encoder = encode_basestring_ascii else: _encoder = encode_basestring if self.encoding != 'utf-8': def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): if isinstance(o, str): o = o.decode(_encoding) return _orig_encoder(o) def floatstr(o, allow_nan=self.allow_nan, _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY): if o != o: text = 'NaN' elif o == _inf: text = 'Infinity' elif o == _neginf: text = '-Infinity' else: return _repr(o) if not allow_nan: raise ValueError( "Out of range float values are not JSON compliant: " + repr(o)) return text # Instead of forcing _one_shot to False, you can also just # remove the first part of this conditional statement and only # call _make_iterencode if (_one_shot and c_make_encoder is not None and self.indent is None and not self.sort_keys): _iterencode = c_make_encoder( markers, self.default, _encoder, self.indent, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, self.allow_nan) else: _iterencode = _make_iterencode( markers, self.default, _encoder, self.indent, floatstr, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, _one_shot, isinstance=self.isinstance) return _iterencode(o, 0)
You can now subclass the CustomObjectEncoder so it correctly serializes your custom objects. The CustomObjectEncoder can also do cool stuff like handle nested objects.
# test.py import json import datetime from customencoder import CustomObjectEncoder class MyEncoder(CustomObjectEncoder): def isinstance(self, obj, cls): if isinstance(obj, (mList, mDict)): return False return isinstance(obj, cls) def default(self, obj): """ Defines custom serialization. To avoid circular references, any object that will always fail self.isinstance must be converted to something that is deserializable here. """ if isinstance(obj, datetime.datetime): return obj.isoformat() elif isinstance(obj, mDict): return {"orig": dict(obj), "attrs": vars(obj)} elif isinstance(obj, mList): return {"orig": list(obj), "attrs": vars(obj)} else: return None class mList(list): pass class mDict(dict): pass def main(): zelda = mList(['zelda']) zelda.src = "oldschool" games = mList(['mario', 'contra', 'tetris', zelda]) games.src = 'console' scores = mDict({'dp': 10, 'pk': 45}) scores.processed = "unprocessed" test_json = {'games': games, 'scores': scores, 'date': datetime.datetime.now()} print(json.dumps(test_json, cls=MyEncoder)) if __name__ == '__main__': main()
Answers 2
The answer by FastTurtle might be a much cleaner solution.
Here's something close to what you want based on the technique as explained in my question/answer: Overriding nested JSON encoding of inherited default supported objects like dict, list
import json import datetime class mDict(dict): pass class mList(list): pass class JsonDebugEncoder(json.JSONEncoder): def _iterencode(self, o, markers=None): if isinstance(o, mDict): yield '{"__mDict__": ' # Encode dictionary yield '{"orig": ' for chunk in super(JsonDebugEncoder, self)._iterencode(o, markers): yield chunk yield ', ' # / End of Encode dictionary # Encode attributes yield '"attr": ' for key, value in o.__dict__.iteritems(): yield '{"' + key + '": ' for chunk in super(JsonDebugEncoder, self)._iterencode(value, markers): yield chunk yield '}' yield '}' # / End of Encode attributes yield '}' elif isinstance(o, mList): yield '{"__mList__": ' # Encode list yield '{"orig": ' for chunk in super(JsonDebugEncoder, self)._iterencode(o, markers): yield chunk yield ', ' # / End of Encode list # Encode attributes yield '"attr": ' for key, value in o.__dict__.iteritems(): yield '{"' + key + '": ' for chunk in super(JsonDebugEncoder, self)._iterencode(value, markers): yield chunk yield '}' yield '}' # / End of Encode attributes yield '}' else: for chunk in super(JsonDebugEncoder, self)._iterencode(o, markers=markers): yield chunk def default(self, obj): if isinstance(obj, datetime.datetime): return obj.isoformat() class JsonDebugDecoder(json.JSONDecoder): def decode(self, s): obj = super(JsonDebugDecoder, self).decode(s) obj = self.recursiveObjectDecode(obj) return obj def recursiveObjectDecode(self, obj): if isinstance(obj, dict): decoders = [("__mList__", self.mListDecode), ("__mDict__", self.mDictDecode)] for placeholder, decoder in decoders: if placeholder in obj: # We assume it's supposed to be converted return decoder(obj[placeholder]) else: for k in obj: obj[k] = self.recursiveObjectDecode(obj[k]) elif isinstance(obj, list): for x in range(len(obj)): obj[x] = self.recursiveObjectDecode(obj[x]) return obj def mDictDecode(self, o): res = mDict() for key, value in o['orig'].iteritems(): res[key] = self.recursiveObjectDecode(value) for key, value in o['attr'].iteritems(): res.__dict__[key] = self.recursiveObjectDecode(value) return res def mListDecode(self, o): res = mList() for value in o['orig']: res.append(self.recursiveObjectDecode(value)) for key, value in o['attr'].iteritems(): res.__dict__[key] = self.recursiveObjectDecode(value) return res def test_debug_json(): games = mList(['mario','contra','tetris']) games.src = 'console' scores = mDict({'dp':10,'pk':45}) scores.processed = "unprocessed" test_json = { 'games' : games, 'scores' : scores ,'date': datetime.datetime.now() } jsonDump = json.dumps(test_json, cls=JsonDebugEncoder) print jsonDump test_pyObject = json.loads(jsonDump, cls=JsonDebugDecoder) print test_pyObject if __name__ == '__main__': test_debug_json()
This results in:
{"date": "2013-05-06T22:28:08.967000", "games": {"__mList__": {"orig": ["mario", "contra", "tetris"], "attr": {"src": "console"}}}, "scores": {"__mDict__": {"orig": {"pk": 45, "dp": 10}, "attr": {"processed": "unprocessed"}}}}
This way you can encode it and decode it back to the python object it came from.
EDIT:
Here's a version that actually encodes it to the output you wanted and can decode it as well. Whenever a dictionary contains 'orig' and 'attr' it will check if 'orig' contains a dictionary or a list, if so it will respectively convert the object back to the mDict or mList.
import json import datetime class mDict(dict): pass class mList(list): pass class JsonDebugEncoder(json.JSONEncoder): def _iterencode(self, o, markers=None): if isinstance(o, mDict): # Encode mDict yield '{"orig": ' for chunk in super(JsonDebugEncoder, self)._iterencode(o, markers): yield chunk yield ', ' yield '"attr": ' for key, value in o.__dict__.iteritems(): yield '{"' + key + '": ' for chunk in super(JsonDebugEncoder, self)._iterencode(value, markers): yield chunk yield '}' yield '}' # / End of Encode attributes elif isinstance(o, mList): # Encode mList yield '{"orig": ' for chunk in super(JsonDebugEncoder, self)._iterencode(o, markers): yield chunk yield ', ' yield '"attr": ' for key, value in o.__dict__.iteritems(): yield '{"' + key + '": ' for chunk in super(JsonDebugEncoder, self)._iterencode(value, markers): yield chunk yield '}' yield '}' else: for chunk in super(JsonDebugEncoder, self)._iterencode(o, markers=markers): yield chunk def default(self, obj): if isinstance(obj, datetime.datetime): # Encode datetime return obj.isoformat() class JsonDebugDecoder(json.JSONDecoder): def decode(self, s): obj = super(JsonDebugDecoder, self).decode(s) obj = self.recursiveObjectDecode(obj) return obj def recursiveObjectDecode(self, obj): if isinstance(obj, dict): if "orig" in obj and "attr" in obj and isinstance(obj["orig"], list): return self.mListDecode(obj) elif "orig" in obj and "attr" in obj and isinstance(obj['orig'], dict): return self.mDictDecode(obj) else: for k in obj: obj[k] = self.recursiveObjectDecode(obj[k]) elif isinstance(obj, list): for x in range(len(obj)): obj[x] = self.recursiveObjectDecode(obj[x]) return obj def mDictDecode(self, o): res = mDict() for key, value in o['orig'].iteritems(): res[key] = self.recursiveObjectDecode(value) for key, value in o['attr'].iteritems(): res.__dict__[key] = self.recursiveObjectDecode(value) return res def mListDecode(self, o): res = mList() for value in o['orig']: res.append(self.recursiveObjectDecode(value)) for key, value in o['attr'].iteritems(): res.__dict__[key] = self.recursiveObjectDecode(value) return res def test_debug_json(): games = mList(['mario','contra','tetris']) games.src = 'console' scores = mDict({'dp':10,'pk':45}) scores.processed = "unprocessed" test_json = { 'games' : games, 'scores' : scores ,'date': datetime.datetime.now() } jsonDump = json.dumps(test_json, cls=JsonDebugEncoder) print jsonDump test_pyObject = json.loads(jsonDump, cls=JsonDebugDecoder) print test_pyObject print test_pyObject['games'].src if __name__ == '__main__': test_debug_json()
Here's some more info about the output:
# Encoded {"date": "2013-05-06T22:41:35.498000", "games": {"orig": ["mario", "contra", "tetris"], "attr": {"src": "console"}}, "scores": {"orig": {"pk": 45, "dp": 10}, "attr": {"processed": "unprocessed"}}} # Decoded ('games' contains the mList with the src attribute and 'scores' contains the mDict processed attribute) # Note that printing the python objects doesn't directly show the processed and src attributes, as seen below. {u'date': u'2013-05-06T22:41:35.498000', u'games': [u'mario', u'contra', u'tetris'], u'scores': {u'pk': 45, u'dp': 10}}
Sorry for any bad naming conventions, it's a quick setup. ;)
Note: The datetime doesn't get decoded back to the python representation. Implementing that could be done by checking for any dict key that is called 'date' and contains a valid string representation of a datetime.
Answers 3
As others have pointed out already, the default handler only gets called for values that aren't one of the recognised types. My suggested solution to this problem is to preprocess the object you want to serialize, recursing over lists, tuples and dictionaries, but wrapping every other value in a custom class.
Something like this:
def debug(obj): class Debug: def __init__(self,obj): self.originalObject = obj if obj.__class__ == list: return [debug(item) for item in obj] elif obj.__class__ == tuple: return (debug(item) for item in obj) elif obj.__class__ == dict: return dict((key,debug(obj[key])) for key in obj) else: return Debug(obj)
You would call this function, before passing your object to json.dumps, like this:
test_json = debug(test_json) print(json.dumps(test_json,default=json_debug_handler))
Note that this code is checking for objects whose class exactly matches a list, tuple or dictionary, so any custom objects that are extended from those types will be wrapped rather than parsed. As a result, the regular lists, tuples, and dictionaries will be serialized as usual, but all other values will be passed on to the default handler.
The end result of all this, is that every value that reaches the the default handler is guaranteed to be wrapped in one of these Debug classes. So the first thing you are going to want to do is extract the original object, like this:
obj = obj.originalObject
You can then check the original object's type and handle whichever types need special processing. For everything else, you should just return the original object (so the last return from the handler should be return obj not return None).
def json_debug_handler(obj): obj = obj.originalObject # Add this line print("object received:") print type(obj) print("\n\n") if isinstance(obj, datetime.datetime): return obj.isoformat() elif isinstance(obj,mDict): return {'orig':obj, 'attrs': vars(obj)} elif isinstance(obj,mList): return {'orig':obj, 'attrs': vars(obj)} else: return obj # Change this line
Note that this code doesn't check for values that aren't serializable. These will fall through the final return obj, then will be rejected by the serializer and passed back to the default handler again - only this time without the Debug wrapper.
If you need to deal with that scenario, you could add a check at the top of the handler like this:
if not hasattr(obj, 'originalObject'): return None
Ideone demo: http://ideone.com/tOloNq
Answers 4
The default function is only called when the node being dumped isn't natively serializable, and your mDict classes serialize as-is. Here's a little demo that shows when default is called and when not:
import json def serializer(obj): print 'serializer called' return str(obj) class mDict(dict): pass class mSet(set): pass d = mDict(dict(a=1)) print json.dumps(d, default=serializer) s = mSet({1, 2, 3,}) print json.dumps(s, default=serializer)
And the output:
{"a": 1} serializer called "mSet([1, 2, 3])"
Note that sets are not natively serializable, but dicts are.
Since your m___ classes are serializable, your handler is never called.
Update #1 -----
You could change JSON encoder code. The details of how to do this depend on which JSON implementation you're using. For example in simplejson, the relevant code is this, in encode.py:
def _iterencode(o, _current_indent_level): ... for_json = _for_json and getattr(o, 'for_json', None) if for_json and callable(for_json): ... elif isinstance(o, list): ... else: _asdict = _namedtuple_as_object and getattr(o, '_asdict', None) if _asdict and callable(_asdict): for chunk in _iterencode_dict(_asdict(), _current_indent_level): yield chunk elif (_tuple_as_array and isinstance(o, tuple)): ... elif isinstance(o, dict): ... elif _use_decimal and isinstance(o, Decimal): ... else: ... o = _default(o) for chunk in _iterencode(o, _current_indent_level): yield chunk ...
In other words, there is a hard-wired behavior that calls default only when the node being encoded isn't one of the recognized base types. You could override this in one of several ways:
1 -- subclass JSONEncoder as you've done above, but add a parameter to its initializer that specifies the function to be used in place of the standard _make_iterencode, in which you add a test that would call default for classes that meet your criteria. This is a clean approach since you aren't changing the JSON module, but you would be reiterating a lot of code from the original _make_iterencode. (Other variations on this approach include monkeypatching _make_iterencode or its sub-function _iterencode_dict).
2 -- alter the JSON module source, and use the __debug__ constant to change behavior:
def _iterencode(o, _current_indent_level): ... for_json = _for_json and getattr(o, 'for_json', None) if for_json and callable(for_json): ... elif isinstance(o, list): ... ## added code below elif __debug__: o = _default(o) for chunk in _iterencode(o, _current_indent_level): yield chunk ## added code above else: ...
Ideally the JSONEncoder class would provide a parameter to specify "use default for all types", but it doesn't. The above is a simple one-time change that does what you're looking for.
Answers 5
Why can't you just create a new object type to pass to the encoder? Try:
class MStuff(object): def __init__(self, content): self.content = content class mDict(MStuff): pass class mList(MStuff): pass def json_debug_handler(obj): print("object received:") print(type(obj)) print("\n\n") if isinstance(obj, datetime.datetime): return obj.isoformat() elif isinstance(obj,MStuff): attrs = {} for key in obj.__dict__: if not ( key.startswith("_") or key == "content"): attrs[key] = obj.__dict__[key] return {'orig':obj.content , 'attrs': attrs} else: return None
You could add validation on the mDict and mList if desired.
Answers 6
If you define these to override __instancecheck__:
def strict_check(builtin): '''creates a new class from the builtin whose instance check method can be overridden to renounce particular types''' class BuiltIn(type): def __instancecheck__(self, other): print 'instance', self, type(other), other if type(other) in strict_check.blacklist: return False return builtin.__instancecheck__(other) # construct a class, whose instance check method is known. return BuiltIn('strict_%s' % builtin.__name__, (builtin,), dict()) # for safety, define it here. strict_check.blacklist = ()
then patch json.encoder like this to override _make_iterencode.func_defaults:
# modify json encoder to use some new list/dict attr. import json.encoder # save old stuff, never know when you need it. old_defaults = json.encoder._make_iterencode.func_defaults old_encoder = json.encoder.c_make_encoder encoder_defaults = list(json.encoder._make_iterencode.func_defaults) for index, default in enumerate(encoder_defaults): if default in (list, dict): encoder_defaults[index] = strict_check(default) # change the defaults for _make_iterencode. json.encoder._make_iterencode.func_defaults = tuple(encoder_defaults) # disable C extension. json.encoder.c_make_encoder = None
... your example would almost work verbatim:
import datetime import json def json_debug_handler(obj): print("object received:") print type(obj) print("\n\n") if isinstance(obj, datetime.datetime): return obj.isoformat() elif isinstance(obj,mDict): # degrade obj to more primitive dict() # to avoid cycles in the encoding. return {'orig': dict(obj) , 'attrs': vars(obj)} elif isinstance(obj,mList): # degrade obj to more primitive list() # to avoid cycles in the encoding. return {'orig': list(obj), 'attrs': vars(obj)} else: return None class mDict(dict): pass class mList(list): pass # set the stuff we want to process differently. strict_check.blacklist = (mDict, mList) def test_debug_json(): global test_json games = mList(['mario','contra','tetris']) games.src = 'console' scores = mDict({'dp':10,'pk':45}) scores.processed = "unprocessed" test_json = { 'games' : games , 'scores' : scores , 'date': datetime.datetime.now() } print(json.dumps(test_json,default=json_debug_handler)) if __name__ == '__main__': test_debug_json()
The things I needed to change were to make sure there were no cycles:
elif isinstance(obj,mDict): # degrade obj to more primitive dict() # to avoid cycles in the encoding. return {'orig': dict(obj) , 'attrs': vars(obj)} elif isinstance(obj,mList): # degrade obj to more primitive list() # to avoid cycles in the encoding. return {'orig': list(obj), 'attrs': vars(obj)}
and add this somewhere before test_debug_json:
# set the stuff we want to process differently. strict_check.blacklist = (mDict, mList)
here is my console output:
>>> test_debug_json() instance <class '__main__.strict_list'> <type 'dict'> {'date': datetime.datetime(2013, 7, 17, 12, 4, 40, 950637), 'games': ['mario', 'contra', 'tetris'], 'scores': {'pk': 45, 'dp': 10}} instance <class '__main__.strict_dict'> <type 'dict'> {'date': datetime.datetime(2013, 7, 17, 12, 4, 40, 950637), 'games': ['mario', 'contra', 'tetris'], 'scores': {'pk': 45, 'dp': 10}} instance <class '__main__.strict_list'> <type 'datetime.datetime'> 2013-07-17 12:04:40.950637 instance <class '__main__.strict_dict'> <type 'datetime.datetime'> 2013-07-17 12:04:40.950637 instance <class '__main__.strict_list'> <type 'datetime.datetime'> 2013-07-17 12:04:40.950637 instance <class '__main__.strict_dict'> <type 'datetime.datetime'> 2013-07-17 12:04:40.950637 object received: <type 'datetime.datetime'> instance <class '__main__.strict_list'> <class '__main__.mList'> ['mario', 'contra', 'tetris'] instance <class '__main__.strict_dict'> <class '__main__.mList'> ['mario', 'contra', 'tetris'] instance <class '__main__.strict_list'> <class '__main__.mList'> ['mario', 'contra', 'tetris'] instance <class '__main__.strict_dict'> <class '__main__.mList'> ['mario', 'contra', 'tetris'] object received: <class '__main__.mList'> instance <class '__main__.strict_list'> <type 'dict'> {'attrs': {'src': 'console'}, 'orig': ['mario', 'contra', 'tetris']} instance <class '__main__.strict_dict'> <type 'dict'> {'attrs': {'src': 'console'}, 'orig': ['mario', 'contra', 'tetris']} instance <class '__main__.strict_list'> <type 'dict'> {'src': 'console'} instance <class '__main__.strict_dict'> <type 'dict'> {'src': 'console'} instance <class '__main__.strict_list'> <type 'list'> ['mario', 'contra', 'tetris'] instance <class '__main__.strict_list'> <class '__main__.mDict'> {'pk': 45, 'dp': 10} instance <class '__main__.strict_dict'> <class '__main__.mDict'> {'pk': 45, 'dp': 10} instance <class '__main__.strict_list'> <class '__main__.mDict'> {'pk': 45, 'dp': 10} instance <class '__main__.strict_dict'> <class '__main__.mDict'> {'pk': 45, 'dp': 10} object received: <class '__main__.mDict'> instance <class '__main__.strict_list'> <type 'dict'> {'attrs': {'processed': 'unprocessed'}, 'orig': {'pk': 45, 'dp': 10}} instance <class '__main__.strict_dict'> <type 'dict'> {'attrs': {'processed': 'unprocessed'}, 'orig': {'pk': 45, 'dp': 10}} instance <class '__main__.strict_list'> <type 'dict'> {'processed': 'unprocessed'} instance <class '__main__.strict_dict'> <type 'dict'> {'processed': 'unprocessed'} instance <class '__main__.strict_list'> <type 'dict'> {'pk': 45, 'dp': 10} instance <class '__main__.strict_dict'> <type 'dict'> {'pk': 45, 'dp': 10} {"date": "2013-07-17T12:04:40.950637", "games": {"attrs": {"src": "console"}, "orig": ["mario", "contra", "tetris"]}, "scores": {"attrs": {"processed": "unprocessed"}, "orig": {"pk": 45, "dp": 10}}}
Answers 7
Try the below. It produces the output you want and looks relatively simple. The only real difference from your encoder class is that we should override both decode and encode methods (since the latter is still called for types the encoder knows how to handle).
import json import datetime class JSONDebugEncoder(json.JSONEncoder): # transform objects known to JSONEncoder here def encode(self, o, *args, **kw): for_json = o if isinstance(o, mDict): for_json = { 'orig' : o, 'attrs' : vars(o) } elif isinstance(o, mList): for_json = { 'orig' : o, 'attrs' : vars(o) } return super(JSONDebugEncoder, self).encode(for_json, *args, **kw) # handle objects not known to JSONEncoder here def default(self, o, *args, **kw): if isinstance(o, datetime.datetime): return o.isoformat() else: return super(JSONDebugEncoder, self).default(o, *args, **kw) class mDict(dict): pass class mList(list): pass def test_debug_json(): games = mList(['mario','contra','tetris']) games.src = 'console' scores = mDict({'dp':10,'pk':45}) scores.processed = "unprocessed" test_json = { 'games' : games , 'scores' : scores , 'date': datetime.datetime.now() } print(json.dumps(test_json,cls=JSONDebugEncoder)) if __name__ == '__main__': test_debug_json()
Answers 8
If you are able to change the way json.dumps is called. You can do all the processing required before the JSON encoder gets his hands on it. This version does not use any kind of copying and will edit the structures in-place. You can add copy() if required.
import datetime import json import collections def json_debug_handler(obj): print("object received:") print type(obj) print("\n\n") if isinstance(obj, collections.Mapping): for key, value in obj.iteritems(): if isinstance(value, (collections.Mapping, collections.MutableSequence)): value = json_debug_handler(value) obj[key] = convert(value) elif isinstance(obj, collections.MutableSequence): for index, value in enumerate(obj): if isinstance(value, (collections.Mapping, collections.MutableSequence)): value = json_debug_handler(value) obj[index] = convert(value) return obj def convert(obj): if isinstance(obj, datetime.datetime): return obj.isoformat() elif isinstance(obj,mDict): return {'orig':obj , 'attrs': vars(obj)} elif isinstance(obj,mList): return {'orig':obj, 'attrs': vars(obj)} else: return obj class mDict(dict): pass class mList(list): pass def test_debug_json(): games = mList(['mario','contra','tetris']) games.src = 'console' scores = mDict({'dp':10,'pk':45}) scores.processed = "qunprocessed" test_json = { 'games' : games , 'scores' : scores , 'date': datetime.datetime.now() } print(json.dumps(json_debug_handler(test_json))) if __name__ == '__main__': test_debug_json()
You call json_debug_handler on the object you are serializing before passing it to the json.dumps. With this pattern you could also easily reverse the changes and/or add extra conversion rules.
edit:
If you can't change how json.dumps is called, you can always monkeypatch it to do what you want. Such as doing this:
json.dumps = lambda obj, *args, **kwargs: json.dumps(json_debug_handler(obj), *args, **kwargs)
Answers 9
You should be able to override JSONEncoder.encode():
class MyEncoder(JSONEncoder): def encode(self, o): if isinstance(o, dict): # directly call JSONEncoder rather than infinite-looping through self.encode() return JSONEncoder.encode(self, {'orig': o, 'attrs': vars(o)}) elif isinstance(o, list): return JSONEncoder.encode(self, {'orig': o, 'attrs': vars(o)}) else: return JSONEncoder.encode(self, o)
and then if you want to patch it into json.dumps it looks from http://docs.buildbot.net/latest/reference/json-pysrc.html like you'll need to replace json._default_encoder with an instance of MyEncoder.
Answers 10
If you are only looking for serialization and not deserialization then you can process the object before sending it to json.dumps. See below example
import datetime import json def is_inherited_from(obj, objtype): return isinstance(obj, objtype) and not type(obj).__mro__[0] == objtype def process_object(data): if isinstance(data, list): if is_inherited_from(data, list): return process_object({"orig": list(data), "attrs": vars(data)}) new_data = [] for d in data: new_data.append(process_object(d)) elif isinstance(data, tuple): if is_inherited_from(data, tuple): return process_object({"orig": tuple(data), "attrs": vars(data)}) new_data = [] for d in data: new_data.append(process_object(d)) return tuple(new_data) elif isinstance(data, dict): if is_inherited_from(data, dict): return process_object({"orig": list(data), "attrs": vars(data)}) new_data = {} for k, v in data.items(): new_data[k] = process_object(v) else: return data return new_data def json_debug_handler(obj): print("object received:") print("\n\n") if isinstance(obj, datetime.datetime): return obj.isoformat() class mDict(dict): pass class mList(list): pass def test_debug_json(): games = mList(['mario', 'contra', 'tetris']) games.src = 'console' scores = mDict({'dp': 10, 'pk': 45}) scores.processed = "unprocessed" test_json = {'games': games, 'scores': scores, 'date': datetime.datetime.now()} new_object = process_object(test_json) print(json.dumps(new_object, default=json_debug_handler)) if __name__ == '__main__': test_debug_json()
The output of the same is
{"games": {"orig": ["mario", "contra", "tetris"], "attrs": {"src": "console"}}, "scores": {"orig": ["dp", "pk"], "attrs": {"processed": "unprocessed"}}, "date": "2018-01-24T12:59:36.581689"}
It is also possible to override the JSONEncoder, but since it uses nested methods, it would be complex and require techniques discussed in below
Can you patch *just* a nested function with closure, or must the whole outer function be repeated?
Since you want to keep things simple, I would not suggest going that route
Answers 11
Along the lines of FastTurtle's suggestion, but requiring somewhat less code and much deeper monkeying, you can override isinstance itself, globally. This is probably Not A Good Idea, and may well break something. But it does work, in that it produces your required output, and it's quite simple.
First, before json is imported anywhere, monkey-patch the builtins module to replace isinstance with one that lies, just a little bit, and only in a specific context:
_original_isinstance = isinstance def _isinstance(obj, class_or_tuple): if '_make_iterencode' in globals(): if not _original_isinstance(class_or_tuple, tuple): class_or_tuple = (class_or_tuple,) for custom in mList, mDict: if _original_isinstance(obj, custom): return custom in class_or_tuple return _original_isinstance(obj, class_or_tuple) try: import builtins # Python 3 except ImportError: import __builtin__ as builtins # Python 2 builtins.isinstance = _isinstance
Then, create your custom encoder, implementing your custom serialization and forcing the use of _make_iterencode (since the c version won't be affected by the monkeypatching):
class CustomEncoder(json.JSONEncoder): def iterencode(self, o, _one_shot = False): return super(CustomEncoder, self).iterencode(o, _one_shot=False) def default(self, obj): if isinstance(obj, datetime.datetime): return obj.isoformat() elif isinstance(obj,mDict): return {'orig':dict(obj) , 'attrs': vars(obj)} elif isinstance(obj,mList): return {'orig':list(obj), 'attrs': vars(obj)} else: return None
And that's really all there is to it! Output from Python 3 and Python 2 below.
Python 3.6.3 (default, Oct 10 2017, 21:06:48) ... >>> from test import test_debug_json >>> test_debug_json() {"games": {"orig": ["mario", "contra", "tetris"], "attrs": {"src": "console"}}, "scores": {"orig": {"dp": 10, "pk": 45}, "attrs": {"processed": "unprocessed"}}, "date": "2018-01-27T13:56:15.666655"} Python 2.7.13 (default, May 9 2017, 12:06:13) ... >>> from test import test_debug_json >>> test_debug_json() {"date": "2018-01-27T13:57:04.681664", "games": {"attrs": {"src": "console"}, "orig": ["mario", "contra", "tetris"]}, "scores": {"attrs": {"processed": "unprocessed"}, "orig": {"pk": 45, "dp": 10}}}
Answers 12
Can we just preprocess the test_json,to make it suitable for your requirement? It's easier to manipulate a python dict than write a useless Encode.
import datetime import json class mDict(dict): pass class mList(list): pass def prepare(obj): if isinstance(obj, datetime.datetime): return obj.isoformat() elif isinstance(obj, mDict): return {'orig':obj , 'attrs': vars(obj)} elif isinstance(obj, mList): return {'orig':obj, 'attrs': vars(obj)} else: return obj def preprocessor(toJson): ret ={} for key, value in toJson.items(): ret[key] = prepare(value) return ret if __name__ == '__main__': def test_debug_json(): games = mList(['mario','contra','tetris']) games.src = 'console' scores = mDict({'dp':10,'pk':45}) scores.processed = "unprocessed" test_json = { 'games' : games, 'scores' : scores , 'date': datetime.datetime.now() } print(json.dumps(preprocessor(test_json))) test_debug_json()
