0001"""
0002Implementation of JSONDecoder
0003"""
0004import re
0005import sys
0006
0007from simplejson.scanner import Scanner, pattern
0008
0009FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
0010
0011def _floatconstants():
0012 import struct
0013 import sys
0014 _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
0015 if sys.byteorder != 'big':
0016 _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
0017 nan, inf = struct.unpack('dd', _BYTES)
0018 return nan, inf, -inf
0019
0020NaN, PosInf, NegInf = _floatconstants()
0021
0022def linecol(doc, pos):
0023 lineno = doc.count('\n', 0, pos) + 1
0024 if lineno == 1:
0025 colno = pos
0026 else:
0027 colno = pos - doc.rindex('\n', 0, pos)
0028 return lineno, colno
0029
0030def errmsg(msg, doc, pos, end=None):
0031 lineno, colno = linecol(doc, pos)
0032 if end is None:
0033 return '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos)
0034 endlineno, endcolno = linecol(doc, end)
0035 return '%s: line %d column %d - line %d column %d (char %d - %d)' % (
0036 msg, lineno, colno, endlineno, endcolno, pos, end)
0037
0038_CONSTANTS = {
0039 '-Infinity': NegInf,
0040 'Infinity': PosInf,
0041 'NaN': NaN,
0042 'true': True,
0043 'false': False,
0044 'null': None,
0045}
0046
0047def JSONConstant(match, context, c=_CONSTANTS):
0048 s = match.group(0)
0049 fn = getattr(context, 'parse_constant', None)
0050 if fn is None:
0051 rval = c[s]
0052 else:
0053 rval = fn(s)
0054 return rval, None
0055pattern('(-?Infinity|NaN|true|false|null)')(JSONConstant)
0056
0057def JSONNumber(match, context):
0058 match = JSONNumber.regex.match(match.string, *match.span())
0059 integer, frac, exp = match.groups()
0060 if frac or exp:
0061 fn = getattr(context, 'parse_float', None) or float
0062 res = fn(integer + (frac or '') + (exp or ''))
0063 else:
0064 fn = getattr(context, 'parse_int', None) or int
0065 res = fn(integer)
0066 return res, None
0067pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber)
0068
0069STRINGCHUNK = re.compile(r'(.*?)(["\\])', FLAGS)
0070BACKSLASH = {
0071 '"': u'"', '\\': u'\\', '/': u'/',
0072 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
0073}
0074
0075DEFAULT_ENCODING = "utf-8"
0076
0077def scanstring(s, end, encoding=None, _b=BACKSLASH, _m=STRINGCHUNK.match):
0078 if encoding is None:
0079 encoding = DEFAULT_ENCODING
0080 chunks = []
0081 _append = chunks.append
0082 begin = end - 1
0083 while 1:
0084 chunk = _m(s, end)
0085 if chunk is None:
0086 raise ValueError(
0087 errmsg("Unterminated string starting at", s, begin))
0088 end = chunk.end()
0089 content, terminator = chunk.groups()
0090 if content:
0091 if not isinstance(content, unicode):
0092 content = unicode(content, encoding)
0093 _append(content)
0094 if terminator == '"':
0095 break
0096 try:
0097 esc = s[end]
0098 except IndexError:
0099 raise ValueError(
0100 errmsg("Unterminated string starting at", s, begin))
0101 if esc != 'u':
0102 try:
0103 m = _b[esc]
0104 except KeyError:
0105 raise ValueError(
0106 errmsg("Invalid \\escape: %r" % (esc,), s, end))
0107 end += 1
0108 else:
0109 esc = s[end + 1:end + 5]
0110 next_end = end + 5
0111 msg = "Invalid \\uXXXX escape"
0112 try:
0113 if len(esc) != 4 or not esc.isalnum():
0114 raise ValueError
0115 uni = int(esc, 16)
0116 if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
0117 msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
0118 if not s[end + 5:end + 7] == '\\u':
0119 raise ValueError
0120 esc2 = s[end + 7:end + 11]
0121 if len(esc2) != 4 or not esc2.isalnum():
0122 raise ValueError
0123 uni2 = int(esc2, 16)
0124 uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
0125 next_end += 6
0126 m = unichr(uni)
0127 except ValueError:
0128 raise ValueError(errmsg(msg, s, end))
0129 end = next_end
0130 _append(m)
0131 return u''.join(chunks), end
0132
0133def JSONString(match, context):
0134 encoding = getattr(context, 'encoding', None)
0135 return scanstring(match.string, match.end(), encoding)
0136pattern(r'"')(JSONString)
0137
0138WHITESPACE = re.compile(r'\s*', FLAGS)
0139
0140def JSONObject(match, context, _w=WHITESPACE.match):
0141 pairs = {}
0142 s = match.string
0143 end = _w(s, match.end()).end()
0144 nextchar = s[end:end + 1]
0145
0146 if nextchar == '}':
0147 return pairs, end + 1
0148 if nextchar != '"':
0149 raise ValueError(errmsg("Expecting property name", s, end))
0150 end += 1
0151 encoding = getattr(context, 'encoding', None)
0152 iterscan = JSONScanner.iterscan
0153 while True:
0154 key, end = scanstring(s, end, encoding)
0155 end = _w(s, end).end()
0156 if s[end:end + 1] != ':':
0157 raise ValueError(errmsg("Expecting : delimiter", s, end))
0158 end = _w(s, end + 1).end()
0159 try:
0160 value, end = iterscan(s, idx=end, context=context).next()
0161 except StopIteration:
0162 raise ValueError(errmsg("Expecting object", s, end))
0163 pairs[key] = value
0164 end = _w(s, end).end()
0165 nextchar = s[end:end + 1]
0166 end += 1
0167 if nextchar == '}':
0168 break
0169 if nextchar != ',':
0170 raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
0171 end = _w(s, end).end()
0172 nextchar = s[end:end + 1]
0173 end += 1
0174 if nextchar != '"':
0175 raise ValueError(errmsg("Expecting property name", s, end - 1))
0176 object_hook = getattr(context, 'object_hook', None)
0177 if object_hook is not None:
0178 pairs = object_hook(pairs)
0179 return pairs, end
0180pattern(r'{')(JSONObject)
0181
0182def JSONArray(match, context, _w=WHITESPACE.match):
0183 values = []
0184 s = match.string
0185 end = _w(s, match.end()).end()
0186
0187 nextchar = s[end:end + 1]
0188 if nextchar == ']':
0189 return values, end + 1
0190 iterscan = JSONScanner.iterscan
0191 while True:
0192 try:
0193 value, end = iterscan(s, idx=end, context=context).next()
0194 except StopIteration:
0195 raise ValueError(errmsg("Expecting object", s, end))
0196 values.append(value)
0197 end = _w(s, end).end()
0198 nextchar = s[end:end + 1]
0199 end += 1
0200 if nextchar == ']':
0201 break
0202 if nextchar != ',':
0203 raise ValueError(errmsg("Expecting , delimiter", s, end))
0204 end = _w(s, end).end()
0205 return values, end
0206pattern(r'\[')(JSONArray)
0207
0208ANYTHING = [
0209 JSONObject,
0210 JSONArray,
0211 JSONString,
0212 JSONConstant,
0213 JSONNumber,
0214]
0215
0216JSONScanner = Scanner(ANYTHING)
0217
0218class JSONDecoder(object):
0219 """
0220 Simple JSON <http://json.org> decoder
0221
0222 Performs the following translations in decoding by default:
0223
0224 +---------------+-------------------+
0225 | JSON | Python |
0226 +===============+===================+
0227 | object | dict |
0228 +---------------+-------------------+
0229 | array | list |
0230 +---------------+-------------------+
0231 | string | unicode |
0232 +---------------+-------------------+
0233 | number (int) | int, long |
0234 +---------------+-------------------+
0235 | number (real) | float |
0236 +---------------+-------------------+
0237 | true | True |
0238 +---------------+-------------------+
0239 | false | False |
0240 +---------------+-------------------+
0241 | null | None |
0242 +---------------+-------------------+
0243
0244 It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
0245 their corresponding ``float`` values, which is outside the JSON spec.
0246 """
0247
0248 _scanner = Scanner(ANYTHING)
0249 __all__ = ['__init__', 'decode', 'raw_decode']
0250
0251 def __init__(self, encoding=None, object_hook=None, parse_float=None,
0252 parse_int=None, parse_constant=None):
0253 """
0254 ``encoding`` determines the encoding used to interpret any ``str``
0255 objects decoded by this instance (utf-8 by default). It has no
0256 effect when decoding ``unicode`` objects.
0257
0258 Note that currently only encodings that are a superset of ASCII work,
0259 strings of other encodings should be passed in as ``unicode``.
0260
0261 ``object_hook``, if specified, will be called with the result
0262 of every JSON object decoded and its return value will be used in
0263 place of the given ``dict``. This can be used to provide custom
0264 deserializations (e.g. to support JSON-RPC class hinting).
0265
0266 ``parse_float``, if specified, will be called with the string
0267 of every JSON float to be decoded. By default this is equivalent to
0268 float(num_str). This can be used to use another datatype or parser
0269 for JSON floats (e.g. decimal.Decimal).
0270
0271 ``parse_int``, if specified, will be called with the string
0272 of every JSON int to be decoded. By default this is equivalent to
0273 int(num_str). This can be used to use another datatype or parser
0274 for JSON integers (e.g. float).
0275
0276 ``parse_constant``, if specified, will be called with one of the
0277 following strings: -Infinity, Infinity, NaN, null, true, false.
0278 This can be used to raise an exception if invalid JSON numbers
0279 are encountered.
0280 """
0281 self.encoding = encoding
0282 self.object_hook = object_hook
0283 self.parse_float = parse_float
0284 self.parse_int = parse_int
0285 self.parse_constant = parse_constant
0286
0287 def decode(self, s, _w=WHITESPACE.match):
0288 """
0289 Return the Python representation of ``s`` (a ``str`` or ``unicode``
0290 instance containing a JSON document)
0291 """
0292 obj, end = self.raw_decode(s, idx=_w(s, 0).end())
0293 end = _w(s, end).end()
0294 if end != len(s):
0295 raise ValueError(errmsg("Extra data", s, end, len(s)))
0296 return obj
0297
0298 def raw_decode(self, s, **kw):
0299 """
0300 Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
0301 with a JSON document) and return a 2-tuple of the Python
0302 representation and the index in ``s`` where the document ended.
0303
0304 This can be used to decode a JSON document from a string that may
0305 have extraneous data at the end.
0306 """
0307 kw.setdefault('context', self)
0308 try:
0309 obj, end = self._scanner.iterscan(s, **kw).next()
0310 except StopIteration:
0311 raise ValueError("No JSON object could be decoded")
0312 return obj, end
0313
0314__all__ = ['JSONDecoder']