diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index 1ba8b48bd..e4c21daaf 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -133,7 +133,7 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). + result in an ``RecursionError`` (or worse). If ``allow_nan`` is false, then it will be a ``ValueError`` to serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) @@ -195,7 +195,7 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). + result in an ``RecursionError`` (or worse). If ``allow_nan`` is false, then it will be a ``ValueError`` to serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in @@ -329,8 +329,6 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. - - The ``encoding`` argument is ignored and deprecated since Python 3.1. """ if isinstance(s, str): if s.startswith('\ufeff'): @@ -342,15 +340,6 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, f'not {s.__class__.__name__}') s = s.decode(detect_encoding(s), 'surrogatepass') - if "encoding" in kw: - import warnings - warnings.warn( - "'encoding' is ignored and deprecated. It will be removed in Python 3.9", - DeprecationWarning, - stacklevel=2 - ) - del kw['encoding'] - if (cls is None and object_hook is None and parse_int is None and parse_float is None and parse_constant is None and object_pairs_hook is None and not kw): diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index c8c78b9c2..21bff2c1a 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -116,7 +116,7 @@ class JSONEncoder(object): If check_circular is true, then lists, dicts, and custom encoded objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause an OverflowError). + prevent an infinite recursion (which would cause an RecursionError). Otherwise, no such check takes place. If allow_nan is true, then NaN, Infinity, and -Infinity will be diff --git a/Lib/json/tool.py b/Lib/json/tool.py index 8db9ea40a..0490b8c0b 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -13,6 +13,7 @@ Usage:: import argparse import json import sys +from pathlib import Path def main(): @@ -25,31 +26,60 @@ def main(): help='a JSON file to be validated or pretty-printed', default=sys.stdin) parser.add_argument('outfile', nargs='?', - type=argparse.FileType('w', encoding="utf-8"), + type=Path, help='write the output of infile to outfile', - default=sys.stdout) + default=None) parser.add_argument('--sort-keys', action='store_true', default=False, help='sort the output of dictionaries alphabetically by key') + parser.add_argument('--no-ensure-ascii', dest='ensure_ascii', action='store_false', + help='disable escaping of non-ASCII characters') parser.add_argument('--json-lines', action='store_true', default=False, - help='parse input using the jsonlines format') + help='parse input using the JSON Lines format. ' + 'Use with --no-indent or --compact to produce valid JSON Lines output.') + group = parser.add_mutually_exclusive_group() + group.add_argument('--indent', default=4, type=int, + help='separate items with newlines and use this number ' + 'of spaces for indentation') + group.add_argument('--tab', action='store_const', dest='indent', + const='\t', help='separate items with newlines and use ' + 'tabs for indentation') + group.add_argument('--no-indent', action='store_const', dest='indent', + const=None, + help='separate items with spaces rather than newlines') + group.add_argument('--compact', action='store_true', + help='suppress all whitespace separation (most compact)') options = parser.parse_args() - infile = options.infile - outfile = options.outfile - sort_keys = options.sort_keys - json_lines = options.json_lines - with infile, outfile: + dump_args = { + 'sort_keys': options.sort_keys, + 'indent': options.indent, + 'ensure_ascii': options.ensure_ascii, + } + if options.compact: + dump_args['indent'] = None + dump_args['separators'] = ',', ':' + + with options.infile as infile: try: - if json_lines: + if options.json_lines: objs = (json.loads(line) for line in infile) else: - objs = (json.load(infile), ) - for obj in objs: - json.dump(obj, outfile, sort_keys=sort_keys, indent=4) - outfile.write('\n') + objs = (json.load(infile),) + + if options.outfile is None: + out = sys.stdout + else: + out = options.outfile.open('w', encoding='utf-8') + with out as outfile: + for obj in objs: + json.dump(obj, outfile, **dump_args) + outfile.write('\n') except ValueError as e: raise SystemExit(e) if __name__ == '__main__': - main() + try: + main() + except BrokenPipeError as exc: + sys.exit(exc.errno) diff --git a/Lib/test/test_json/__init__.py b/Lib/test/test_json/__init__.py index e236bb23e..f43bded1a 100644 --- a/Lib/test/test_json/__init__.py +++ b/Lib/test/test_json/__init__.py @@ -6,6 +6,7 @@ import unittest from test import support from test.support import import_helper + # import json with and without accelerations # XXX RUSTPYTHON: we don't import _json as fresh since the fresh module isn't placed # into the sys.modules cache, and therefore the vm can't recognize the _json.Scanner class @@ -40,7 +41,6 @@ class TestPyTest(PyTest): 'json.encoder') class TestCTest(CTest): - @unittest.expectedFailure def test_cjson(self): self.assertEqual(self.json.scanner.make_scanner.__module__, '_json') self.assertEqual(self.json.decoder.scanstring.__module__, '_json') diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index 1ab3a2aa7..e48e36c54 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -97,10 +97,6 @@ class TestDecode: d = self.json.JSONDecoder() self.assertRaises(ValueError, d.raw_decode, 'a'*42, -50000) - def test_deprecated_encode(self): - with self.assertWarns(DeprecationWarning): - self.loads('{}', encoding='fake') - class TestPyDecode(TestDecode, PyTest): pass # TODO: RUSTPYTHON class TestCDecode(TestDecode, CTest): # pass diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 877dc448b..9919d7fbe 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -1,3 +1,4 @@ +from test import support from test.test_json import PyTest, CTest @@ -52,7 +53,7 @@ class TestRecursion: return [JSONTestObject] else: return 'JSONTestObject' - return pyjson.JSONEncoder.default(o) + return self.json.JSONEncoder.default(o) enc = RecursiveJSONEncoder() self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"') @@ -69,11 +70,14 @@ class TestRecursion: # test that loading highly-nested objects doesn't segfault when C # accelerations are used. See #12017 with self.assertRaises(RecursionError): - self.loads('{"a":' * 100000 + '1' + '}' * 100000) + with support.infinite_recursion(): + self.loads('{"a":' * 100000 + '1' + '}' * 100000) with self.assertRaises(RecursionError): - self.loads('{"a":' * 100000 + '[1]' + '}' * 100000) + with support.infinite_recursion(): + self.loads('{"a":' * 100000 + '[1]' + '}' * 100000) with self.assertRaises(RecursionError): - self.loads('[' * 100000 + '1' + ']' * 100000) + with support.infinite_recursion(): + self.loads('[' * 100000 + '1' + ']' * 100000) def test_highly_nested_objects_encoding(self): # See #12051 @@ -81,9 +85,11 @@ class TestRecursion: for x in range(100000): l, d = [l], {'k':d} with self.assertRaises(RecursionError): - self.dumps(l) + with support.infinite_recursion(): + self.dumps(l) with self.assertRaises(RecursionError): - self.dumps(d) + with support.infinite_recursion(): + self.dumps(d) def test_endless_recursion(self): # See #12051 @@ -93,7 +99,8 @@ class TestRecursion: return [o] with self.assertRaises(RecursionError): - EndlessJSONEncoder(check_circular=False).encode(5j) + with support.infinite_recursion(): + EndlessJSONEncoder(check_circular=False).encode(5j) class TestPyRecursion(TestRecursion, PyTest): pass diff --git a/Lib/test/test_json/test_speedups.py b/Lib/test/test_json/test_speedups.py index 1bf34ecba..214bfa45c 100644 --- a/Lib/test/test_json/test_speedups.py +++ b/Lib/test/test_json/test_speedups.py @@ -62,6 +62,15 @@ class TestEncode(CTest): with self.assertRaises(ZeroDivisionError): enc('spam', 4) + def test_bad_markers_argument_to_encoder(self): + # https://bugs.python.org/issue45269 + with self.assertRaisesRegex( + TypeError, + r'make_encoder\(\) argument 1 must be dict or None, not int', + ): + self.json.encoder.c_make_encoder(1, None, None, None, ': ', ', ', + False, False, False) + # TODO: RUSTPYTHON, translate the encoder to Rust @unittest.expectedFailure def test_bad_bool_args(self): diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 9f2329ad2..1d7fca6ef 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -1,8 +1,10 @@ +import errno import os import sys import textwrap import unittest -from subprocess import Popen, PIPE +import subprocess + from test import support from test.support import os_helper from test.support.script_helper import assert_python_ok @@ -85,10 +87,9 @@ class TestTool(unittest.TestCase): def test_stdin_stdout(self): args = sys.executable, '-m', 'json.tool' - with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc: - out, err = proc.communicate(self.data.encode()) - self.assertEqual(out.splitlines(), self.expect.encode().splitlines()) - self.assertEqual(err, b'') + process = subprocess.run(args, input=self.data, capture_output=True, text=True, check=True) + self.assertEqual(process.stdout, self.expect) + self.assertEqual(process.stderr, '') def _create_infile(self, data=None): infile = os_helper.TESTFN @@ -124,7 +125,16 @@ class TestTool(unittest.TestCase): outfile = os_helper.TESTFN + '.out' rc, out, err = assert_python_ok('-m', 'json.tool', infile, outfile) self.addCleanup(os.remove, outfile) - with open(outfile, "r") as fp: + with open(outfile, "r", encoding="utf-8") as fp: + self.assertEqual(fp.read(), self.expect) + self.assertEqual(rc, 0) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + + def test_writing_in_place(self): + infile = self._create_infile() + rc, out, err = assert_python_ok('-m', 'json.tool', infile, infile) + with open(infile, "r", encoding="utf-8") as fp: self.assertEqual(fp.read(), self.expect) self.assertEqual(rc, 0) self.assertEqual(out, b'') @@ -132,10 +142,9 @@ class TestTool(unittest.TestCase): def test_jsonlines(self): args = sys.executable, '-m', 'json.tool', '--json-lines' - with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc: - out, err = proc.communicate(self.jsonlines_raw.encode()) - self.assertEqual(out.splitlines(), self.jsonlines_expect.encode().splitlines()) - self.assertEqual(err, b'') + process = subprocess.run(args, input=self.jsonlines_raw, capture_output=True, text=True, check=True) + self.assertEqual(process.stdout, self.jsonlines_expect) + self.assertEqual(process.stderr, '') def test_help_flag(self): rc, out, err = assert_python_ok('-m', 'json.tool', '-h') @@ -150,3 +159,73 @@ class TestTool(unittest.TestCase): self.assertEqual(out.splitlines(), self.expect_without_sort_keys.encode().splitlines()) self.assertEqual(err, b'') + + def test_indent(self): + input_ = '[1, 2]' + expect = textwrap.dedent('''\ + [ + 1, + 2 + ] + ''') + args = sys.executable, '-m', 'json.tool', '--indent', '2' + process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) + self.assertEqual(process.stdout, expect) + self.assertEqual(process.stderr, '') + + def test_no_indent(self): + input_ = '[1,\n2]' + expect = '[1, 2]\n' + args = sys.executable, '-m', 'json.tool', '--no-indent' + process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) + self.assertEqual(process.stdout, expect) + self.assertEqual(process.stderr, '') + + def test_tab(self): + input_ = '[1, 2]' + expect = '[\n\t1,\n\t2\n]\n' + args = sys.executable, '-m', 'json.tool', '--tab' + process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) + self.assertEqual(process.stdout, expect) + self.assertEqual(process.stderr, '') + + def test_compact(self): + input_ = '[ 1 ,\n 2]' + expect = '[1,2]\n' + args = sys.executable, '-m', 'json.tool', '--compact' + process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) + self.assertEqual(process.stdout, expect) + self.assertEqual(process.stderr, '') + + def test_no_ensure_ascii_flag(self): + infile = self._create_infile('{"key":"💩"}') + outfile = os_helper.TESTFN + '.out' + self.addCleanup(os.remove, outfile) + assert_python_ok('-m', 'json.tool', '--no-ensure-ascii', infile, outfile) + with open(outfile, "rb") as f: + lines = f.read().splitlines() + # asserting utf-8 encoded output file + expected = [b'{', b' "key": "\xf0\x9f\x92\xa9"', b"}"] + self.assertEqual(lines, expected) + + def test_ensure_ascii_default(self): + infile = self._create_infile('{"key":"💩"}') + outfile = os_helper.TESTFN + '.out' + self.addCleanup(os.remove, outfile) + assert_python_ok('-m', 'json.tool', infile, outfile) + with open(outfile, "rb") as f: + lines = f.read().splitlines() + # asserting an ascii encoded output file + expected = [b'{', rb' "key": "\ud83d\udca9"', b"}"] + self.assertEqual(lines, expected) + + @unittest.skipIf(sys.platform =="win32", "The test is failed with ValueError on Windows") + def test_broken_pipe_error(self): + cmd = [sys.executable, '-m', 'json.tool'] + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE) + # bpo-39828: Closing before json.tool attempts to write into stdout. + proc.stdout.close() + proc.communicate(b'"{}"') + self.assertEqual(proc.returncode, errno.EPIPE)