From ae815839e89132be64ff55e02b04eea94b2508c2 Mon Sep 17 00:00:00 2001 From: Robert Booth Date: Tue, 20 Apr 2021 00:09:17 +0000 Subject: [PATCH 1/2] Change Rust zlib compression code for zlib tests - decompress method: Amend error message - Pass data chunks on a per-slice basis to compression API --- Lib/test/test_zlib.py | 4 ---- vm/src/stdlib/zlib.rs | 23 ++++++++++++++--------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 3064346bc..6499ed7ab 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -187,8 +187,6 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): for ob in x, bytearray(x): self.assertEqual(zlib.decompress(ob), data) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_incomplete_stream(self): # A useful error message is given x = zlib.compress(HAMLET_SCENE) @@ -482,7 +480,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): "mode=%i, level=%i") % (sync, level)) del obj - @unittest.skip("TODO: RUSTPYTHON, panics") @unittest.skipUnless(hasattr(zlib, 'Z_SYNC_FLUSH'), 'requires zlib.Z_SYNC_FLUSH') def test_odd_flush(self): @@ -746,7 +743,6 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): # Memory use of the following functions takes into account overallocation - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'range start index 27394048 out of range for slice of length 10485760'") @bigmemtest(size=_1G + 1024 * 1024, memuse=3) def test_big_compress_buffer(self, size): c = zlib.compressobj(1) diff --git a/vm/src/stdlib/zlib.rs b/vm/src/stdlib/zlib.rs index ffa1b9a63..06cdb6959 100644 --- a/vm/src/stdlib/zlib.rs +++ b/vm/src/stdlib/zlib.rs @@ -246,7 +246,10 @@ mod decl { if stream_end { Ok(buf) } else { - Err(new_zlib_error("incomplete or truncated stream", vm)) + Err(new_zlib_error( + "Error -5 while decompressing data: incomplete or truncated stream", + vm, + )) } }) }) @@ -477,34 +480,36 @@ mod decl { const CHUNKSIZE: usize = u32::MAX as usize; impl CompressInner { - fn save_unconsumed_input(&mut self, data: &[u8], orig_in: u64) { - let leftover = &data[(self.compress.total_in() - orig_in) as usize..]; + fn save_unconsumed_input(&mut self, data: &[u8], cur_in: usize) { + let leftover = &data[cur_in..]; self.unconsumed.extend_from_slice(leftover); } fn compress(&mut self, data: &[u8], vm: &VirtualMachine) -> PyResult> { - let orig_in = self.compress.total_in(); + let orig_in = self.compress.total_in() as usize; + let mut cur_in = 0; let unconsumed = std::mem::take(&mut self.unconsumed); let mut buf = Vec::new(); 'outer: for chunk in unconsumed.chunks(CHUNKSIZE).chain(data.chunks(CHUNKSIZE)) { - loop { + while cur_in < chunk.len() { buf.reserve(DEF_BUF_SIZE); let status = self .compress - .compress_vec(chunk, &mut buf, FlushCompress::None) + .compress_vec(&chunk[cur_in..], &mut buf, FlushCompress::None) .map_err(|_| { - self.save_unconsumed_input(data, orig_in); + self.save_unconsumed_input(data, cur_in); new_zlib_error("error while compressing", vm) })?; + cur_in = (self.compress.total_in() as usize) - orig_in; match status { - _ if buf.len() == buf.capacity() => continue, + Status::Ok => continue, Status::StreamEnd => break 'outer, _ => break, } } } - self.save_unconsumed_input(data, orig_in); + self.save_unconsumed_input(data, cur_in); buf.shrink_to_fit(); Ok(buf) From cc2382b9b9fa30c37ba7eefe74bdc5fdeab79000 Mon Sep 17 00:00:00 2001 From: Robert Booth Date: Thu, 22 Apr 2021 07:08:16 +0900 Subject: [PATCH 2/2] inline code from and remove save_unconsumed_input method --- vm/src/stdlib/zlib.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/vm/src/stdlib/zlib.rs b/vm/src/stdlib/zlib.rs index 06cdb6959..fd3637538 100644 --- a/vm/src/stdlib/zlib.rs +++ b/vm/src/stdlib/zlib.rs @@ -480,11 +480,6 @@ mod decl { const CHUNKSIZE: usize = u32::MAX as usize; impl CompressInner { - fn save_unconsumed_input(&mut self, data: &[u8], cur_in: usize) { - let leftover = &data[cur_in..]; - self.unconsumed.extend_from_slice(leftover); - } - fn compress(&mut self, data: &[u8], vm: &VirtualMachine) -> PyResult> { let orig_in = self.compress.total_in() as usize; let mut cur_in = 0; @@ -498,7 +493,7 @@ mod decl { .compress .compress_vec(&chunk[cur_in..], &mut buf, FlushCompress::None) .map_err(|_| { - self.save_unconsumed_input(data, cur_in); + self.unconsumed.extend_from_slice(&data[cur_in..]); new_zlib_error("error while compressing", vm) })?; cur_in = (self.compress.total_in() as usize) - orig_in; @@ -509,7 +504,7 @@ mod decl { } } } - self.save_unconsumed_input(data, cur_in); + self.unconsumed.extend_from_slice(&data[cur_in..]); buf.shrink_to_fit(); Ok(buf)