diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7271390c8..a7be2b4c9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,6 +22,7 @@ jobs: "C:\Program Files\Git\mingw64\bin\curl.exe" -sSf -o rustup-init.exe https://win.rustup.rs/ .\rustup-init.exe -y set PATH=%PATH%;%USERPROFILE%\.cargo\bin + rustup update rustc -V cargo -V displayName: 'Installing Rust' diff --git a/tests/snippets/test_collections.py b/tests/snippets/test_collections.py new file mode 100644 index 000000000..ecc1945cc --- /dev/null +++ b/tests/snippets/test_collections.py @@ -0,0 +1,39 @@ +from collections import deque + + +d = deque([0, 1, 2]) + +d.append(1) +d.appendleft(3) + +assert d == deque([3, 0, 1, 2, 1]) + +assert d <= deque([4]) + +assert d.copy() is not d + +d = deque([1, 2, 3], 5) + +d.extend([4, 5, 6]) + +assert d == deque([2, 3, 4, 5, 6]) + +d.remove(4) + +assert d == deque([2, 3, 5, 6]) + +d.clear() + +assert d == deque() + +assert d == deque([], 4) + +assert deque([1, 2, 3]) * 2 == deque([1, 2, 3, 1, 2, 3]) + +assert deque([1, 2, 3], 4) * 2 == deque([3, 1, 2, 3]) + +assert deque(maxlen=3) == deque() + +assert deque([1, 2, 3, 4], maxlen=2) == deque([3, 4]) + +assert len(deque([1, 2, 3, 4])) == 4 diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 77c1b4c9f..b4d3a3f0f 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -392,12 +392,16 @@ impl PyListRef { } fn mul(self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { - let new_elements = seq_mul(&self.elements.borrow(), counter); + let new_elements = seq_mul(&self.elements.borrow().as_slice(), counter) + .cloned() + .collect(); vm.ctx.new_list(new_elements) } fn imul(self, counter: isize, _vm: &VirtualMachine) -> Self { - let new_elements = seq_mul(&self.elements.borrow(), counter); + let new_elements = seq_mul(&self.elements.borrow().as_slice(), counter) + .cloned() + .collect(); self.elements.replace(new_elements); self } @@ -491,7 +495,7 @@ impl PyListRef { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); let other = get_elements_list(&other); - let res = seq_equal(vm, &zelf, &other)?; + let res = seq_equal(vm, &zelf.as_slice(), &other.as_slice())?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -502,7 +506,7 @@ impl PyListRef { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); let other = get_elements_list(&other); - let res = seq_lt(vm, &zelf, &other)?; + let res = seq_lt(vm, &zelf.as_slice(), &other.as_slice())?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -513,7 +517,7 @@ impl PyListRef { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); let other = get_elements_list(&other); - let res = seq_gt(vm, &zelf, &other)?; + let res = seq_gt(vm, &zelf.as_slice(), &other.as_slice())?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -524,7 +528,7 @@ impl PyListRef { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); let other = get_elements_list(&other); - let res = seq_ge(vm, &zelf, &other)?; + let res = seq_ge(vm, &zelf.as_slice(), &other.as_slice())?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -535,7 +539,7 @@ impl PyListRef { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); let other = get_elements_list(&other); - let res = seq_le(vm, &zelf, &other)?; + let res = seq_le(vm, &zelf.as_slice(), &other.as_slice())?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index d4226d51c..447322c8e 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -216,10 +216,38 @@ pub fn get_item( } } +type DynPyIter<'a> = Box + 'a>; + +#[allow(clippy::len_without_is_empty)] +pub trait SimpleSeq { + fn len(&self) -> usize; + fn iter(&self) -> DynPyIter; +} + +impl SimpleSeq for &[PyObjectRef] { + fn len(&self) -> usize { + (&**self).len() + } + fn iter(&self) -> DynPyIter { + Box::new((&**self).iter()) + } +} + +impl SimpleSeq for std::collections::VecDeque { + fn len(&self) -> usize { + self.len() + } + fn iter(&self) -> DynPyIter { + Box::new(self.iter()) + } +} + +// impl<'a, I> + pub fn seq_equal( vm: &VirtualMachine, - zelf: &[PyObjectRef], - other: &[PyObjectRef], + zelf: &dyn SimpleSeq, + other: &dyn SimpleSeq, ) -> Result { if zelf.len() == other.len() { for (a, b) in Iterator::zip(zelf.iter(), other.iter()) { @@ -239,8 +267,8 @@ pub fn seq_equal( pub fn seq_lt( vm: &VirtualMachine, - zelf: &[PyObjectRef], - other: &[PyObjectRef], + zelf: &dyn SimpleSeq, + other: &dyn SimpleSeq, ) -> Result { if zelf.len() == other.len() { for (a, b) in Iterator::zip(zelf.iter(), other.iter()) { @@ -279,8 +307,8 @@ pub fn seq_lt( pub fn seq_gt( vm: &VirtualMachine, - zelf: &[PyObjectRef], - other: &[PyObjectRef], + zelf: &dyn SimpleSeq, + other: &dyn SimpleSeq, ) -> Result { if zelf.len() == other.len() { for (a, b) in Iterator::zip(zelf.iter(), other.iter()) { @@ -318,30 +346,58 @@ pub fn seq_gt( pub fn seq_ge( vm: &VirtualMachine, - zelf: &[PyObjectRef], - other: &[PyObjectRef], + zelf: &dyn SimpleSeq, + other: &dyn SimpleSeq, ) -> Result { Ok(seq_gt(vm, zelf, other)? || seq_equal(vm, zelf, other)?) } pub fn seq_le( vm: &VirtualMachine, - zelf: &[PyObjectRef], - other: &[PyObjectRef], + zelf: &dyn SimpleSeq, + other: &dyn SimpleSeq, ) -> Result { Ok(seq_lt(vm, zelf, other)? || seq_equal(vm, zelf, other)?) } -pub fn seq_mul(elements: &[PyObjectRef], counter: isize) -> Vec { - let current_len = elements.len(); - let new_len = counter.max(0) as usize * current_len; - let mut new_elements = Vec::with_capacity(new_len); - - for _ in 0..counter { - new_elements.extend(elements.to_owned()); +pub struct SeqMul<'a> { + seq: &'a dyn SimpleSeq, + repetitions: usize, + iter: Option>, +} +impl<'a> Iterator for SeqMul<'a> { + type Item = &'a PyObjectRef; + fn next(&mut self) -> Option { + if self.seq.len() == 0 { + return None; + } + match self.iter.as_mut().and_then(Iterator::next) { + Some(item) => Some(item), + None => { + if self.repetitions == 0 { + None + } else { + self.repetitions -= 1; + self.iter = Some(self.seq.iter()); + self.next() + } + } + } } + fn size_hint(&self) -> (usize, Option) { + let size = self.iter.as_ref().map_or(0, ExactSizeIterator::len) + + (self.repetitions * self.seq.len()); + (size, Some(size)) + } +} +impl ExactSizeIterator for SeqMul<'_> {} - new_elements +pub fn seq_mul(seq: &dyn SimpleSeq, repetitions: isize) -> SeqMul { + SeqMul { + seq, + repetitions: repetitions.max(0) as usize, + iter: None, + } } pub fn get_elements_cell<'a>(obj: &'a PyObjectRef) -> &'a RefCell> { diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index a6e181f3b..e13d3160f 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -53,7 +53,7 @@ impl PyTupleRef { fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { let other = get_elements_tuple(&other); - let res = seq_lt(vm, &self.elements, &other)?; + let res = seq_lt(vm, &self.elements.as_slice(), &other.as_slice())?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -63,7 +63,7 @@ impl PyTupleRef { fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { let other = get_elements_tuple(&other); - let res = seq_gt(vm, &self.elements, &other)?; + let res = seq_gt(vm, &self.elements.as_slice(), &other.as_slice())?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -73,7 +73,7 @@ impl PyTupleRef { fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { let other = get_elements_tuple(&other); - let res = seq_ge(vm, &self.elements, &other)?; + let res = seq_ge(vm, &self.elements.as_slice(), &other.as_slice())?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -83,7 +83,7 @@ impl PyTupleRef { fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { let other = get_elements_tuple(&other); - let res = seq_le(vm, &self.elements, &other)?; + let res = seq_le(vm, &self.elements.as_slice(), &other.as_slice())?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -122,7 +122,7 @@ impl PyTupleRef { fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { let other = get_elements_tuple(&other); - let res = seq_equal(vm, &self.elements, &other)?; + let res = seq_equal(vm, &self.elements.as_slice(), &other.as_slice())?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -164,13 +164,14 @@ impl PyTupleRef { } fn mul(self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { - let new_elements = seq_mul(&self.elements, counter); + let new_elements = seq_mul(&self.elements.as_slice(), counter) + .cloned() + .collect(); vm.ctx.new_tuple(new_elements) } fn rmul(self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { - let new_elements = seq_mul(&self.elements, counter); - vm.ctx.new_tuple(new_elements) + self.mul(counter, vm) } fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/stdlib/collections.rs b/vm/src/stdlib/collections.rs new file mode 100644 index 000000000..f28912af1 --- /dev/null +++ b/vm/src/stdlib/collections.rs @@ -0,0 +1,346 @@ +use crate::function::OptionalArg; +use crate::obj::{objbool, objsequence, objtype::PyClassRef}; +use crate::pyobject::{IdProtocol, PyClassImpl, PyIterable, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::vm::ReprGuard; +use crate::VirtualMachine; +use itertools::Itertools; +use std::cell::{Cell, RefCell}; +use std::collections::VecDeque; + +#[pyclass(name = "deque")] +#[derive(Debug, Clone)] +struct PyDeque { + deque: RefCell>, + maxlen: Cell>, +} + +impl PyValue for PyDeque { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("_collections", "deque") + } +} + +#[derive(FromArgs)] +struct PyDequeOptions { + #[pyarg(positional_or_keyword, default = "None")] + maxlen: Option, +} + +#[pyimpl] +impl PyDeque { + #[pymethod(name = "__new__")] + fn new( + cls: PyClassRef, + iter: OptionalArg, + PyDequeOptions { maxlen }: PyDequeOptions, + vm: &VirtualMachine, + ) -> PyResult> { + let py_deque = PyDeque { + deque: RefCell::default(), + maxlen: maxlen.into(), + }; + if let OptionalArg::Present(iter) = iter { + py_deque.extend(iter, vm)?; + } + py_deque.into_ref_with_type(vm, cls) + } + + #[pymethod] + fn append(&self, obj: PyObjectRef, _vm: &VirtualMachine) { + let mut deque = self.deque.borrow_mut(); + if self.maxlen.get() == Some(deque.len()) { + deque.pop_front(); + } + deque.push_back(obj); + } + + #[pymethod] + fn appendleft(&self, obj: PyObjectRef, _vm: &VirtualMachine) { + let mut deque = self.deque.borrow_mut(); + if self.maxlen.get() == Some(deque.len()) { + deque.pop_back(); + } + deque.push_front(obj); + } + + #[pymethod] + fn clear(&self, _vm: &VirtualMachine) { + self.deque.borrow_mut().clear() + } + + #[pymethod] + fn copy(&self, _vm: &VirtualMachine) -> Self { + self.clone() + } + + #[pymethod] + fn count(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let mut count = 0; + for elem in self.deque.borrow().iter() { + if objbool::boolval(vm, vm._eq(elem.clone(), obj.clone())?)? { + count += 1; + } + } + Ok(count) + } + + #[pymethod] + fn extend(&self, iter: PyIterable, vm: &VirtualMachine) -> PyResult<()> { + // TODO: use length_hint here and for extendleft + for elem in iter.iter(vm)? { + self.append(elem?, vm); + } + Ok(()) + } + + #[pymethod] + fn extendleft(&self, iter: PyIterable, vm: &VirtualMachine) -> PyResult<()> { + for elem in iter.iter(vm)? { + self.appendleft(elem?, vm); + } + Ok(()) + } + + #[pymethod] + fn index( + &self, + obj: PyObjectRef, + start: OptionalArg, + stop: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let deque = self.deque.borrow(); + let start = start.unwrap_or(0); + let stop = stop.unwrap_or_else(|| deque.len()); + for (i, elem) in deque.iter().skip(start).take(stop - start).enumerate() { + if objbool::boolval(vm, vm._eq(elem.clone(), obj.clone())?)? { + return Ok(i); + } + } + Err(vm.new_value_error( + vm.to_repr(&obj) + .map(|repr| format!("{} is not in deque", repr)) + .unwrap_or_else(|_| String::new()), + )) + } + + #[pymethod] + fn insert(&self, idx: i32, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let mut deque = self.deque.borrow_mut(); + + if self.maxlen.get() == Some(deque.len()) { + return Err(vm.new_index_error("deque already at its maximum size".to_string())); + } + + let idx = if idx < 0 { + if -idx as usize > deque.len() { + 0 + } else { + deque.len() - ((-idx) as usize) + } + } else if idx as usize >= deque.len() { + deque.len() - 1 + } else { + idx as usize + }; + + deque.insert(idx, obj); + + Ok(()) + } + + #[pymethod] + fn pop(&self, vm: &VirtualMachine) -> PyResult { + self.deque + .borrow_mut() + .pop_back() + .ok_or_else(|| vm.new_index_error("pop from an empty deque".to_string())) + } + + #[pymethod] + fn popleft(&self, vm: &VirtualMachine) -> PyResult { + self.deque + .borrow_mut() + .pop_front() + .ok_or_else(|| vm.new_index_error("pop from an empty deque".to_string())) + } + + #[pymethod] + fn remove(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let mut deque = self.deque.borrow_mut(); + let mut idx = None; + for (i, elem) in deque.iter().enumerate() { + if objbool::boolval(vm, vm._eq(elem.clone(), obj.clone())?)? { + idx = Some(i); + break; + } + } + idx.map(|idx| deque.remove(idx).unwrap()) + .ok_or_else(|| vm.new_value_error("deque.remove(x): x not in deque".to_string())) + } + + #[pymethod] + fn reverse(&self, _vm: &VirtualMachine) { + self.deque + .replace_with(|deque| deque.iter().cloned().rev().collect()); + } + + #[pymethod] + fn rotate(&self, mid: OptionalArg, _vm: &VirtualMachine) { + let mut deque = self.deque.borrow_mut(); + let mid = mid.unwrap_or(1); + if mid < 0 { + deque.rotate_left(-mid as usize); + } else { + deque.rotate_right(mid as usize); + } + } + + #[pyproperty] + fn maxlen(&self, _vm: &VirtualMachine) -> Option { + self.maxlen.get() + } + #[pyproperty(setter)] + fn set_maxlen(&self, maxlen: Option, vm: &VirtualMachine) -> PyResult { + self.maxlen.set(maxlen); + Ok(vm.get_none()) + } + + #[pymethod(name = "__repr__")] + fn repr(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + let repr = if let Some(_guard) = ReprGuard::enter(zelf.as_object()) { + let elements = zelf + .deque + .borrow() + .iter() + .map(|obj| vm.to_repr(obj)) + .collect::, _>>()?; + let maxlen = zelf + .maxlen + .get() + .map(|maxlen| format!(", maxlen={}", maxlen)) + .unwrap_or_default(); + format!("deque([{}]{})", elements.into_iter().format(", "), maxlen) + } else { + "[...]".to_string() + }; + Ok(repr) + } + + #[pymethod(name = "__eq__")] + fn eq(zelf: PyRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if zelf.as_object().is(&other) { + return Ok(vm.new_bool(true)); + } + + let other = match_class!(other, + other @ Self => other, + _ => return Ok(vm.ctx.not_implemented()), + ); + + let lhs: &VecDeque<_> = &zelf.deque.borrow(); + let rhs: &VecDeque<_> = &other.deque.borrow(); + + let eq = objsequence::seq_equal(vm, lhs, rhs)?; + Ok(vm.new_bool(eq)) + } + + #[pymethod(name = "__lt__")] + fn lt(zelf: PyRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if zelf.as_object().is(&other) { + return Ok(vm.new_bool(true)); + } + + let other = match_class!(other, + other @ Self => other, + _ => return Ok(vm.ctx.not_implemented()), + ); + + let lhs: &VecDeque<_> = &zelf.deque.borrow(); + let rhs: &VecDeque<_> = &other.deque.borrow(); + + let eq = objsequence::seq_lt(vm, lhs, rhs)?; + Ok(vm.new_bool(eq)) + } + + #[pymethod(name = "__gt__")] + fn gt(zelf: PyRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if zelf.as_object().is(&other) { + return Ok(vm.new_bool(true)); + } + + let other = match_class!(other, + other @ Self => other, + _ => return Ok(vm.ctx.not_implemented()), + ); + + let lhs: &VecDeque<_> = &zelf.deque.borrow(); + let rhs: &VecDeque<_> = &other.deque.borrow(); + + let eq = objsequence::seq_gt(vm, lhs, rhs)?; + Ok(vm.new_bool(eq)) + } + + #[pymethod(name = "__le__")] + fn le(zelf: PyRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if zelf.as_object().is(&other) { + return Ok(vm.new_bool(true)); + } + + let other = match_class!(other, + other @ Self => other, + _ => return Ok(vm.ctx.not_implemented()), + ); + + let lhs: &VecDeque<_> = &zelf.deque.borrow(); + let rhs: &VecDeque<_> = &other.deque.borrow(); + + let eq = objsequence::seq_le(vm, lhs, rhs)?; + Ok(vm.new_bool(eq)) + } + + #[pymethod(name = "__ge__")] + fn ge(zelf: PyRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if zelf.as_object().is(&other) { + return Ok(vm.new_bool(true)); + } + + let other = match_class!(other, + other @ Self => other, + _ => return Ok(vm.ctx.not_implemented()), + ); + + let lhs: &VecDeque<_> = &zelf.deque.borrow(); + let rhs: &VecDeque<_> = &other.deque.borrow(); + + let eq = objsequence::seq_ge(vm, lhs, rhs)?; + Ok(vm.new_bool(eq)) + } + + #[pymethod(name = "__mul__")] + fn mul(&self, n: isize, _vm: &VirtualMachine) -> Self { + let deque: &VecDeque<_> = &self.deque.borrow(); + let mul = objsequence::seq_mul(deque, n); + let skipped = if let Some(maxlen) = self.maxlen.get() { + mul.len() - maxlen + } else { + 0 + }; + let deque = mul.skip(skipped).cloned().collect(); + PyDeque { + deque: RefCell::new(deque), + maxlen: self.maxlen.clone(), + } + } + + #[pymethod(name = "__len__")] + fn len(&self, _vm: &VirtualMachine) -> usize { + self.deque.borrow().len() + } +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + py_module!(vm, "_collections", { + "deque" => PyDeque::make_class(&vm.ctx), + }) +} diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 458a8cfb5..09249a375 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -1,6 +1,7 @@ #[cfg(feature = "rustpython-parser")] mod ast; mod binascii; +mod collections; mod dis; mod hashlib; mod imp; @@ -42,7 +43,8 @@ pub fn get_module_inits() -> HashMap { #[allow(unused_mut)] let mut modules = hashmap! { "binascii".to_string() => Box::new(binascii::make_module) as StdlibInitFunc, - "dis".to_string() => Box::new(dis::make_module) as StdlibInitFunc, + "dis".to_string() => Box::new(dis::make_module), + "_collections".to_string() => Box::new(collections::make_module), "hashlib".to_string() => Box::new(hashlib::make_module), "itertools".to_string() => Box::new(itertools::make_module), "json".to_string() => Box::new(json::make_module),