Merge pull request #2 from RustPython/gen-tests

Compile regex patterns for tests with a script

generate_tests.py

@@ -0,0 +1,37 @@
+import os
+from pathlib import Path
+import re
+import sre_constants
+import sre_compile
+import sre_parse
+import json
+
+m = re.search(r"const SRE_MAGIC: usize = (\d+);", open("src/constants.rs").read())
+sre_engine_magic = int(m.group(1))
+del m
+
+assert sre_constants.MAGIC == sre_engine_magic
+
+class CompiledPattern:
+    @classmethod
+    def compile(cls, pattern, flags=0):
+        p = sre_parse.parse(pattern)
+        code = sre_compile._code(p, flags)
+        self = cls()
+        self.pattern = pattern
+        self.code = code
+        self.flags = re.RegexFlag(flags | p.state.flags)
+        return self
+
+for k, v in re.RegexFlag.__members__.items():
+    setattr(CompiledPattern, k, v)
+
+with os.scandir("tests") as d:
+    for f in d:
+        path = Path(f.path)
+        if path.suffix == ".py":
+            pattern = eval(path.read_text(), {"re": CompiledPattern})
+            path.with_suffix(".re").write_text(
+                f"// {pattern.pattern!r}, flags={pattern.flags!r}\n"
+                f"Pattern {{ code: &{json.dumps(pattern.code)}, flags: SreFlag::from_bits_truncate({int(pattern.flags)}) }}"
+            )

src/engine.rs

@@ -153,6 +153,18 @@ pub enum StrDrive<'a> {
     Str(&'a str),
     Bytes(&'a [u8]),
 }
+
+impl<'a> From<&'a str> for StrDrive<'a> {
+    fn from(s: &'a str) -> Self {
+        Self::Str(s)
+    }
+}
+impl<'a> From<&'a [u8]> for StrDrive<'a> {
+    fn from(b: &'a [u8]) -> Self {
+        Self::Bytes(b)
+    }
+}
+
 impl<'a> StrDrive<'a> {
     fn offset(&self, offset: usize, skip: usize) -> usize {
         match *self {

src/lib.rs

@@ -1,7 +1,5 @@
 pub mod constants;
 pub mod engine;
-#[cfg(test)]
-mod tests;
 
 pub const CODESIZE: usize = 4;
 

src/tests.rs

@@ -1,19 +0,0 @@
-use engine::{State, StrDrive};
-
-use super::*;
-
-#[test]
-fn test_2427() {
-    let str_drive = StrDrive::Str("x");
-    // r'(?<!\.)x\b'
-    let code: Vec<u32> = vec![15, 4, 0, 1, 1, 5, 5, 1, 17, 46, 1, 17, 120, 6, 10, 1];
-    let mut state = State::new(
-        str_drive,
-        0,
-        std::usize::MAX,
-        constants::SreFlag::UNICODE,
-        &code,
-    );
-    state = state.pymatch();
-    assert!(state.has_matched == Some(true));
-}

tests/lookbehind.py

@@ -0,0 +1 @@
+re.compile(r'(?<!\.)x\b')

tests/lookbehind.re

@@ -0,0 +1,2 @@
+// '(?<!\\.)x\\b', flags=re.UNICODE
+Pattern { code: &[15, 4, 0, 1, 1, 5, 5, 1, 17, 46, 1, 17, 120, 6, 10, 1], flags: SreFlag::from_bits_truncate(32) }

tests/tests.rs

@@ -0,0 +1,26 @@
+use sre_engine::constants::SreFlag;
+use sre_engine::engine;
+
+struct Pattern {
+    code: &'static [u32],
+    flags: SreFlag,
+}
+
+impl Pattern {
+    fn state<'a>(
+        &self,
+        string: impl Into<engine::StrDrive<'a>>,
+        range: std::ops::Range<usize>,
+    ) -> engine::State<'a> {
+        engine::State::new(string.into(), range.start, range.end, self.flags, self.code)
+    }
+}
+
+#[test]
+fn test_2427() {
+    // r'(?<!\.)x\b'
+    let pattern = include!("lookbehind.re");
+    let mut state = pattern.state("x", 0..usize::MAX);
+    state = state.pymatch();
+    assert!(state.has_matched == Some(true));
+}