fix: non-utf8 source causes panic, close #3798 (#3884)

Co-authored-by: fanninpm <fanninpm@miamioh.edu>

extra_tests/snippets/syntax_non_utf8.py

@@ -0,0 +1,12 @@
+import os
+import platform
+
+from testutils import assert_raises
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+# TODO: RUSTPYTHON, RustPython raises a SyntaxError here, but cpython raise a ValueError
+error = SyntaxError if platform.python_implementation() == 'RustPython' else ValueError
+with assert_raises(error):
+    with open(os.path.join(dir_path , "non_utf8.txt")) as f:
+        eval(f.read())

parser/src/lexer.rs

@@ -145,6 +145,9 @@ where
     }
 }
 
+/// unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
+const MAX_UNICODE_NAME: usize = 88;
+
 impl<T> Lexer<T>
 where
     T: Iterator<Item = char>,
@@ -466,6 +469,14 @@ where
                 }
             }
         }
+
+        if name.len() > MAX_UNICODE_NAME {
+            return Err(LexicalError {
+                error: LexicalErrorType::UnicodeError,
+                location: self.get_pos(),
+            });
+        }
+
         unicode_names2::character(&name).ok_or(LexicalError {
             error: LexicalErrorType::UnicodeError,
             location: start_pos,