Merge pull request #3522 from fanninpm/test-shlex

Update test_shlex.py to CPython 3.9

Lib/shlex.py

@@ -14,7 +14,7 @@ from collections import deque
 
 from io import StringIO
 
-__all__ = ["shlex", "split", "quote"]
+__all__ = ["shlex", "split", "quote", "join"]
 
 class shlex:
     "A lexical analyzer class for simple shell-like syntaxes."
@@ -55,7 +55,7 @@ class shlex:
             punctuation_chars = ''
         elif punctuation_chars is True:
             punctuation_chars = '();<>|&'
-        self.punctuation_chars = punctuation_chars
+        self._punctuation_chars = punctuation_chars
         if punctuation_chars:
             # _pushback_chars is a push back queue used by lookahead logic
             self._pushback_chars = deque()
@@ -65,6 +65,10 @@ class shlex:
             t = self.wordchars.maketrans(dict.fromkeys(punctuation_chars))
             self.wordchars = self.wordchars.translate(t)
 
+    @property
+    def punctuation_chars(self):
+        return self._punctuation_chars
+
     def push_token(self, tok):
         "Push a token onto the stack popped by the get_token method"
         if self.debug >= 1:
@@ -246,7 +250,8 @@ class shlex:
                     escapedstate = 'a'
                     self.state = nextchar
                 elif (nextchar in self.wordchars or nextchar in self.quotes
-                      or self.whitespace_split):
+                      or (self.whitespace_split and
+                          nextchar not in self.punctuation_chars)):
                     self.token += nextchar
                 else:
                     if self.punctuation_chars:
@@ -298,6 +303,11 @@ class shlex:
         return token
 
 def split(s, comments=False, posix=True):
+    """Split the string *s* using shell-like syntax."""
+    if s is None:
+        import warnings
+        warnings.warn("Passing None for 's' to shlex.split() is deprecated.",
+                      DeprecationWarning, stacklevel=2)
     lex = shlex(s, posix=posix)
     lex.whitespace_split = True
     if not comments:
@@ -305,6 +315,11 @@ def split(s, comments=False, posix=True):
     return list(lex)
 
 
+def join(split_command):
+    """Return a shell-escaped string from *split_command*."""
+    return ' '.join(quote(arg) for arg in split_command)
+
+
 _find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search
 
 def quote(s):

Lib/test/test_shlex.py

@@ -3,7 +3,7 @@ import itertools
 import shlex
 import string
 import unittest
-
+from unittest import mock
 
 
 # The original test data set was from shellwords, by Hartmut Goebel.
@@ -162,6 +162,11 @@ class ShlexTest(unittest.TestCase):
             tok = lex.get_token()
         return ret
 
+    @mock.patch('sys.stdin', io.StringIO())
+    def testSplitNoneDeprecation(self):
+        with self.assertWarns(DeprecationWarning):
+            shlex.split(None)
+
     # TODO: RUSTPYTHON
     @unittest.expectedFailure
     def testSplitPosix(self):
@@ -360,8 +365,6 @@ class ShlexTest(unittest.TestCase):
             self.assertEqual(shlex.quote("test%s'name'" % u),
                              "'test%s'\"'\"'name'\"'\"''" % u)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def testJoin(self):
         for split_command, command in [
             (['a ', 'b'], "'a ' b"),