forked from Rust-related/RustPython
1239 lines
37 KiB
Python
Vendored
1239 lines
37 KiB
Python
Vendored
# Note:
|
|
# This *is* now explicitly RPython.
|
|
# Please make sure not to break this.
|
|
|
|
# XXX RUSTPYTHON: this was originally from PyPy and has been updated to run on
|
|
# Python 3. It's currently in the process of being rewritten
|
|
# to a native Rust module in vm/src/stdlib/codecs.rs
|
|
|
|
"""
|
|
|
|
_codecs -- Provides access to the codec registry and the builtin
|
|
codecs.
|
|
|
|
This module should never be imported directly. The standard library
|
|
module "codecs" wraps this builtin module for use within Python.
|
|
|
|
The codec registry is accessible via:
|
|
|
|
register(search_function) -> None
|
|
|
|
lookup(encoding) -> (encoder, decoder, stream_reader, stream_writer)
|
|
|
|
The builtin Unicode codecs use the following interface:
|
|
|
|
<encoding>_encode(Unicode_object[,errors='strict']) ->
|
|
(string object, bytes consumed)
|
|
|
|
<encoding>_decode(char_buffer_obj[,errors='strict']) ->
|
|
(Unicode object, bytes consumed)
|
|
|
|
<encoding>_encode() interfaces also accept non-Unicode object as
|
|
input. The objects are then converted to Unicode using
|
|
PyUnicode_FromObject() prior to applying the conversion.
|
|
|
|
These <encoding>s are available: utf_8, unicode_escape,
|
|
raw_unicode_escape, unicode_internal, latin_1, ascii (7-bit),
|
|
mbcs (on win32).
|
|
|
|
|
|
Written by Marc-Andre Lemburg (mal@lemburg.com).
|
|
|
|
Copyright (c) Corporation for National Research Initiatives.
|
|
|
|
From PyPy v1.0.0
|
|
|
|
"""
|
|
#from unicodecodec import *
|
|
|
|
__all__ = ['register', 'lookup', 'lookup_error', 'register_error', 'encode', 'decode',
|
|
'latin_1_encode', 'mbcs_decode', 'readbuffer_encode', 'escape_encode',
|
|
'utf_8_decode', 'raw_unicode_escape_decode', 'utf_7_decode',
|
|
'unicode_escape_encode', 'latin_1_decode', 'utf_16_decode',
|
|
'unicode_escape_decode', 'ascii_decode', 'charmap_encode', 'charmap_build',
|
|
'unicode_internal_encode', 'unicode_internal_decode', 'utf_16_ex_decode',
|
|
'escape_decode', 'charmap_decode', 'utf_7_encode', 'mbcs_encode',
|
|
'ascii_encode', 'utf_16_encode', 'raw_unicode_escape_encode', 'utf_8_encode',
|
|
'utf_16_le_encode', 'utf_16_be_encode', 'utf_16_le_decode', 'utf_16_be_decode',]
|
|
|
|
import sys
|
|
import warnings
|
|
from _codecs import *
|
|
|
|
|
|
def latin_1_encode( obj, errors='strict'):
|
|
"""None
|
|
"""
|
|
res = PyUnicode_EncodeLatin1(obj, len(obj), errors)
|
|
res = bytes(res)
|
|
return res, len(obj)
|
|
# XXX MBCS codec might involve ctypes ?
|
|
def mbcs_decode():
|
|
"""None
|
|
"""
|
|
pass
|
|
|
|
def readbuffer_encode( obj, errors='strict'):
|
|
"""None
|
|
"""
|
|
if isinstance(obj, str):
|
|
res = obj.encode()
|
|
else:
|
|
res = bytes(obj)
|
|
return res, len(obj)
|
|
|
|
def escape_encode( obj, errors='strict'):
|
|
"""None
|
|
"""
|
|
if not isinstance(obj, bytes):
|
|
raise TypeError("must be bytes")
|
|
s = repr(obj).encode()
|
|
v = s[2:-1]
|
|
if s[1] == ord('"'):
|
|
v = v.replace(b"'", b"\\'").replace(b'\\"', b'"')
|
|
return v, len(obj)
|
|
|
|
def raw_unicode_escape_decode( data, errors='strict', final=False):
|
|
"""None
|
|
"""
|
|
res = PyUnicode_DecodeRawUnicodeEscape(data, len(data), errors, final)
|
|
res = ''.join(res)
|
|
return res, len(data)
|
|
|
|
def utf_7_decode( data, errors='strict'):
|
|
"""None
|
|
"""
|
|
res = PyUnicode_DecodeUTF7(data, len(data), errors)
|
|
res = ''.join(res)
|
|
return res, len(data)
|
|
|
|
def unicode_escape_encode( obj, errors='strict'):
|
|
"""None
|
|
"""
|
|
res = unicodeescape_string(obj, len(obj), 0)
|
|
res = b''.join(res)
|
|
return res, len(obj)
|
|
|
|
def latin_1_decode( data, errors='strict'):
|
|
"""None
|
|
"""
|
|
res = PyUnicode_DecodeLatin1(data, len(data), errors)
|
|
res = ''.join(res)
|
|
return res, len(data)
|
|
|
|
def utf_16_decode( data, errors='strict', final=False):
|
|
"""None
|
|
"""
|
|
consumed = len(data)
|
|
if final:
|
|
consumed = 0
|
|
res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'native', final)
|
|
res = ''.join(res)
|
|
return res, consumed
|
|
|
|
def unicode_escape_decode( data, errors='strict', final=False):
|
|
"""None
|
|
"""
|
|
res = PyUnicode_DecodeUnicodeEscape(data, len(data), errors, final)
|
|
res = ''.join(res)
|
|
return res, len(data)
|
|
|
|
|
|
def ascii_decode( data, errors='strict'):
|
|
"""None
|
|
"""
|
|
res = PyUnicode_DecodeASCII(data, len(data), errors)
|
|
res = ''.join(res)
|
|
return res, len(data)
|
|
|
|
def charmap_encode(obj, errors='strict', mapping='latin-1'):
|
|
"""None
|
|
"""
|
|
|
|
res = PyUnicode_EncodeCharmap(obj, len(obj), mapping, errors)
|
|
res = bytes(res)
|
|
return res, len(obj)
|
|
|
|
def charmap_build(s):
|
|
return {ord(c): i for i, c in enumerate(s)}
|
|
|
|
if sys.maxunicode == 65535:
|
|
unicode_bytes = 2
|
|
else:
|
|
unicode_bytes = 4
|
|
|
|
def unicode_internal_encode( obj, errors='strict'):
|
|
"""None
|
|
"""
|
|
if type(obj) == str:
|
|
p = bytearray()
|
|
t = [ord(x) for x in obj]
|
|
for i in t:
|
|
b = bytearray()
|
|
for j in range(unicode_bytes):
|
|
b.append(i%256)
|
|
i >>= 8
|
|
if sys.byteorder == "big":
|
|
b.reverse()
|
|
p += b
|
|
res = bytes(p)
|
|
return res, len(res)
|
|
else:
|
|
res = "You can do better than this" # XXX make this right
|
|
return res, len(res)
|
|
|
|
def unicode_internal_decode( unistr, errors='strict'):
|
|
"""None
|
|
"""
|
|
if type(unistr) == str:
|
|
return unistr, len(unistr)
|
|
else:
|
|
p = []
|
|
i = 0
|
|
if sys.byteorder == "big":
|
|
start = unicode_bytes - 1
|
|
stop = -1
|
|
step = -1
|
|
else:
|
|
start = 0
|
|
stop = unicode_bytes
|
|
step = 1
|
|
while i < len(unistr)-unicode_bytes+1:
|
|
t = 0
|
|
h = 0
|
|
for j in range(start, stop, step):
|
|
t += ord(unistr[i+j])<<(h*8)
|
|
h += 1
|
|
i += unicode_bytes
|
|
p += chr(t)
|
|
res = ''.join(p)
|
|
return res, len(res)
|
|
|
|
def utf_16_ex_decode( data, errors='strict', byteorder=0, final=0):
|
|
"""None
|
|
"""
|
|
if byteorder == 0:
|
|
bm = 'native'
|
|
elif byteorder == -1:
|
|
bm = 'little'
|
|
else:
|
|
bm = 'big'
|
|
consumed = len(data)
|
|
if final:
|
|
consumed = 0
|
|
res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, bm, final)
|
|
res = ''.join(res)
|
|
return res, consumed, byteorder
|
|
|
|
# XXX needs error messages when the input is invalid
|
|
def escape_decode(data, errors='strict'):
|
|
"""None
|
|
"""
|
|
l = len(data)
|
|
i = 0
|
|
res = bytearray()
|
|
while i < l:
|
|
|
|
if data[i] == '\\':
|
|
i += 1
|
|
if i >= l:
|
|
raise ValueError("Trailing \\ in string")
|
|
else:
|
|
if data[i] == '\\':
|
|
res += b'\\'
|
|
elif data[i] == 'n':
|
|
res += b'\n'
|
|
elif data[i] == 't':
|
|
res += b'\t'
|
|
elif data[i] == 'r':
|
|
res += b'\r'
|
|
elif data[i] == 'b':
|
|
res += b'\b'
|
|
elif data[i] == '\'':
|
|
res += b'\''
|
|
elif data[i] == '\"':
|
|
res += b'\"'
|
|
elif data[i] == 'f':
|
|
res += b'\f'
|
|
elif data[i] == 'a':
|
|
res += b'\a'
|
|
elif data[i] == 'v':
|
|
res += b'\v'
|
|
elif '0' <= data[i] <= '9':
|
|
# emulate a strange wrap-around behavior of CPython:
|
|
# \400 is the same as \000 because 0400 == 256
|
|
octal = data[i:i+3]
|
|
res.append(int(octal, 8) & 0xFF)
|
|
i += 2
|
|
elif data[i] == 'x':
|
|
hexa = data[i+1:i+3]
|
|
res.append(int(hexa, 16))
|
|
i += 2
|
|
else:
|
|
res.append(data[i])
|
|
i += 1
|
|
res = bytes(res)
|
|
return res, len(res)
|
|
|
|
def charmap_decode( data, errors='strict', mapping=None):
|
|
"""None
|
|
"""
|
|
res = PyUnicode_DecodeCharmap(data, len(data), mapping, errors)
|
|
res = ''.join(res)
|
|
return res, len(data)
|
|
|
|
|
|
def utf_7_encode( obj, errors='strict'):
|
|
"""None
|
|
"""
|
|
res = PyUnicode_EncodeUTF7(obj, len(obj), 0, 0, errors)
|
|
res = b''.join(res)
|
|
return res, len(obj)
|
|
|
|
def mbcs_encode( obj, errors='strict'):
|
|
"""None
|
|
"""
|
|
pass
|
|
## return (PyUnicode_EncodeMBCS(
|
|
## (obj),
|
|
## len(obj),
|
|
## errors),
|
|
## len(obj))
|
|
|
|
|
|
def ascii_encode( obj, errors='strict'):
|
|
"""None
|
|
"""
|
|
res = PyUnicode_EncodeASCII(obj, len(obj), errors)
|
|
res = bytes(res)
|
|
return res, len(obj)
|
|
|
|
def utf_16_encode( obj, errors='strict'):
|
|
"""None
|
|
"""
|
|
res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'native')
|
|
res = bytes(res)
|
|
return res, len(obj)
|
|
|
|
def raw_unicode_escape_encode( obj, errors='strict'):
|
|
"""None
|
|
"""
|
|
res = PyUnicode_EncodeRawUnicodeEscape(obj, len(obj))
|
|
res = bytes(res)
|
|
return res, len(obj)
|
|
|
|
def utf_16_le_encode( obj, errors='strict'):
|
|
"""None
|
|
"""
|
|
res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'little')
|
|
res = bytes(res)
|
|
return res, len(obj)
|
|
|
|
def utf_16_be_encode( obj, errors='strict'):
|
|
"""None
|
|
"""
|
|
res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'big')
|
|
res = bytes(res)
|
|
return res, len(obj)
|
|
|
|
def utf_16_le_decode( data, errors='strict', byteorder=0, final = 0):
|
|
"""None
|
|
"""
|
|
consumed = len(data)
|
|
if final:
|
|
consumed = 0
|
|
res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'little', final)
|
|
res = ''.join(res)
|
|
return res, consumed
|
|
|
|
def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0):
|
|
"""None
|
|
"""
|
|
consumed = len(data)
|
|
if final:
|
|
consumed = 0
|
|
res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'big', final)
|
|
res = ''.join(res)
|
|
return res, consumed
|
|
|
|
|
|
|
|
|
|
# ----------------------------------------------------------------------
|
|
|
|
##import sys
|
|
##""" Python implementation of CPythons builtin unicode codecs.
|
|
##
|
|
## Generally the functions in this module take a list of characters an returns
|
|
## a list of characters.
|
|
##
|
|
## For use in the PyPy project"""
|
|
|
|
|
|
## indicate whether a UTF-7 character is special i.e. cannot be directly
|
|
## encoded:
|
|
## 0 - not special
|
|
## 1 - special
|
|
## 2 - whitespace (optional)
|
|
## 3 - RFC2152 Set O (optional)
|
|
|
|
utf7_special = [
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1,
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0,
|
|
3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3,
|
|
3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1,
|
|
]
|
|
unicode_latin1 = [None]*256
|
|
|
|
|
|
def SPECIAL(c, encodeO, encodeWS):
|
|
c = ord(c)
|
|
return (c>127 or utf7_special[c] == 1) or \
|
|
(encodeWS and (utf7_special[(c)] == 2)) or \
|
|
(encodeO and (utf7_special[(c)] == 3))
|
|
def B64(n):
|
|
return bytes([b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(n) & 0x3f]])
|
|
def B64CHAR(c):
|
|
return (c.isalnum() or (c) == b'+' or (c) == b'/')
|
|
def UB64(c):
|
|
if (c) == b'+' :
|
|
return 62
|
|
elif (c) == b'/':
|
|
return 63
|
|
elif (c) >= b'a':
|
|
return ord(c) - 71
|
|
elif (c) >= b'A':
|
|
return ord(c) - 65
|
|
else:
|
|
return ord(c) + 4
|
|
|
|
def ENCODE( ch, bits) :
|
|
out = []
|
|
while (bits >= 6):
|
|
out += B64(ch >> (bits-6))
|
|
bits -= 6
|
|
return out, bits
|
|
|
|
def PyUnicode_DecodeUTF7(s, size, errors):
|
|
|
|
starts = s
|
|
errmsg = ""
|
|
inShift = 0
|
|
bitsleft = 0
|
|
charsleft = 0
|
|
surrogate = 0
|
|
p = []
|
|
errorHandler = None
|
|
exc = None
|
|
|
|
if (size == 0):
|
|
return ''
|
|
i = 0
|
|
while i < size:
|
|
|
|
ch = bytes([s[i]])
|
|
if (inShift):
|
|
if ((ch == b'-') or not B64CHAR(ch)):
|
|
inShift = 0
|
|
i += 1
|
|
|
|
while (bitsleft >= 16):
|
|
outCh = ((charsleft) >> (bitsleft-16)) & 0xffff
|
|
bitsleft -= 16
|
|
|
|
if (surrogate):
|
|
## We have already generated an error for the high surrogate
|
|
## so let's not bother seeing if the low surrogate is correct or not
|
|
surrogate = 0
|
|
elif (0xDC00 <= (outCh) and (outCh) <= 0xDFFF):
|
|
## This is a surrogate pair. Unfortunately we can't represent
|
|
## it in a 16-bit character
|
|
surrogate = 1
|
|
msg = "code pairs are not supported"
|
|
out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, i-1, i)
|
|
p.append(out)
|
|
bitsleft = 0
|
|
break
|
|
else:
|
|
p.append(chr(outCh ))
|
|
#p += out
|
|
if (bitsleft >= 6):
|
|
## /* The shift sequence has a partial character in it. If
|
|
## bitsleft < 6 then we could just classify it as padding
|
|
## but that is not the case here */
|
|
msg = "partial character in shift sequence"
|
|
out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, i-1, i)
|
|
|
|
## /* According to RFC2152 the remaining bits should be zero. We
|
|
## choose to signal an error/insert a replacement character
|
|
## here so indicate the potential of a misencoded character. */
|
|
|
|
## /* On x86, a << b == a << (b%32) so make sure that bitsleft != 0 */
|
|
## if (bitsleft and (charsleft << (sizeof(charsleft) * 8 - bitsleft))):
|
|
## raise UnicodeDecodeError, "non-zero padding bits in shift sequence"
|
|
if (ch == b'-') :
|
|
if ((i < size) and (s[i] == '-')) :
|
|
p += '-'
|
|
inShift = 1
|
|
|
|
elif SPECIAL(ch, 0, 0) :
|
|
raise UnicodeDecodeError("unexpected special character")
|
|
|
|
else:
|
|
p.append(chr(ord(ch)))
|
|
else:
|
|
charsleft = (charsleft << 6) | UB64(ch)
|
|
bitsleft += 6
|
|
i += 1
|
|
## /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);
|
|
elif ( ch == b'+' ):
|
|
startinpos = i
|
|
i += 1
|
|
if (i<size and s[i] == '-'):
|
|
i += 1
|
|
p.append(b'+')
|
|
else:
|
|
inShift = 1
|
|
bitsleft = 0
|
|
|
|
elif (SPECIAL(ch, 0, 0)):
|
|
i += 1
|
|
raise UnicodeDecodeError("unexpected special character")
|
|
else:
|
|
p.append(chr(ord(ch)))
|
|
i += 1
|
|
|
|
if (inShift) :
|
|
#XXX This aint right
|
|
endinpos = size
|
|
raise UnicodeDecodeError("unterminated shift sequence")
|
|
|
|
return p
|
|
|
|
def PyUnicode_EncodeUTF7(s, size, encodeSetO, encodeWhiteSpace, errors):
|
|
|
|
# /* It might be possible to tighten this worst case */
|
|
inShift = False
|
|
i = 0
|
|
bitsleft = 0
|
|
charsleft = 0
|
|
out = []
|
|
for ch in s:
|
|
if (not inShift) :
|
|
if (ch == '+'):
|
|
out.append(b'+-')
|
|
elif (SPECIAL(ch, encodeSetO, encodeWhiteSpace)):
|
|
charsleft = ord(ch)
|
|
bitsleft = 16
|
|
out.append(b'+')
|
|
p, bitsleft = ENCODE( charsleft, bitsleft)
|
|
out.append(p)
|
|
inShift = bitsleft > 0
|
|
else:
|
|
out.append(bytes([ord(ch)]))
|
|
else:
|
|
if (not SPECIAL(ch, encodeSetO, encodeWhiteSpace)):
|
|
out.append(B64((charsleft) << (6-bitsleft)))
|
|
charsleft = 0
|
|
bitsleft = 0
|
|
## /* Characters not in the BASE64 set implicitly unshift the sequence
|
|
## so no '-' is required, except if the character is itself a '-' */
|
|
if (B64CHAR(ch) or ch == '-'):
|
|
out.append(b'-')
|
|
inShift = False
|
|
out.append(bytes([ord(ch)]))
|
|
else:
|
|
bitsleft += 16
|
|
charsleft = (((charsleft) << 16) | ord(ch))
|
|
p, bitsleft = ENCODE(charsleft, bitsleft)
|
|
out.append(p)
|
|
## /* If the next character is special then we dont' need to terminate
|
|
## the shift sequence. If the next character is not a BASE64 character
|
|
## or '-' then the shift sequence will be terminated implicitly and we
|
|
## don't have to insert a '-'. */
|
|
|
|
if (bitsleft == 0):
|
|
if (i + 1 < size):
|
|
ch2 = s[i+1]
|
|
|
|
if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)):
|
|
pass
|
|
elif (B64CHAR(ch2) or ch2 == '-'):
|
|
out.append(b'-')
|
|
inShift = False
|
|
else:
|
|
inShift = False
|
|
else:
|
|
out.append(b'-')
|
|
inShift = False
|
|
i += 1
|
|
|
|
if (bitsleft):
|
|
out.append(B64(charsleft << (6-bitsleft) ) )
|
|
out.append(b'-')
|
|
|
|
return out
|
|
|
|
unicode_empty = ''
|
|
|
|
def unicodeescape_string(s, size, quotes):
|
|
|
|
p = []
|
|
if (quotes) :
|
|
if (s.find('\'') != -1 and s.find('"') == -1):
|
|
p.append(b'"')
|
|
else:
|
|
p.append(b'\'')
|
|
pos = 0
|
|
while (pos < size):
|
|
ch = s[pos]
|
|
#/* Escape quotes */
|
|
if (quotes and (ch == p[1] or ch == '\\')):
|
|
p.append(b'\\%c' % ord(ch))
|
|
pos += 1
|
|
continue
|
|
|
|
#ifdef Py_UNICODE_WIDE
|
|
#/* Map 21-bit characters to '\U00xxxxxx' */
|
|
elif (ord(ch) >= 0x10000):
|
|
p.append(b'\\U%08x' % ord(ch))
|
|
pos += 1
|
|
continue
|
|
#endif
|
|
#/* Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes */
|
|
elif (ord(ch) >= 0xD800 and ord(ch) < 0xDC00):
|
|
pos += 1
|
|
ch2 = s[pos]
|
|
|
|
if (ord(ch2) >= 0xDC00 and ord(ch2) <= 0xDFFF):
|
|
ucs = (((ord(ch) & 0x03FF) << 10) | (ord(ch2) & 0x03FF)) + 0x00010000
|
|
p.append(b'\\U%08x' % ucs)
|
|
pos += 1
|
|
continue
|
|
|
|
#/* Fall through: isolated surrogates are copied as-is */
|
|
pos -= 1
|
|
|
|
#/* Map 16-bit characters to '\uxxxx' */
|
|
if (ord(ch) >= 256):
|
|
p.append(b'\\u%04x' % ord(ch))
|
|
|
|
#/* Map special whitespace to '\t', \n', '\r' */
|
|
elif (ch == '\t'):
|
|
p.append(b'\\t')
|
|
|
|
elif (ch == '\n'):
|
|
p.append(b'\\n')
|
|
|
|
elif (ch == '\r'):
|
|
p.append(b'\\r')
|
|
|
|
elif (ch == '\\'):
|
|
p.append(b'\\\\')
|
|
|
|
#/* Map non-printable US ASCII to '\xhh' */
|
|
elif (ch < ' ' or ch >= chr(0x7F)) :
|
|
p.append(b'\\x%02x' % ord(ch))
|
|
#/* Copy everything else as-is */
|
|
else:
|
|
p.append(bytes([ord(ch)]))
|
|
pos += 1
|
|
if (quotes):
|
|
p.append(p[0])
|
|
return p
|
|
|
|
def PyUnicode_DecodeASCII(s, size, errors):
|
|
|
|
# /* ASCII is equivalent to the first 128 ordinals in Unicode. */
|
|
if (size == 1 and ord(s) < 128) :
|
|
return [chr(ord(s))]
|
|
if (size == 0):
|
|
return [''] #unicode('')
|
|
p = []
|
|
pos = 0
|
|
while pos < len(s):
|
|
c = s[pos]
|
|
if c < 128:
|
|
p += chr(c)
|
|
pos += 1
|
|
else:
|
|
|
|
res = unicode_call_errorhandler(
|
|
errors, "ascii", "ordinal not in range(128)",
|
|
s, pos, pos+1)
|
|
p += res[0]
|
|
pos = res[1]
|
|
return p
|
|
|
|
def PyUnicode_EncodeASCII(p, size, errors):
|
|
|
|
return unicode_encode_ucs1(p, size, errors, 128)
|
|
|
|
def PyUnicode_AsASCIIString(unistr):
|
|
|
|
if not type(unistr) == str:
|
|
raise TypeError
|
|
return PyUnicode_EncodeASCII(str(unistr),
|
|
len(str),
|
|
None)
|
|
|
|
def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=True):
|
|
|
|
bo = 0 #/* assume native ordering by default */
|
|
consumed = 0
|
|
errmsg = ""
|
|
|
|
if sys.byteorder == 'little':
|
|
ihi = 1
|
|
ilo = 0
|
|
else:
|
|
ihi = 0
|
|
ilo = 1
|
|
|
|
|
|
#/* Unpack UTF-16 encoded data */
|
|
|
|
## /* Check for BOM marks (U+FEFF) in the input and adjust current
|
|
## byte order setting accordingly. In native mode, the leading BOM
|
|
## mark is skipped, in all other modes, it is copied to the output
|
|
## stream as-is (giving a ZWNBSP character). */
|
|
q = 0
|
|
p = []
|
|
if byteorder == 'native':
|
|
if (size >= 2):
|
|
bom = (s[ihi] << 8) | s[ilo]
|
|
#ifdef BYTEORDER_IS_LITTLE_ENDIAN
|
|
if sys.byteorder == 'little':
|
|
if (bom == 0xFEFF):
|
|
q += 2
|
|
bo = -1
|
|
elif bom == 0xFFFE:
|
|
q += 2
|
|
bo = 1
|
|
else:
|
|
if bom == 0xFEFF:
|
|
q += 2
|
|
bo = 1
|
|
elif bom == 0xFFFE:
|
|
q += 2
|
|
bo = -1
|
|
elif byteorder == 'little':
|
|
bo = -1
|
|
else:
|
|
bo = 1
|
|
|
|
if (size == 0):
|
|
return [''], 0, bo
|
|
|
|
if (bo == -1):
|
|
#/* force LE */
|
|
ihi = 1
|
|
ilo = 0
|
|
|
|
elif (bo == 1):
|
|
#/* force BE */
|
|
ihi = 0
|
|
ilo = 1
|
|
|
|
while (q < len(s)):
|
|
|
|
#/* remaining bytes at the end? (size should be even) */
|
|
if (len(s)-q<2):
|
|
if not final:
|
|
break
|
|
errmsg = "truncated data"
|
|
startinpos = q
|
|
endinpos = len(s)
|
|
unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True)
|
|
# /* The remaining input chars are ignored if the callback
|
|
## chooses to skip the input */
|
|
|
|
ch = (s[q+ihi] << 8) | s[q+ilo]
|
|
q += 2
|
|
|
|
if (ch < 0xD800 or ch > 0xDFFF):
|
|
p.append(chr(ch))
|
|
continue
|
|
|
|
#/* UTF-16 code pair: */
|
|
if (q >= len(s)):
|
|
errmsg = "unexpected end of data"
|
|
startinpos = q-2
|
|
endinpos = len(s)
|
|
unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True)
|
|
|
|
if (0xD800 <= ch and ch <= 0xDBFF):
|
|
ch2 = (s[q+ihi] << 8) | s[q+ilo]
|
|
q += 2
|
|
if (0xDC00 <= ch2 and ch2 <= 0xDFFF):
|
|
#ifndef Py_UNICODE_WIDE
|
|
if sys.maxunicode < 65536:
|
|
p += [chr(ch), chr(ch2)]
|
|
else:
|
|
p.append(chr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000))
|
|
#endif
|
|
continue
|
|
|
|
else:
|
|
errmsg = "illegal UTF-16 surrogate"
|
|
startinpos = q-4
|
|
endinpos = startinpos+2
|
|
unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True)
|
|
|
|
errmsg = "illegal encoding"
|
|
startinpos = q-2
|
|
endinpos = startinpos+2
|
|
unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True)
|
|
|
|
return p, q, bo
|
|
|
|
# moved out of local scope, especially because it didn't
|
|
# have any nested variables.
|
|
|
|
def STORECHAR(CH, byteorder):
|
|
hi = (CH >> 8) & 0xff
|
|
lo = CH & 0xff
|
|
if byteorder == 'little':
|
|
return [lo, hi]
|
|
else:
|
|
return [hi, lo]
|
|
|
|
def PyUnicode_EncodeUTF16(s, size, errors, byteorder='little'):
|
|
|
|
# /* Offsets from p for storing byte pairs in the right order. */
|
|
|
|
|
|
p = []
|
|
bom = sys.byteorder
|
|
if (byteorder == 'native'):
|
|
|
|
bom = sys.byteorder
|
|
p += STORECHAR(0xFEFF, bom)
|
|
|
|
if (size == 0):
|
|
return ""
|
|
|
|
if (byteorder == 'little' ):
|
|
bom = 'little'
|
|
elif (byteorder == 'big'):
|
|
bom = 'big'
|
|
|
|
|
|
for c in s:
|
|
ch = ord(c)
|
|
ch2 = 0
|
|
if (ch >= 0x10000) :
|
|
ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF)
|
|
ch = 0xD800 | ((ch-0x10000) >> 10)
|
|
|
|
p += STORECHAR(ch, bom)
|
|
if (ch2):
|
|
p += STORECHAR(ch2, bom)
|
|
|
|
return p
|
|
|
|
|
|
def PyUnicode_DecodeMBCS(s, size, errors):
|
|
pass
|
|
|
|
def PyUnicode_EncodeMBCS(p, size, errors):
|
|
pass
|
|
|
|
def unicode_call_errorhandler(errors, encoding,
|
|
reason, input, startinpos, endinpos, decode=True):
|
|
|
|
errorHandler = lookup_error(errors)
|
|
if decode:
|
|
exceptionObject = UnicodeDecodeError(encoding, input, startinpos, endinpos, reason)
|
|
else:
|
|
exceptionObject = UnicodeEncodeError(encoding, input, startinpos, endinpos, reason)
|
|
res = errorHandler(exceptionObject)
|
|
if isinstance(res, tuple) and isinstance(res[0], str) and isinstance(res[1], int):
|
|
newpos = res[1]
|
|
if (newpos < 0):
|
|
newpos = len(input) + newpos
|
|
if newpos < 0 or newpos > len(input):
|
|
raise IndexError( "position %d from error handler out of bounds" % newpos)
|
|
return res[0], newpos
|
|
else:
|
|
raise TypeError("encoding error handler must return (unicode, int) tuple, not %s" % repr(res))
|
|
|
|
#/* --- Latin-1 Codec ------------------------------------------------------ */
|
|
|
|
def PyUnicode_DecodeLatin1(s, size, errors):
|
|
#/* Latin-1 is equivalent to the first 256 ordinals in Unicode. */
|
|
## if (size == 1):
|
|
## return [PyUnicode_FromUnicode(s, 1)]
|
|
pos = 0
|
|
p = []
|
|
while (pos < size):
|
|
p += chr(s[pos])
|
|
pos += 1
|
|
return p
|
|
|
|
def unicode_encode_ucs1(p, size, errors, limit):
|
|
|
|
if limit == 256:
|
|
reason = "ordinal not in range(256)"
|
|
encoding = "latin-1"
|
|
else:
|
|
reason = "ordinal not in range(128)"
|
|
encoding = "ascii"
|
|
|
|
if (size == 0):
|
|
return []
|
|
res = bytearray()
|
|
pos = 0
|
|
while pos < len(p):
|
|
#for ch in p:
|
|
ch = p[pos]
|
|
|
|
if ord(ch) < limit:
|
|
res.append(ord(ch))
|
|
pos += 1
|
|
else:
|
|
#/* startpos for collecting unencodable chars */
|
|
collstart = pos
|
|
collend = pos+1
|
|
while collend < len(p) and ord(p[collend]) >= limit:
|
|
collend += 1
|
|
x = unicode_call_errorhandler(errors, encoding, reason, p, collstart, collend, False)
|
|
res += x[0].encode()
|
|
pos = x[1]
|
|
|
|
return res
|
|
|
|
def PyUnicode_EncodeLatin1(p, size, errors):
|
|
res = unicode_encode_ucs1(p, size, errors, 256)
|
|
return res
|
|
|
|
hexdigits = [ord(hex(i)[-1]) for i in range(16)]+[ord(hex(i)[-1].upper()) for i in range(10, 16)]
|
|
|
|
def hex_number_end(s, pos, digits):
|
|
target_end = pos + digits
|
|
while pos < target_end and pos < len(s) and s[pos] in hexdigits:
|
|
pos += 1
|
|
return pos
|
|
|
|
def hexescape(s, pos, digits, message, errors):
|
|
ch = 0
|
|
p = []
|
|
number_end = hex_number_end(s, pos, digits)
|
|
if number_end - pos != digits:
|
|
x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-2, number_end)
|
|
p.append(x[0])
|
|
pos = x[1]
|
|
else:
|
|
ch = int(s[pos:pos+digits], 16)
|
|
#/* when we get here, ch is a 32-bit unicode character */
|
|
if ch <= sys.maxunicode:
|
|
p.append(chr(ch))
|
|
pos += digits
|
|
|
|
elif (ch <= 0x10ffff):
|
|
ch -= 0x10000
|
|
p.append(chr(0xD800 + (ch >> 10)))
|
|
p.append(chr(0xDC00 + (ch & 0x03FF)))
|
|
pos += digits
|
|
else:
|
|
message = "illegal Unicode character"
|
|
x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-2,
|
|
pos+digits)
|
|
p.append(x[0])
|
|
pos = x[1]
|
|
res = p
|
|
return res, pos
|
|
|
|
def PyUnicode_DecodeUnicodeEscape(s, size, errors, final):
|
|
|
|
if (size == 0):
|
|
return ''
|
|
|
|
if isinstance(s, str):
|
|
s = s.encode()
|
|
|
|
found_invalid_escape = False
|
|
|
|
p = []
|
|
pos = 0
|
|
while (pos < size):
|
|
## /* Non-escape characters are interpreted as Unicode ordinals */
|
|
if (chr(s[pos]) != '\\') :
|
|
p.append(chr(s[pos]))
|
|
pos += 1
|
|
continue
|
|
## /* \ - Escapes */
|
|
else:
|
|
pos += 1
|
|
if pos >= len(s):
|
|
errmessage = "\\ at end of string"
|
|
unicode_call_errorhandler(errors, "unicodeescape", errmessage, s, pos-1, size)
|
|
ch = chr(s[pos])
|
|
pos += 1
|
|
## /* \x escapes */
|
|
if ch == '\n': pass
|
|
elif ch == '\\': p += '\\'
|
|
elif ch == '\'': p += '\''
|
|
elif ch == '\"': p += '\"'
|
|
elif ch == 'b' : p += '\b'
|
|
elif ch == 'f' : p += '\014' #/* FF */
|
|
elif ch == 't' : p += '\t'
|
|
elif ch == 'n' : p += '\n'
|
|
elif ch == 'r' : p += '\r'
|
|
elif ch == 'v' : p += '\013' #break; /* VT */
|
|
elif ch == 'a' : p += '\007' # break; /* BEL, not classic C */
|
|
elif '0' <= ch <= '7':
|
|
x = ord(ch) - ord('0')
|
|
if pos < size:
|
|
ch = chr(s[pos])
|
|
if '0' <= ch <= '7':
|
|
pos += 1
|
|
x = (x<<3) + ord(ch) - ord('0')
|
|
if pos < size:
|
|
ch = chr(s[pos])
|
|
if '0' <= ch <= '7':
|
|
pos += 1
|
|
x = (x<<3) + ord(ch) - ord('0')
|
|
p.append(chr(x))
|
|
## /* hex escapes */
|
|
## /* \xXX */
|
|
elif ch == 'x':
|
|
digits = 2
|
|
message = "truncated \\xXX escape"
|
|
x = hexescape(s, pos, digits, message, errors)
|
|
p += x[0]
|
|
pos = x[1]
|
|
|
|
# /* \uXXXX */
|
|
elif ch == 'u':
|
|
digits = 4
|
|
message = "truncated \\uXXXX escape"
|
|
x = hexescape(s, pos, digits, message, errors)
|
|
p += x[0]
|
|
pos = x[1]
|
|
|
|
# /* \UXXXXXXXX */
|
|
elif ch == 'U':
|
|
digits = 8
|
|
message = "truncated \\UXXXXXXXX escape"
|
|
x = hexescape(s, pos, digits, message, errors)
|
|
p += x[0]
|
|
pos = x[1]
|
|
## /* \N{name} */
|
|
elif ch == 'N':
|
|
message = "malformed \\N character escape"
|
|
# pos += 1
|
|
look = pos
|
|
try:
|
|
import unicodedata
|
|
except ImportError:
|
|
message = "\\N escapes not supported (can't load unicodedata module)"
|
|
unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, size)
|
|
if look < size and chr(s[look]) == '{':
|
|
#/* look for the closing brace */
|
|
while (look < size and chr(s[look]) != '}'):
|
|
look += 1
|
|
if (look > pos+1 and look < size and chr(s[look]) == '}'):
|
|
#/* found a name. look it up in the unicode database */
|
|
message = "unknown Unicode character name"
|
|
st = s[pos+1:look]
|
|
try:
|
|
chr_codec = unicodedata.lookup("%s" % st)
|
|
except LookupError as e:
|
|
x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1)
|
|
else:
|
|
x = chr_codec, look + 1
|
|
p.append(x[0])
|
|
pos = x[1]
|
|
else:
|
|
x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1)
|
|
else:
|
|
x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1)
|
|
else:
|
|
if not found_invalid_escape:
|
|
found_invalid_escape = True
|
|
warnings.warn("invalid escape sequence '\\%c'" % ch, DeprecationWarning, 2)
|
|
p.append('\\')
|
|
p.append(ch)
|
|
return p
|
|
|
|
def PyUnicode_EncodeRawUnicodeEscape(s, size):
|
|
|
|
if (size == 0):
|
|
return b''
|
|
|
|
p = bytearray()
|
|
for ch in s:
|
|
# /* Map 32-bit characters to '\Uxxxxxxxx' */
|
|
if (ord(ch) >= 0x10000):
|
|
p += b'\\U%08x' % ord(ch)
|
|
elif (ord(ch) >= 256) :
|
|
# /* Map 16-bit characters to '\uxxxx' */
|
|
p += b'\\u%04x' % (ord(ch))
|
|
# /* Copy everything else as-is */
|
|
else:
|
|
p.append(ord(ch))
|
|
|
|
#p += '\0'
|
|
return p
|
|
|
|
def charmapencode_output(c, mapping):
|
|
|
|
rep = mapping[c]
|
|
if isinstance(rep, int) or isinstance(rep, int):
|
|
if rep < 256:
|
|
return rep
|
|
else:
|
|
raise TypeError("character mapping must be in range(256)")
|
|
elif isinstance(rep, str):
|
|
return ord(rep)
|
|
elif rep == None:
|
|
raise KeyError("character maps to <undefined>")
|
|
else:
|
|
raise TypeError("character mapping must return integer, None or str")
|
|
|
|
def PyUnicode_EncodeCharmap(p, size, mapping='latin-1', errors='strict'):
|
|
|
|
## /* the following variable is used for caching string comparisons
|
|
## * -1=not initialized, 0=unknown, 1=strict, 2=replace,
|
|
## * 3=ignore, 4=xmlcharrefreplace */
|
|
|
|
# /* Default to Latin-1 */
|
|
if mapping == 'latin-1':
|
|
return PyUnicode_EncodeLatin1(p, size, errors)
|
|
if (size == 0):
|
|
return b''
|
|
inpos = 0
|
|
res = []
|
|
while (inpos<size):
|
|
#/* try to encode it */
|
|
try:
|
|
x = charmapencode_output(ord(p[inpos]), mapping)
|
|
res += [x]
|
|
except KeyError:
|
|
x = unicode_call_errorhandler(errors, "charmap",
|
|
"character maps to <undefined>", p, inpos, inpos+1, False)
|
|
try:
|
|
res += [charmapencode_output(ord(y), mapping) for y in x[0]]
|
|
except KeyError:
|
|
raise UnicodeEncodeError("charmap", p, inpos, inpos+1,
|
|
"character maps to <undefined>")
|
|
inpos += 1
|
|
return res
|
|
|
|
def PyUnicode_DecodeCharmap(s, size, mapping, errors):
|
|
|
|
## /* Default to Latin-1 */
|
|
if (mapping == None):
|
|
return PyUnicode_DecodeLatin1(s, size, errors)
|
|
|
|
if (size == 0):
|
|
return ''
|
|
p = []
|
|
inpos = 0
|
|
while (inpos< len(s)):
|
|
|
|
#/* Get mapping (char ordinal -> integer, Unicode char or None) */
|
|
ch = s[inpos]
|
|
try:
|
|
x = mapping[ch]
|
|
if isinstance(x, int):
|
|
if x < 65536:
|
|
p += chr(x)
|
|
else:
|
|
raise TypeError("character mapping must be in range(65536)")
|
|
elif isinstance(x, str):
|
|
p += x
|
|
elif not x:
|
|
raise KeyError
|
|
else:
|
|
raise TypeError
|
|
except KeyError:
|
|
x = unicode_call_errorhandler(errors, "charmap",
|
|
"character maps to <undefined>", s, inpos, inpos+1)
|
|
p += x[0]
|
|
inpos += 1
|
|
return p
|
|
|
|
def PyUnicode_DecodeRawUnicodeEscape(s, size, errors, final):
|
|
|
|
if (size == 0):
|
|
return ''
|
|
|
|
if isinstance(s, str):
|
|
s = s.encode()
|
|
|
|
pos = 0
|
|
p = []
|
|
while (pos < len(s)):
|
|
ch = chr(s[pos])
|
|
#/* Non-escape characters are interpreted as Unicode ordinals */
|
|
if (ch != '\\'):
|
|
p.append(ch)
|
|
pos += 1
|
|
continue
|
|
startinpos = pos
|
|
## /* \u-escapes are only interpreted iff the number of leading
|
|
## backslashes is odd */
|
|
bs = pos
|
|
while pos < size:
|
|
if (s[pos] != ord('\\')):
|
|
break
|
|
p.append(chr(s[pos]))
|
|
pos += 1
|
|
|
|
if (pos >= size):
|
|
break
|
|
if (((pos - bs) & 1) == 0 or
|
|
(s[pos] != ord('u') and s[pos] != ord('U'))) :
|
|
p.append(chr(s[pos]))
|
|
pos += 1
|
|
continue
|
|
|
|
p.pop(-1)
|
|
if s[pos] == ord('u'):
|
|
count = 4
|
|
else:
|
|
count = 8
|
|
pos += 1
|
|
|
|
#/* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */
|
|
number_end = hex_number_end(s, pos, count)
|
|
if number_end - pos != count:
|
|
res = unicode_call_errorhandler(
|
|
errors, "rawunicodeescape", "truncated \\uXXXX",
|
|
s, pos-2, number_end)
|
|
p.append(res[0])
|
|
pos = res[1]
|
|
else:
|
|
x = int(s[pos:pos+count], 16)
|
|
#ifndef Py_UNICODE_WIDE
|
|
if sys.maxunicode > 0xffff:
|
|
if (x > sys.maxunicode):
|
|
res = unicode_call_errorhandler(
|
|
errors, "rawunicodeescape", "\\Uxxxxxxxx out of range",
|
|
s, pos-2, pos+count)
|
|
pos = res[1]
|
|
p.append(res[0])
|
|
else:
|
|
p.append(chr(x))
|
|
pos += count
|
|
else:
|
|
if (x > 0x10000):
|
|
res = unicode_call_errorhandler(
|
|
errors, "rawunicodeescape", "\\Uxxxxxxxx out of range",
|
|
s, pos-2, pos+count)
|
|
pos = res[1]
|
|
p.append(res[0])
|
|
|
|
#endif
|
|
else:
|
|
p.append(chr(x))
|
|
pos += count
|
|
|
|
return p
|