|
| 1 | +""" |
| 2 | +Partial backport of new functionality in Python 3.5's os module: |
| 3 | +
|
| 4 | + fsencode (new in Python 3.2) |
| 5 | + fsdecode (new in Python 3.2) |
| 6 | +
|
| 7 | +Backport modifications are marked with "XXX backport" and "TODO backport". |
| 8 | +""" |
| 9 | +from __future__ import unicode_literals |
| 10 | + |
| 11 | +import sys |
| 12 | + |
| 13 | +# XXX backport: unicode on Python 2 |
| 14 | +_str = unicode if sys.version_info < (3,) else str |
| 15 | + |
| 16 | +# XXX backport: Use backported surrogateescape for Python 2 |
| 17 | +# TODO backport: Find a way to do this without pulling in the entire future package? |
| 18 | +if sys.version_info < (3,): |
| 19 | + from future.utils.surrogateescape import register_surrogateescape |
| 20 | + register_surrogateescape() |
| 21 | + |
| 22 | + |
| 23 | +# XXX backport: This invalid_utf8_indexes() helper is shamelessly copied from |
| 24 | +# Bob Ippolito's pyutf8 package (pyutf8/ref.py), in order to help support the |
| 25 | +# Python 2 UTF-8 decoding hack in fsdecode() below. |
| 26 | +# |
| 27 | +# URL: https://github.com/etrepum/pyutf8/blob/master/pyutf8/ref.py |
| 28 | +# |
| 29 | +def _invalid_utf8_indexes(bytes): |
| 30 | + skips = [] |
| 31 | + i = 0 |
| 32 | + len_bytes = len(bytes) |
| 33 | + while i < len_bytes: |
| 34 | + c1 = bytes[i] |
| 35 | + if c1 < 0x80: |
| 36 | + # U+0000 - U+007F - 7 bits |
| 37 | + i += 1 |
| 38 | + continue |
| 39 | + try: |
| 40 | + c2 = bytes[i + 1] |
| 41 | + if ((c1 & 0xE0 == 0xC0) and (c2 & 0xC0 == 0x80)): |
| 42 | + # U+0080 - U+07FF - 11 bits |
| 43 | + c = (((c1 & 0x1F) << 6) | |
| 44 | + (c2 & 0x3F)) |
| 45 | + if c < 0x80: |
| 46 | + # Overlong encoding |
| 47 | + skips.extend([i, i + 1]) |
| 48 | + i += 2 |
| 49 | + continue |
| 50 | + c3 = bytes[i + 2] |
| 51 | + if ((c1 & 0xF0 == 0xE0) and |
| 52 | + (c2 & 0xC0 == 0x80) and |
| 53 | + (c3 & 0xC0 == 0x80)): |
| 54 | + # U+0800 - U+FFFF - 16 bits |
| 55 | + c = (((((c1 & 0x0F) << 6) | |
| 56 | + (c2 & 0x3F)) << 6) | |
| 57 | + (c3 & 0x3f)) |
| 58 | + if ((c < 0x800) or (0xD800 <= c <= 0xDFFF)): |
| 59 | + # Overlong encoding or surrogate. |
| 60 | + skips.extend([i, i + 1, i + 2]) |
| 61 | + i += 3 |
| 62 | + continue |
| 63 | + c4 = bytes[i + 3] |
| 64 | + if ((c1 & 0xF8 == 0xF0) and |
| 65 | + (c2 & 0xC0 == 0x80) and |
| 66 | + (c3 & 0xC0 == 0x80) and |
| 67 | + (c4 & 0xC0 == 0x80)): |
| 68 | + # U+10000 - U+10FFFF - 21 bits |
| 69 | + c = (((((((c1 & 0x0F) << 6) | |
| 70 | + (c2 & 0x3F)) << 6) | |
| 71 | + (c3 & 0x3F)) << 6) | |
| 72 | + (c4 & 0x3F)) |
| 73 | + if (c < 0x10000) or (c > 0x10FFFF): |
| 74 | + # Overlong encoding or invalid code point. |
| 75 | + skips.extend([i, i + 1, i + 2, i + 3]) |
| 76 | + i += 4 |
| 77 | + continue |
| 78 | + except IndexError: |
| 79 | + pass |
| 80 | + skips.append(i) |
| 81 | + i += 1 |
| 82 | + return skips |
| 83 | + |
| 84 | + |
| 85 | +# XXX backport: Another helper to support the Python 2 UTF-8 decoding hack. |
| 86 | +def _chunks(b, indexes): |
| 87 | + i = 0 |
| 88 | + for j in indexes: |
| 89 | + yield b[i:j] |
| 90 | + yield b[j:j + 1] |
| 91 | + i = j + 1 |
| 92 | + yield b[i:] |
| 93 | + |
| 94 | + |
| 95 | +def _fscodec(): |
| 96 | + encoding = sys.getfilesystemencoding() |
| 97 | + if encoding == 'mbcs': |
| 98 | + errors = 'strict' |
| 99 | + else: |
| 100 | + errors = 'surrogateescape' |
| 101 | + |
| 102 | + # XXX backport: Do we need to hack around Python 2's UTF-8 codec? |
| 103 | + import codecs # Use codecs.lookup() for name normalisation. |
| 104 | + _HACK_AROUND_PY2_UTF8 = (sys.version_info < (3,) and |
| 105 | + codecs.lookup(encoding) == codecs.lookup('utf-8')) |
| 106 | + |
| 107 | + # XXX backport: chr(octet) became bytes([octet]) |
| 108 | + _byte = chr if sys.version_info < (3,) else lambda i: bytes([i]) |
| 109 | + |
| 110 | + def fsencode(filename): |
| 111 | + """ |
| 112 | + Encode filename to the filesystem encoding with 'surrogateescape' error |
| 113 | + handler, return bytes unchanged. On Windows, use 'strict' error handler if |
| 114 | + the file system encoding is 'mbcs' (which is the default encoding). |
| 115 | + """ |
| 116 | + if isinstance(filename, bytes): |
| 117 | + return filename |
| 118 | + elif isinstance(filename, _str): |
| 119 | + if _HACK_AROUND_PY2_UTF8: |
| 120 | + # XXX backport: Unlike Python 3, Python 2's UTF-8 codec does not |
| 121 | + # consider surrogate codepoints invalid, so the surrogateescape |
| 122 | + # error handler never gets invoked to encode them back into high |
| 123 | + # bytes. |
| 124 | + # |
| 125 | + # This code hacks around that by manually encoding the surrogate |
| 126 | + # codepoints to high bytes, without relying on surrogateescape. |
| 127 | + # |
| 128 | + return b''.join( |
| 129 | + (_byte(ord(c) - 0xDC00) if 0xDC00 <= ord(c) <= 0xDCFF else |
| 130 | + c.encode(encoding)) |
| 131 | + for c in filename) |
| 132 | + else: |
| 133 | + return filename.encode(encoding, errors) |
| 134 | + else: |
| 135 | + # XXX backport: unicode instead of str for Python 2 |
| 136 | + raise TypeError("expect bytes or {_str}, not {}".format(type(filename).__name__, |
| 137 | + _str=_str.__name__, )) |
| 138 | + |
| 139 | + def fsdecode(filename): |
| 140 | + """ |
| 141 | + Decode filename from the filesystem encoding with 'surrogateescape' error |
| 142 | + handler, return str unchanged. On Windows, use 'strict' error handler if |
| 143 | + the file system encoding is 'mbcs' (which is the default encoding). |
| 144 | + """ |
| 145 | + if isinstance(filename, _str): |
| 146 | + return filename |
| 147 | + elif isinstance(filename, bytes): |
| 148 | + if _HACK_AROUND_PY2_UTF8: |
| 149 | + # XXX backport: See the remarks in fsencode() above. |
| 150 | + # |
| 151 | + # This case is slightly trickier: Python 2 will invoke the |
| 152 | + # surrogateescape error handler for most bad high byte |
| 153 | + # sequences, *except* for full UTF-8 sequences that happen to |
| 154 | + # decode to surrogate codepoints. |
| 155 | + # |
| 156 | + # For decoding, it's not trivial to sidestep the UTF-8 codec |
| 157 | + # only for surrogates like fsencode() does, but as a hack we can |
| 158 | + # split the input into separate chunks around each invalid byte, |
| 159 | + # decode the chunks separately, and join the results. |
| 160 | + # |
| 161 | + # This prevents Python 2's UTF-8 codec from seeing the encoded |
| 162 | + # surrogate sequences as valid, which lets surrogateescape take |
| 163 | + # over and escape the individual bytes. |
| 164 | + # |
| 165 | + # TODO: Improve this. |
| 166 | + # |
| 167 | + from array import array |
| 168 | + indexes = _invalid_utf8_indexes(array(str('B'), filename)) |
| 169 | + return ''.join(chunk.decode(encoding, errors) |
| 170 | + for chunk in _chunks(filename, indexes)) |
| 171 | + else: |
| 172 | + return filename.decode(encoding, errors) |
| 173 | + else: |
| 174 | + # XXX backport: unicode instead of str for Python 2 |
| 175 | + raise TypeError("expect bytes or {_str}, not {}".format(type(filename).__name__, |
| 176 | + _str=_str.__name__, )) |
| 177 | + |
| 178 | + return fsencode, fsdecode |
| 179 | + |
| 180 | +fsencode, fsdecode = _fscodec() |
| 181 | +del _fscodec |
0 commit comments