_compat.py 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302
  1. """Private module full of compatibility hacks.
  2. Primarily this is for downstream redistributions of requests that unvendor
  3. urllib3 without providing a shim.
  4. .. warning::
  5. This module is private. If you use it, and something breaks, you were
  6. warned
  7. """
  8. import sys
  9. import requests
  10. try:
  11. from requests.packages.urllib3 import fields
  12. from requests.packages.urllib3 import filepost
  13. from requests.packages.urllib3 import poolmanager
  14. except ImportError:
  15. from urllib3 import fields
  16. from urllib3 import filepost
  17. from urllib3 import poolmanager
  18. try:
  19. from requests.packages.urllib3.connection import HTTPConnection
  20. from requests.packages.urllib3 import connection
  21. except ImportError:
  22. try:
  23. from urllib3.connection import HTTPConnection
  24. from urllib3 import connection
  25. except ImportError:
  26. HTTPConnection = None
  27. connection = None
  28. if requests.__build__ < 0x020300:
  29. timeout = None
  30. else:
  31. try:
  32. from requests.packages.urllib3.util import timeout
  33. except ImportError:
  34. from urllib3.util import timeout
  35. PY3 = sys.version_info > (3, 0)
  36. if PY3:
  37. from collections.abc import Mapping, MutableMapping
  38. import queue
  39. from urllib.parse import urlencode, urljoin
  40. else:
  41. from collections import Mapping, MutableMapping
  42. import Queue as queue
  43. from urllib import urlencode
  44. from urlparse import urljoin
  45. try:
  46. basestring = basestring
  47. except NameError:
  48. basestring = (str, bytes)
  49. class HTTPHeaderDict(MutableMapping):
  50. """
  51. :param headers:
  52. An iterable of field-value pairs. Must not contain multiple field names
  53. when compared case-insensitively.
  54. :param kwargs:
  55. Additional field-value pairs to pass in to ``dict.update``.
  56. A ``dict`` like container for storing HTTP Headers.
  57. Field names are stored and compared case-insensitively in compliance with
  58. RFC 7230. Iteration provides the first case-sensitive key seen for each
  59. case-insensitive pair.
  60. Using ``__setitem__`` syntax overwrites fields that compare equal
  61. case-insensitively in order to maintain ``dict``'s api. For fields that
  62. compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add``
  63. in a loop.
  64. If multiple fields that are equal case-insensitively are passed to the
  65. constructor or ``.update``, the behavior is undefined and some will be
  66. lost.
  67. >>> headers = HTTPHeaderDict()
  68. >>> headers.add('Set-Cookie', 'foo=bar')
  69. >>> headers.add('set-cookie', 'baz=quxx')
  70. >>> headers['content-length'] = '7'
  71. >>> headers['SET-cookie']
  72. 'foo=bar, baz=quxx'
  73. >>> headers['Content-Length']
  74. '7'
  75. """
  76. def __init__(self, headers=None, **kwargs):
  77. super(HTTPHeaderDict, self).__init__()
  78. self._container = {}
  79. if headers is not None:
  80. if isinstance(headers, HTTPHeaderDict):
  81. self._copy_from(headers)
  82. else:
  83. self.extend(headers)
  84. if kwargs:
  85. self.extend(kwargs)
  86. def __setitem__(self, key, val):
  87. self._container[key.lower()] = (key, val)
  88. return self._container[key.lower()]
  89. def __getitem__(self, key):
  90. val = self._container[key.lower()]
  91. return ', '.join(val[1:])
  92. def __delitem__(self, key):
  93. del self._container[key.lower()]
  94. def __contains__(self, key):
  95. return key.lower() in self._container
  96. def __eq__(self, other):
  97. if not isinstance(other, Mapping) and not hasattr(other, 'keys'):
  98. return False
  99. if not isinstance(other, type(self)):
  100. other = type(self)(other)
  101. return ({k.lower(): v for k, v in self.itermerged()} ==
  102. {k.lower(): v for k, v in other.itermerged()})
  103. def __ne__(self, other):
  104. return not self.__eq__(other)
  105. if not PY3: # Python 2
  106. iterkeys = MutableMapping.iterkeys
  107. itervalues = MutableMapping.itervalues
  108. __marker = object()
  109. def __len__(self):
  110. return len(self._container)
  111. def __iter__(self):
  112. # Only provide the originally cased names
  113. for vals in self._container.values():
  114. yield vals[0]
  115. def pop(self, key, default=__marker):
  116. """D.pop(k[,d]) -> v, remove specified key and return its value.
  117. If key is not found, d is returned if given, otherwise KeyError is
  118. raised.
  119. """
  120. # Using the MutableMapping function directly fails due to the private
  121. # marker.
  122. # Using ordinary dict.pop would expose the internal structures.
  123. # So let's reinvent the wheel.
  124. try:
  125. value = self[key]
  126. except KeyError:
  127. if default is self.__marker:
  128. raise
  129. return default
  130. else:
  131. del self[key]
  132. return value
  133. def discard(self, key):
  134. try:
  135. del self[key]
  136. except KeyError:
  137. pass
  138. def add(self, key, val):
  139. """Adds a (name, value) pair, doesn't overwrite the value if it already
  140. exists.
  141. >>> headers = HTTPHeaderDict(foo='bar')
  142. >>> headers.add('Foo', 'baz')
  143. >>> headers['foo']
  144. 'bar, baz'
  145. """
  146. key_lower = key.lower()
  147. new_vals = key, val
  148. # Keep the common case aka no item present as fast as possible
  149. vals = self._container.setdefault(key_lower, new_vals)
  150. if new_vals is not vals:
  151. # new_vals was not inserted, as there was a previous one
  152. if isinstance(vals, list):
  153. # If already several items got inserted, we have a list
  154. vals.append(val)
  155. else:
  156. # vals should be a tuple then, i.e. only one item so far
  157. # Need to convert the tuple to list for further extension
  158. self._container[key_lower] = [vals[0], vals[1], val]
  159. def extend(self, *args, **kwargs):
  160. """Generic import function for any type of header-like object.
  161. Adapted version of MutableMapping.update in order to insert items
  162. with self.add instead of self.__setitem__
  163. """
  164. if len(args) > 1:
  165. raise TypeError("extend() takes at most 1 positional "
  166. "arguments ({} given)".format(len(args)))
  167. other = args[0] if len(args) >= 1 else ()
  168. if isinstance(other, HTTPHeaderDict):
  169. for key, val in other.iteritems():
  170. self.add(key, val)
  171. elif isinstance(other, Mapping):
  172. for key in other:
  173. self.add(key, other[key])
  174. elif hasattr(other, "keys"):
  175. for key in other.keys():
  176. self.add(key, other[key])
  177. else:
  178. for key, value in other:
  179. self.add(key, value)
  180. for key, value in kwargs.items():
  181. self.add(key, value)
  182. def getlist(self, key):
  183. """Returns a list of all the values for the named field. Returns an
  184. empty list if the key doesn't exist."""
  185. try:
  186. vals = self._container[key.lower()]
  187. except KeyError:
  188. return []
  189. else:
  190. if isinstance(vals, tuple):
  191. return [vals[1]]
  192. else:
  193. return vals[1:]
  194. # Backwards compatibility for httplib
  195. getheaders = getlist
  196. getallmatchingheaders = getlist
  197. iget = getlist
  198. def __repr__(self):
  199. return "%s(%s)" % (type(self).__name__, dict(self.itermerged()))
  200. def _copy_from(self, other):
  201. for key in other:
  202. val = other.getlist(key)
  203. if isinstance(val, list):
  204. # Don't need to convert tuples
  205. val = list(val)
  206. self._container[key.lower()] = [key] + val
  207. def copy(self):
  208. clone = type(self)()
  209. clone._copy_from(self)
  210. return clone
  211. def iteritems(self):
  212. """Iterate over all header lines, including duplicate ones."""
  213. for key in self:
  214. vals = self._container[key.lower()]
  215. for val in vals[1:]:
  216. yield vals[0], val
  217. def itermerged(self):
  218. """Iterate over all headers, merging duplicate ones together."""
  219. for key in self:
  220. val = self._container[key.lower()]
  221. yield val[0], ', '.join(val[1:])
  222. def items(self):
  223. return list(self.iteritems())
  224. @classmethod
  225. def from_httplib(cls, message): # Python 2
  226. """Read headers from a Python 2 httplib message object."""
  227. # python2.7 does not expose a proper API for exporting multiheaders
  228. # efficiently. This function re-reads raw lines from the message
  229. # object and extracts the multiheaders properly.
  230. headers = []
  231. for line in message.headers:
  232. if line.startswith((' ', '\t')):
  233. key, value = headers[-1]
  234. headers[-1] = (key, value + '\r\n' + line.rstrip())
  235. continue
  236. key, value = line.split(':', 1)
  237. headers.append((key, value.strip()))
  238. return cls(headers)
  239. __all__ = (
  240. 'basestring',
  241. 'connection',
  242. 'fields',
  243. 'filepost',
  244. 'poolmanager',
  245. 'timeout',
  246. 'HTTPHeaderDict',
  247. 'queue',
  248. 'urlencode',
  249. 'urljoin',
  250. )