response.py 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285
  1. from __future__ import annotations
  2. import json as _json
  3. import logging
  4. import typing
  5. from contextlib import contextmanager
  6. from dataclasses import dataclass
  7. from http.client import HTTPException as HTTPException
  8. from io import BytesIO, IOBase
  9. from ...exceptions import InvalidHeader, TimeoutError
  10. from ...response import BaseHTTPResponse
  11. from ...util.retry import Retry
  12. from .request import EmscriptenRequest
  13. if typing.TYPE_CHECKING:
  14. from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection
  15. log = logging.getLogger(__name__)
  16. @dataclass
  17. class EmscriptenResponse:
  18. status_code: int
  19. headers: dict[str, str]
  20. body: IOBase | bytes
  21. request: EmscriptenRequest
  22. class EmscriptenHttpResponseWrapper(BaseHTTPResponse):
  23. def __init__(
  24. self,
  25. internal_response: EmscriptenResponse,
  26. url: str | None = None,
  27. connection: BaseHTTPConnection | BaseHTTPSConnection | None = None,
  28. ):
  29. self._pool = None # set by pool class
  30. self._body = None
  31. self._response = internal_response
  32. self._url = url
  33. self._connection = connection
  34. self._closed = False
  35. super().__init__(
  36. headers=internal_response.headers,
  37. status=internal_response.status_code,
  38. request_url=url,
  39. version=0,
  40. version_string="HTTP/?",
  41. reason="",
  42. decode_content=True,
  43. )
  44. self.length_remaining = self._init_length(self._response.request.method)
  45. self.length_is_certain = False
  46. @property
  47. def url(self) -> str | None:
  48. return self._url
  49. @url.setter
  50. def url(self, url: str | None) -> None:
  51. self._url = url
  52. @property
  53. def connection(self) -> BaseHTTPConnection | BaseHTTPSConnection | None:
  54. return self._connection
  55. @property
  56. def retries(self) -> Retry | None:
  57. return self._retries
  58. @retries.setter
  59. def retries(self, retries: Retry | None) -> None:
  60. # Override the request_url if retries has a redirect location.
  61. self._retries = retries
  62. def stream(
  63. self, amt: int | None = 2**16, decode_content: bool | None = None
  64. ) -> typing.Generator[bytes]:
  65. """
  66. A generator wrapper for the read() method. A call will block until
  67. ``amt`` bytes have been read from the connection or until the
  68. connection is closed.
  69. :param amt:
  70. How much of the content to read. The generator will return up to
  71. much data per iteration, but may return less. This is particularly
  72. likely when using compressed data. However, the empty string will
  73. never be returned.
  74. :param decode_content:
  75. If True, will attempt to decode the body based on the
  76. 'content-encoding' header.
  77. """
  78. while True:
  79. data = self.read(amt=amt, decode_content=decode_content)
  80. if data:
  81. yield data
  82. else:
  83. break
  84. def _init_length(self, request_method: str | None) -> int | None:
  85. length: int | None
  86. content_length: str | None = self.headers.get("content-length")
  87. if content_length is not None:
  88. try:
  89. # RFC 7230 section 3.3.2 specifies multiple content lengths can
  90. # be sent in a single Content-Length header
  91. # (e.g. Content-Length: 42, 42). This line ensures the values
  92. # are all valid ints and that as long as the `set` length is 1,
  93. # all values are the same. Otherwise, the header is invalid.
  94. lengths = {int(val) for val in content_length.split(",")}
  95. if len(lengths) > 1:
  96. raise InvalidHeader(
  97. "Content-Length contained multiple "
  98. "unmatching values (%s)" % content_length
  99. )
  100. length = lengths.pop()
  101. except ValueError:
  102. length = None
  103. else:
  104. if length < 0:
  105. length = None
  106. else: # if content_length is None
  107. length = None
  108. # Check for responses that shouldn't include a body
  109. if (
  110. self.status in (204, 304)
  111. or 100 <= self.status < 200
  112. or request_method == "HEAD"
  113. ):
  114. length = 0
  115. return length
  116. def read(
  117. self,
  118. amt: int | None = None,
  119. decode_content: bool | None = None, # ignored because browser decodes always
  120. cache_content: bool = False,
  121. ) -> bytes:
  122. if (
  123. self._closed
  124. or self._response is None
  125. or (isinstance(self._response.body, IOBase) and self._response.body.closed)
  126. ):
  127. return b""
  128. with self._error_catcher():
  129. # body has been preloaded as a string by XmlHttpRequest
  130. if not isinstance(self._response.body, IOBase):
  131. self.length_remaining = len(self._response.body)
  132. self.length_is_certain = True
  133. # wrap body in IOStream
  134. self._response.body = BytesIO(self._response.body)
  135. if amt is not None and amt >= 0:
  136. # don't cache partial content
  137. cache_content = False
  138. data = self._response.body.read(amt)
  139. if self.length_remaining is not None:
  140. self.length_remaining = max(self.length_remaining - len(data), 0)
  141. if (self.length_is_certain and self.length_remaining == 0) or len(
  142. data
  143. ) < amt:
  144. # definitely finished reading, close response stream
  145. self._response.body.close()
  146. return typing.cast(bytes, data)
  147. else: # read all we can (and cache it)
  148. data = self._response.body.read()
  149. if cache_content:
  150. self._body = data
  151. if self.length_remaining is not None:
  152. self.length_remaining = max(self.length_remaining - len(data), 0)
  153. if len(data) == 0 or (
  154. self.length_is_certain and self.length_remaining == 0
  155. ):
  156. # definitely finished reading, close response stream
  157. self._response.body.close()
  158. return typing.cast(bytes, data)
  159. def read_chunked(
  160. self,
  161. amt: int | None = None,
  162. decode_content: bool | None = None,
  163. ) -> typing.Generator[bytes]:
  164. # chunked is handled by browser
  165. while True:
  166. bytes = self.read(amt, decode_content)
  167. if not bytes:
  168. break
  169. yield bytes
  170. def release_conn(self) -> None:
  171. if not self._pool or not self._connection:
  172. return None
  173. self._pool._put_conn(self._connection)
  174. self._connection = None
  175. def drain_conn(self) -> None:
  176. self.close()
  177. @property
  178. def data(self) -> bytes:
  179. if self._body:
  180. return self._body
  181. else:
  182. return self.read(cache_content=True)
  183. def json(self) -> typing.Any:
  184. """
  185. Deserializes the body of the HTTP response as a Python object.
  186. The body of the HTTP response must be encoded using UTF-8, as per
  187. `RFC 8529 Section 8.1 <https://www.rfc-editor.org/rfc/rfc8259#section-8.1>`_.
  188. To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to
  189. your custom decoder instead.
  190. If the body of the HTTP response is not decodable to UTF-8, a
  191. `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a
  192. valid JSON document, a `json.JSONDecodeError` will be raised.
  193. Read more :ref:`here <json_content>`.
  194. :returns: The body of the HTTP response as a Python object.
  195. """
  196. data = self.data.decode("utf-8")
  197. return _json.loads(data)
  198. def close(self) -> None:
  199. if not self._closed:
  200. if isinstance(self._response.body, IOBase):
  201. self._response.body.close()
  202. if self._connection:
  203. self._connection.close()
  204. self._connection = None
  205. self._closed = True
  206. @contextmanager
  207. def _error_catcher(self) -> typing.Generator[None]:
  208. """
  209. Catch Emscripten specific exceptions thrown by fetch.py,
  210. instead re-raising urllib3 variants, so that low-level exceptions
  211. are not leaked in the high-level api.
  212. On exit, release the connection back to the pool.
  213. """
  214. from .fetch import _RequestError, _TimeoutError # avoid circular import
  215. clean_exit = False
  216. try:
  217. yield
  218. # If no exception is thrown, we should avoid cleaning up
  219. # unnecessarily.
  220. clean_exit = True
  221. except _TimeoutError as e:
  222. raise TimeoutError(str(e))
  223. except _RequestError as e:
  224. raise HTTPException(str(e))
  225. finally:
  226. # If we didn't terminate cleanly, we need to throw away our
  227. # connection.
  228. if not clean_exit:
  229. # The response may not be closed but we're not going to use it
  230. # anymore so close it now
  231. if (
  232. isinstance(self._response.body, IOBase)
  233. and not self._response.body.closed
  234. ):
  235. self._response.body.close()
  236. # release the connection back to the pool
  237. self.release_conn()
  238. else:
  239. # If we have read everything from the response stream,
  240. # return the connection back to the pool.
  241. if (
  242. isinstance(self._response.body, IOBase)
  243. and self._response.body.closed
  244. ):
  245. self.release_conn()