connection.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044
  1. from __future__ import annotations
  2. import datetime
  3. import http.client
  4. import logging
  5. import os
  6. import re
  7. import socket
  8. import sys
  9. import threading
  10. import typing
  11. import warnings
  12. from http.client import HTTPConnection as _HTTPConnection
  13. from http.client import HTTPException as HTTPException # noqa: F401
  14. from http.client import ResponseNotReady
  15. from socket import timeout as SocketTimeout
  16. if typing.TYPE_CHECKING:
  17. from .response import HTTPResponse
  18. from .util.ssl_ import _TYPE_PEER_CERT_RET_DICT
  19. from .util.ssltransport import SSLTransport
  20. from ._collections import HTTPHeaderDict
  21. from .http2 import probe as http2_probe
  22. from .util.response import assert_header_parsing
  23. from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT, Timeout
  24. from .util.util import to_str
  25. from .util.wait import wait_for_read
  26. try: # Compiled with SSL?
  27. import ssl
  28. BaseSSLError = ssl.SSLError
  29. except (ImportError, AttributeError):
  30. ssl = None # type: ignore[assignment]
  31. class BaseSSLError(BaseException): # type: ignore[no-redef]
  32. pass
  33. from ._base_connection import _TYPE_BODY
  34. from ._base_connection import ProxyConfig as ProxyConfig
  35. from ._base_connection import _ResponseOptions as _ResponseOptions
  36. from ._version import __version__
  37. from .exceptions import (
  38. ConnectTimeoutError,
  39. HeaderParsingError,
  40. NameResolutionError,
  41. NewConnectionError,
  42. ProxyError,
  43. SystemTimeWarning,
  44. )
  45. from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection, ssl_
  46. from .util.request import body_to_chunks
  47. from .util.ssl_ import assert_fingerprint as _assert_fingerprint
  48. from .util.ssl_ import (
  49. create_urllib3_context,
  50. is_ipaddress,
  51. resolve_cert_reqs,
  52. resolve_ssl_version,
  53. ssl_wrap_socket,
  54. )
  55. from .util.ssl_match_hostname import CertificateError, match_hostname
  56. from .util.url import Url
  57. # Not a no-op, we're adding this to the namespace so it can be imported.
  58. ConnectionError = ConnectionError
  59. BrokenPipeError = BrokenPipeError
  60. log = logging.getLogger(__name__)
  61. port_by_scheme = {"http": 80, "https": 443}
  62. # When it comes time to update this value as a part of regular maintenance
  63. # (ie test_recent_date is failing) update it to ~6 months before the current date.
  64. RECENT_DATE = datetime.date(2023, 6, 1)
  65. _CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]")
  66. class HTTPConnection(_HTTPConnection):
  67. """
  68. Based on :class:`http.client.HTTPConnection` but provides an extra constructor
  69. backwards-compatibility layer between older and newer Pythons.
  70. Additional keyword parameters are used to configure attributes of the connection.
  71. Accepted parameters include:
  72. - ``source_address``: Set the source address for the current connection.
  73. - ``socket_options``: Set specific options on the underlying socket. If not specified, then
  74. defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
  75. Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.
  76. For example, if you wish to enable TCP Keep Alive in addition to the defaults,
  77. you might pass:
  78. .. code-block:: python
  79. HTTPConnection.default_socket_options + [
  80. (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
  81. ]
  82. Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
  83. """
  84. default_port: typing.ClassVar[int] = port_by_scheme["http"] # type: ignore[misc]
  85. #: Disable Nagle's algorithm by default.
  86. #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
  87. default_socket_options: typing.ClassVar[connection._TYPE_SOCKET_OPTIONS] = [
  88. (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
  89. ]
  90. #: Whether this connection verifies the host's certificate.
  91. is_verified: bool = False
  92. #: Whether this proxy connection verified the proxy host's certificate.
  93. # If no proxy is currently connected to the value will be ``None``.
  94. proxy_is_verified: bool | None = None
  95. blocksize: int
  96. source_address: tuple[str, int] | None
  97. socket_options: connection._TYPE_SOCKET_OPTIONS | None
  98. _has_connected_to_proxy: bool
  99. _response_options: _ResponseOptions | None
  100. _tunnel_host: str | None
  101. _tunnel_port: int | None
  102. _tunnel_scheme: str | None
  103. def __init__(
  104. self,
  105. host: str,
  106. port: int | None = None,
  107. *,
  108. timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
  109. source_address: tuple[str, int] | None = None,
  110. blocksize: int = 16384,
  111. socket_options: None | (
  112. connection._TYPE_SOCKET_OPTIONS
  113. ) = default_socket_options,
  114. proxy: Url | None = None,
  115. proxy_config: ProxyConfig | None = None,
  116. ) -> None:
  117. super().__init__(
  118. host=host,
  119. port=port,
  120. timeout=Timeout.resolve_default_timeout(timeout),
  121. source_address=source_address,
  122. blocksize=blocksize,
  123. )
  124. self.socket_options = socket_options
  125. self.proxy = proxy
  126. self.proxy_config = proxy_config
  127. self._has_connected_to_proxy = False
  128. self._response_options = None
  129. self._tunnel_host: str | None = None
  130. self._tunnel_port: int | None = None
  131. self._tunnel_scheme: str | None = None
  132. @property
  133. def host(self) -> str:
  134. """
  135. Getter method to remove any trailing dots that indicate the hostname is an FQDN.
  136. In general, SSL certificates don't include the trailing dot indicating a
  137. fully-qualified domain name, and thus, they don't validate properly when
  138. checked against a domain name that includes the dot. In addition, some
  139. servers may not expect to receive the trailing dot when provided.
  140. However, the hostname with trailing dot is critical to DNS resolution; doing a
  141. lookup with the trailing dot will properly only resolve the appropriate FQDN,
  142. whereas a lookup without a trailing dot will search the system's search domain
  143. list. Thus, it's important to keep the original host around for use only in
  144. those cases where it's appropriate (i.e., when doing DNS lookup to establish the
  145. actual TCP connection across which we're going to send HTTP requests).
  146. """
  147. return self._dns_host.rstrip(".")
  148. @host.setter
  149. def host(self, value: str) -> None:
  150. """
  151. Setter for the `host` property.
  152. We assume that only urllib3 uses the _dns_host attribute; httplib itself
  153. only uses `host`, and it seems reasonable that other libraries follow suit.
  154. """
  155. self._dns_host = value
  156. def _new_conn(self) -> socket.socket:
  157. """Establish a socket connection and set nodelay settings on it.
  158. :return: New socket connection.
  159. """
  160. try:
  161. sock = connection.create_connection(
  162. (self._dns_host, self.port),
  163. self.timeout,
  164. source_address=self.source_address,
  165. socket_options=self.socket_options,
  166. )
  167. except socket.gaierror as e:
  168. raise NameResolutionError(self.host, self, e) from e
  169. except SocketTimeout as e:
  170. raise ConnectTimeoutError(
  171. self,
  172. f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
  173. ) from e
  174. except OSError as e:
  175. raise NewConnectionError(
  176. self, f"Failed to establish a new connection: {e}"
  177. ) from e
  178. sys.audit("http.client.connect", self, self.host, self.port)
  179. return sock
  180. def set_tunnel(
  181. self,
  182. host: str,
  183. port: int | None = None,
  184. headers: typing.Mapping[str, str] | None = None,
  185. scheme: str = "http",
  186. ) -> None:
  187. if scheme not in ("http", "https"):
  188. raise ValueError(
  189. f"Invalid proxy scheme for tunneling: {scheme!r}, must be either 'http' or 'https'"
  190. )
  191. super().set_tunnel(host, port=port, headers=headers)
  192. self._tunnel_scheme = scheme
  193. if sys.version_info < (3, 11, 4):
  194. def _tunnel(self) -> None:
  195. _MAXLINE = http.client._MAXLINE # type: ignore[attr-defined]
  196. connect = b"CONNECT %s:%d HTTP/1.0\r\n" % ( # type: ignore[str-format]
  197. self._tunnel_host.encode("ascii"), # type: ignore[union-attr]
  198. self._tunnel_port,
  199. )
  200. headers = [connect]
  201. for header, value in self._tunnel_headers.items(): # type: ignore[attr-defined]
  202. headers.append(f"{header}: {value}\r\n".encode("latin-1"))
  203. headers.append(b"\r\n")
  204. # Making a single send() call instead of one per line encourages
  205. # the host OS to use a more optimal packet size instead of
  206. # potentially emitting a series of small packets.
  207. self.send(b"".join(headers))
  208. del headers
  209. response = self.response_class(self.sock, method=self._method) # type: ignore[attr-defined]
  210. try:
  211. (version, code, message) = response._read_status() # type: ignore[attr-defined]
  212. if code != http.HTTPStatus.OK:
  213. self.close()
  214. raise OSError(f"Tunnel connection failed: {code} {message.strip()}")
  215. while True:
  216. line = response.fp.readline(_MAXLINE + 1)
  217. if len(line) > _MAXLINE:
  218. raise http.client.LineTooLong("header line")
  219. if not line:
  220. # for sites which EOF without sending a trailer
  221. break
  222. if line in (b"\r\n", b"\n", b""):
  223. break
  224. if self.debuglevel > 0:
  225. print("header:", line.decode())
  226. finally:
  227. response.close()
  228. def connect(self) -> None:
  229. self.sock = self._new_conn()
  230. if self._tunnel_host:
  231. # If we're tunneling it means we're connected to our proxy.
  232. self._has_connected_to_proxy = True
  233. # TODO: Fix tunnel so it doesn't depend on self.sock state.
  234. self._tunnel()
  235. # If there's a proxy to be connected to we are fully connected.
  236. # This is set twice (once above and here) due to forwarding proxies
  237. # not using tunnelling.
  238. self._has_connected_to_proxy = bool(self.proxy)
  239. if self._has_connected_to_proxy:
  240. self.proxy_is_verified = False
  241. @property
  242. def is_closed(self) -> bool:
  243. return self.sock is None
  244. @property
  245. def is_connected(self) -> bool:
  246. if self.sock is None:
  247. return False
  248. return not wait_for_read(self.sock, timeout=0.0)
  249. @property
  250. def has_connected_to_proxy(self) -> bool:
  251. return self._has_connected_to_proxy
  252. @property
  253. def proxy_is_forwarding(self) -> bool:
  254. """
  255. Return True if a forwarding proxy is configured, else return False
  256. """
  257. return bool(self.proxy) and self._tunnel_host is None
  258. @property
  259. def proxy_is_tunneling(self) -> bool:
  260. """
  261. Return True if a tunneling proxy is configured, else return False
  262. """
  263. return self._tunnel_host is not None
  264. def close(self) -> None:
  265. try:
  266. super().close()
  267. finally:
  268. # Reset all stateful properties so connection
  269. # can be re-used without leaking prior configs.
  270. self.sock = None
  271. self.is_verified = False
  272. self.proxy_is_verified = None
  273. self._has_connected_to_proxy = False
  274. self._response_options = None
  275. self._tunnel_host = None
  276. self._tunnel_port = None
  277. self._tunnel_scheme = None
  278. def putrequest(
  279. self,
  280. method: str,
  281. url: str,
  282. skip_host: bool = False,
  283. skip_accept_encoding: bool = False,
  284. ) -> None:
  285. """"""
  286. # Empty docstring because the indentation of CPython's implementation
  287. # is broken but we don't want this method in our documentation.
  288. match = _CONTAINS_CONTROL_CHAR_RE.search(method)
  289. if match:
  290. raise ValueError(
  291. f"Method cannot contain non-token characters {method!r} (found at least {match.group()!r})"
  292. )
  293. return super().putrequest(
  294. method, url, skip_host=skip_host, skip_accept_encoding=skip_accept_encoding
  295. )
  296. def putheader(self, header: str, *values: str) -> None: # type: ignore[override]
  297. """"""
  298. if not any(isinstance(v, str) and v == SKIP_HEADER for v in values):
  299. super().putheader(header, *values)
  300. elif to_str(header.lower()) not in SKIPPABLE_HEADERS:
  301. skippable_headers = "', '".join(
  302. [str.title(header) for header in sorted(SKIPPABLE_HEADERS)]
  303. )
  304. raise ValueError(
  305. f"urllib3.util.SKIP_HEADER only supports '{skippable_headers}'"
  306. )
  307. # `request` method's signature intentionally violates LSP.
  308. # urllib3's API is different from `http.client.HTTPConnection` and the subclassing is only incidental.
  309. def request( # type: ignore[override]
  310. self,
  311. method: str,
  312. url: str,
  313. body: _TYPE_BODY | None = None,
  314. headers: typing.Mapping[str, str] | None = None,
  315. *,
  316. chunked: bool = False,
  317. preload_content: bool = True,
  318. decode_content: bool = True,
  319. enforce_content_length: bool = True,
  320. ) -> None:
  321. # Update the inner socket's timeout value to send the request.
  322. # This only triggers if the connection is re-used.
  323. if self.sock is not None:
  324. self.sock.settimeout(self.timeout)
  325. # Store these values to be fed into the HTTPResponse
  326. # object later. TODO: Remove this in favor of a real
  327. # HTTP lifecycle mechanism.
  328. # We have to store these before we call .request()
  329. # because sometimes we can still salvage a response
  330. # off the wire even if we aren't able to completely
  331. # send the request body.
  332. self._response_options = _ResponseOptions(
  333. request_method=method,
  334. request_url=url,
  335. preload_content=preload_content,
  336. decode_content=decode_content,
  337. enforce_content_length=enforce_content_length,
  338. )
  339. if headers is None:
  340. headers = {}
  341. header_keys = frozenset(to_str(k.lower()) for k in headers)
  342. skip_accept_encoding = "accept-encoding" in header_keys
  343. skip_host = "host" in header_keys
  344. self.putrequest(
  345. method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host
  346. )
  347. # Transform the body into an iterable of sendall()-able chunks
  348. # and detect if an explicit Content-Length is doable.
  349. chunks_and_cl = body_to_chunks(body, method=method, blocksize=self.blocksize)
  350. chunks = chunks_and_cl.chunks
  351. content_length = chunks_and_cl.content_length
  352. # When chunked is explicit set to 'True' we respect that.
  353. if chunked:
  354. if "transfer-encoding" not in header_keys:
  355. self.putheader("Transfer-Encoding", "chunked")
  356. else:
  357. # Detect whether a framing mechanism is already in use. If so
  358. # we respect that value, otherwise we pick chunked vs content-length
  359. # depending on the type of 'body'.
  360. if "content-length" in header_keys:
  361. chunked = False
  362. elif "transfer-encoding" in header_keys:
  363. chunked = True
  364. # Otherwise we go off the recommendation of 'body_to_chunks()'.
  365. else:
  366. chunked = False
  367. if content_length is None:
  368. if chunks is not None:
  369. chunked = True
  370. self.putheader("Transfer-Encoding", "chunked")
  371. else:
  372. self.putheader("Content-Length", str(content_length))
  373. # Now that framing headers are out of the way we send all the other headers.
  374. if "user-agent" not in header_keys:
  375. self.putheader("User-Agent", _get_default_user_agent())
  376. for header, value in headers.items():
  377. self.putheader(header, value)
  378. self.endheaders()
  379. # If we're given a body we start sending that in chunks.
  380. if chunks is not None:
  381. for chunk in chunks:
  382. # Sending empty chunks isn't allowed for TE: chunked
  383. # as it indicates the end of the body.
  384. if not chunk:
  385. continue
  386. if isinstance(chunk, str):
  387. chunk = chunk.encode("utf-8")
  388. if chunked:
  389. self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
  390. else:
  391. self.send(chunk)
  392. # Regardless of whether we have a body or not, if we're in
  393. # chunked mode we want to send an explicit empty chunk.
  394. if chunked:
  395. self.send(b"0\r\n\r\n")
  396. def request_chunked(
  397. self,
  398. method: str,
  399. url: str,
  400. body: _TYPE_BODY | None = None,
  401. headers: typing.Mapping[str, str] | None = None,
  402. ) -> None:
  403. """
  404. Alternative to the common request method, which sends the
  405. body with chunked encoding and not as one block
  406. """
  407. warnings.warn(
  408. "HTTPConnection.request_chunked() is deprecated and will be removed "
  409. "in urllib3 v2.1.0. Instead use HTTPConnection.request(..., chunked=True).",
  410. category=DeprecationWarning,
  411. stacklevel=2,
  412. )
  413. self.request(method, url, body=body, headers=headers, chunked=True)
  414. def getresponse( # type: ignore[override]
  415. self,
  416. ) -> HTTPResponse:
  417. """
  418. Get the response from the server.
  419. If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable.
  420. If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed.
  421. """
  422. # Raise the same error as http.client.HTTPConnection
  423. if self._response_options is None:
  424. raise ResponseNotReady()
  425. # Reset this attribute for being used again.
  426. resp_options = self._response_options
  427. self._response_options = None
  428. # Since the connection's timeout value may have been updated
  429. # we need to set the timeout on the socket.
  430. self.sock.settimeout(self.timeout)
  431. # This is needed here to avoid circular import errors
  432. from .response import HTTPResponse
  433. # Save a reference to the shutdown function before ownership is passed
  434. # to httplib_response
  435. # TODO should we implement it everywhere?
  436. _shutdown = getattr(self.sock, "shutdown", None)
  437. # Get the response from http.client.HTTPConnection
  438. httplib_response = super().getresponse()
  439. try:
  440. assert_header_parsing(httplib_response.msg)
  441. except (HeaderParsingError, TypeError) as hpe:
  442. log.warning(
  443. "Failed to parse headers (url=%s): %s",
  444. _url_from_connection(self, resp_options.request_url),
  445. hpe,
  446. exc_info=True,
  447. )
  448. headers = HTTPHeaderDict(httplib_response.msg.items())
  449. response = HTTPResponse(
  450. body=httplib_response,
  451. headers=headers,
  452. status=httplib_response.status,
  453. version=httplib_response.version,
  454. version_string=getattr(self, "_http_vsn_str", "HTTP/?"),
  455. reason=httplib_response.reason,
  456. preload_content=resp_options.preload_content,
  457. decode_content=resp_options.decode_content,
  458. original_response=httplib_response,
  459. enforce_content_length=resp_options.enforce_content_length,
  460. request_method=resp_options.request_method,
  461. request_url=resp_options.request_url,
  462. sock_shutdown=_shutdown,
  463. )
  464. return response
  465. class HTTPSConnection(HTTPConnection):
  466. """
  467. Many of the parameters to this constructor are passed to the underlying SSL
  468. socket by means of :py:func:`urllib3.util.ssl_wrap_socket`.
  469. """
  470. default_port = port_by_scheme["https"] # type: ignore[misc]
  471. cert_reqs: int | str | None = None
  472. ca_certs: str | None = None
  473. ca_cert_dir: str | None = None
  474. ca_cert_data: None | str | bytes = None
  475. ssl_version: int | str | None = None
  476. ssl_minimum_version: int | None = None
  477. ssl_maximum_version: int | None = None
  478. assert_fingerprint: str | None = None
  479. _connect_callback: typing.Callable[..., None] | None = None
  480. def __init__(
  481. self,
  482. host: str,
  483. port: int | None = None,
  484. *,
  485. timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
  486. source_address: tuple[str, int] | None = None,
  487. blocksize: int = 16384,
  488. socket_options: None | (
  489. connection._TYPE_SOCKET_OPTIONS
  490. ) = HTTPConnection.default_socket_options,
  491. proxy: Url | None = None,
  492. proxy_config: ProxyConfig | None = None,
  493. cert_reqs: int | str | None = None,
  494. assert_hostname: None | str | typing.Literal[False] = None,
  495. assert_fingerprint: str | None = None,
  496. server_hostname: str | None = None,
  497. ssl_context: ssl.SSLContext | None = None,
  498. ca_certs: str | None = None,
  499. ca_cert_dir: str | None = None,
  500. ca_cert_data: None | str | bytes = None,
  501. ssl_minimum_version: int | None = None,
  502. ssl_maximum_version: int | None = None,
  503. ssl_version: int | str | None = None, # Deprecated
  504. cert_file: str | None = None,
  505. key_file: str | None = None,
  506. key_password: str | None = None,
  507. ) -> None:
  508. super().__init__(
  509. host,
  510. port=port,
  511. timeout=timeout,
  512. source_address=source_address,
  513. blocksize=blocksize,
  514. socket_options=socket_options,
  515. proxy=proxy,
  516. proxy_config=proxy_config,
  517. )
  518. self.key_file = key_file
  519. self.cert_file = cert_file
  520. self.key_password = key_password
  521. self.ssl_context = ssl_context
  522. self.server_hostname = server_hostname
  523. self.assert_hostname = assert_hostname
  524. self.assert_fingerprint = assert_fingerprint
  525. self.ssl_version = ssl_version
  526. self.ssl_minimum_version = ssl_minimum_version
  527. self.ssl_maximum_version = ssl_maximum_version
  528. self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
  529. self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
  530. self.ca_cert_data = ca_cert_data
  531. # cert_reqs depends on ssl_context so calculate last.
  532. if cert_reqs is None:
  533. if self.ssl_context is not None:
  534. cert_reqs = self.ssl_context.verify_mode
  535. else:
  536. cert_reqs = resolve_cert_reqs(None)
  537. self.cert_reqs = cert_reqs
  538. self._connect_callback = None
  539. def set_cert(
  540. self,
  541. key_file: str | None = None,
  542. cert_file: str | None = None,
  543. cert_reqs: int | str | None = None,
  544. key_password: str | None = None,
  545. ca_certs: str | None = None,
  546. assert_hostname: None | str | typing.Literal[False] = None,
  547. assert_fingerprint: str | None = None,
  548. ca_cert_dir: str | None = None,
  549. ca_cert_data: None | str | bytes = None,
  550. ) -> None:
  551. """
  552. This method should only be called once, before the connection is used.
  553. """
  554. warnings.warn(
  555. "HTTPSConnection.set_cert() is deprecated and will be removed "
  556. "in urllib3 v2.1.0. Instead provide the parameters to the "
  557. "HTTPSConnection constructor.",
  558. category=DeprecationWarning,
  559. stacklevel=2,
  560. )
  561. # If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also
  562. # have an SSLContext object in which case we'll use its verify_mode.
  563. if cert_reqs is None:
  564. if self.ssl_context is not None:
  565. cert_reqs = self.ssl_context.verify_mode
  566. else:
  567. cert_reqs = resolve_cert_reqs(None)
  568. self.key_file = key_file
  569. self.cert_file = cert_file
  570. self.cert_reqs = cert_reqs
  571. self.key_password = key_password
  572. self.assert_hostname = assert_hostname
  573. self.assert_fingerprint = assert_fingerprint
  574. self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
  575. self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
  576. self.ca_cert_data = ca_cert_data
  577. def connect(self) -> None:
  578. # Today we don't need to be doing this step before the /actual/ socket
  579. # connection, however in the future we'll need to decide whether to
  580. # create a new socket or re-use an existing "shared" socket as a part
  581. # of the HTTP/2 handshake dance.
  582. if self._tunnel_host is not None and self._tunnel_port is not None:
  583. probe_http2_host = self._tunnel_host
  584. probe_http2_port = self._tunnel_port
  585. else:
  586. probe_http2_host = self.host
  587. probe_http2_port = self.port
  588. # Check if the target origin supports HTTP/2.
  589. # If the value comes back as 'None' it means that the current thread
  590. # is probing for HTTP/2 support. Otherwise, we're waiting for another
  591. # probe to complete, or we get a value right away.
  592. target_supports_http2: bool | None
  593. if "h2" in ssl_.ALPN_PROTOCOLS:
  594. target_supports_http2 = http2_probe.acquire_and_get(
  595. host=probe_http2_host, port=probe_http2_port
  596. )
  597. else:
  598. # If HTTP/2 isn't going to be offered it doesn't matter if
  599. # the target supports HTTP/2. Don't want to make a probe.
  600. target_supports_http2 = False
  601. if self._connect_callback is not None:
  602. self._connect_callback(
  603. "before connect",
  604. thread_id=threading.get_ident(),
  605. target_supports_http2=target_supports_http2,
  606. )
  607. try:
  608. sock: socket.socket | ssl.SSLSocket
  609. self.sock = sock = self._new_conn()
  610. server_hostname: str = self.host
  611. tls_in_tls = False
  612. # Do we need to establish a tunnel?
  613. if self.proxy_is_tunneling:
  614. # We're tunneling to an HTTPS origin so need to do TLS-in-TLS.
  615. if self._tunnel_scheme == "https":
  616. # _connect_tls_proxy will verify and assign proxy_is_verified
  617. self.sock = sock = self._connect_tls_proxy(self.host, sock)
  618. tls_in_tls = True
  619. elif self._tunnel_scheme == "http":
  620. self.proxy_is_verified = False
  621. # If we're tunneling it means we're connected to our proxy.
  622. self._has_connected_to_proxy = True
  623. self._tunnel()
  624. # Override the host with the one we're requesting data from.
  625. server_hostname = typing.cast(str, self._tunnel_host)
  626. if self.server_hostname is not None:
  627. server_hostname = self.server_hostname
  628. is_time_off = datetime.date.today() < RECENT_DATE
  629. if is_time_off:
  630. warnings.warn(
  631. (
  632. f"System time is way off (before {RECENT_DATE}). This will probably "
  633. "lead to SSL verification errors"
  634. ),
  635. SystemTimeWarning,
  636. )
  637. # Remove trailing '.' from fqdn hostnames to allow certificate validation
  638. server_hostname_rm_dot = server_hostname.rstrip(".")
  639. sock_and_verified = _ssl_wrap_socket_and_match_hostname(
  640. sock=sock,
  641. cert_reqs=self.cert_reqs,
  642. ssl_version=self.ssl_version,
  643. ssl_minimum_version=self.ssl_minimum_version,
  644. ssl_maximum_version=self.ssl_maximum_version,
  645. ca_certs=self.ca_certs,
  646. ca_cert_dir=self.ca_cert_dir,
  647. ca_cert_data=self.ca_cert_data,
  648. cert_file=self.cert_file,
  649. key_file=self.key_file,
  650. key_password=self.key_password,
  651. server_hostname=server_hostname_rm_dot,
  652. ssl_context=self.ssl_context,
  653. tls_in_tls=tls_in_tls,
  654. assert_hostname=self.assert_hostname,
  655. assert_fingerprint=self.assert_fingerprint,
  656. )
  657. self.sock = sock_and_verified.socket
  658. # If an error occurs during connection/handshake we may need to release
  659. # our lock so another connection can probe the origin.
  660. except BaseException:
  661. if self._connect_callback is not None:
  662. self._connect_callback(
  663. "after connect failure",
  664. thread_id=threading.get_ident(),
  665. target_supports_http2=target_supports_http2,
  666. )
  667. if target_supports_http2 is None:
  668. http2_probe.set_and_release(
  669. host=probe_http2_host, port=probe_http2_port, supports_http2=None
  670. )
  671. raise
  672. # If this connection doesn't know if the origin supports HTTP/2
  673. # we report back to the HTTP/2 probe our result.
  674. if target_supports_http2 is None:
  675. supports_http2 = sock_and_verified.socket.selected_alpn_protocol() == "h2"
  676. http2_probe.set_and_release(
  677. host=probe_http2_host,
  678. port=probe_http2_port,
  679. supports_http2=supports_http2,
  680. )
  681. # Forwarding proxies can never have a verified target since
  682. # the proxy is the one doing the verification. Should instead
  683. # use a CONNECT tunnel in order to verify the target.
  684. # See: https://github.com/urllib3/urllib3/issues/3267.
  685. if self.proxy_is_forwarding:
  686. self.is_verified = False
  687. else:
  688. self.is_verified = sock_and_verified.is_verified
  689. # If there's a proxy to be connected to we are fully connected.
  690. # This is set twice (once above and here) due to forwarding proxies
  691. # not using tunnelling.
  692. self._has_connected_to_proxy = bool(self.proxy)
  693. # Set `self.proxy_is_verified` unless it's already set while
  694. # establishing a tunnel.
  695. if self._has_connected_to_proxy and self.proxy_is_verified is None:
  696. self.proxy_is_verified = sock_and_verified.is_verified
  697. def _connect_tls_proxy(self, hostname: str, sock: socket.socket) -> ssl.SSLSocket:
  698. """
  699. Establish a TLS connection to the proxy using the provided SSL context.
  700. """
  701. # `_connect_tls_proxy` is called when self._tunnel_host is truthy.
  702. proxy_config = typing.cast(ProxyConfig, self.proxy_config)
  703. ssl_context = proxy_config.ssl_context
  704. sock_and_verified = _ssl_wrap_socket_and_match_hostname(
  705. sock,
  706. cert_reqs=self.cert_reqs,
  707. ssl_version=self.ssl_version,
  708. ssl_minimum_version=self.ssl_minimum_version,
  709. ssl_maximum_version=self.ssl_maximum_version,
  710. ca_certs=self.ca_certs,
  711. ca_cert_dir=self.ca_cert_dir,
  712. ca_cert_data=self.ca_cert_data,
  713. server_hostname=hostname,
  714. ssl_context=ssl_context,
  715. assert_hostname=proxy_config.assert_hostname,
  716. assert_fingerprint=proxy_config.assert_fingerprint,
  717. # Features that aren't implemented for proxies yet:
  718. cert_file=None,
  719. key_file=None,
  720. key_password=None,
  721. tls_in_tls=False,
  722. )
  723. self.proxy_is_verified = sock_and_verified.is_verified
  724. return sock_and_verified.socket # type: ignore[return-value]
  725. class _WrappedAndVerifiedSocket(typing.NamedTuple):
  726. """
  727. Wrapped socket and whether the connection is
  728. verified after the TLS handshake
  729. """
  730. socket: ssl.SSLSocket | SSLTransport
  731. is_verified: bool
  732. def _ssl_wrap_socket_and_match_hostname(
  733. sock: socket.socket,
  734. *,
  735. cert_reqs: None | str | int,
  736. ssl_version: None | str | int,
  737. ssl_minimum_version: int | None,
  738. ssl_maximum_version: int | None,
  739. cert_file: str | None,
  740. key_file: str | None,
  741. key_password: str | None,
  742. ca_certs: str | None,
  743. ca_cert_dir: str | None,
  744. ca_cert_data: None | str | bytes,
  745. assert_hostname: None | str | typing.Literal[False],
  746. assert_fingerprint: str | None,
  747. server_hostname: str | None,
  748. ssl_context: ssl.SSLContext | None,
  749. tls_in_tls: bool = False,
  750. ) -> _WrappedAndVerifiedSocket:
  751. """Logic for constructing an SSLContext from all TLS parameters, passing
  752. that down into ssl_wrap_socket, and then doing certificate verification
  753. either via hostname or fingerprint. This function exists to guarantee
  754. that both proxies and targets have the same behavior when connecting via TLS.
  755. """
  756. default_ssl_context = False
  757. if ssl_context is None:
  758. default_ssl_context = True
  759. context = create_urllib3_context(
  760. ssl_version=resolve_ssl_version(ssl_version),
  761. ssl_minimum_version=ssl_minimum_version,
  762. ssl_maximum_version=ssl_maximum_version,
  763. cert_reqs=resolve_cert_reqs(cert_reqs),
  764. )
  765. else:
  766. context = ssl_context
  767. context.verify_mode = resolve_cert_reqs(cert_reqs)
  768. # In some cases, we want to verify hostnames ourselves
  769. if (
  770. # `ssl` can't verify fingerprints or alternate hostnames
  771. assert_fingerprint
  772. or assert_hostname
  773. # assert_hostname can be set to False to disable hostname checking
  774. or assert_hostname is False
  775. # We still support OpenSSL 1.0.2, which prevents us from verifying
  776. # hostnames easily: https://github.com/pyca/pyopenssl/pull/933
  777. or ssl_.IS_PYOPENSSL
  778. or not ssl_.HAS_NEVER_CHECK_COMMON_NAME
  779. ):
  780. context.check_hostname = False
  781. # Try to load OS default certs if none are given. We need to do the hasattr() check
  782. # for custom pyOpenSSL SSLContext objects because they don't support
  783. # load_default_certs().
  784. if (
  785. not ca_certs
  786. and not ca_cert_dir
  787. and not ca_cert_data
  788. and default_ssl_context
  789. and hasattr(context, "load_default_certs")
  790. ):
  791. context.load_default_certs()
  792. # Ensure that IPv6 addresses are in the proper format and don't have a
  793. # scope ID. Python's SSL module fails to recognize scoped IPv6 addresses
  794. # and interprets them as DNS hostnames.
  795. if server_hostname is not None:
  796. normalized = server_hostname.strip("[]")
  797. if "%" in normalized:
  798. normalized = normalized[: normalized.rfind("%")]
  799. if is_ipaddress(normalized):
  800. server_hostname = normalized
  801. ssl_sock = ssl_wrap_socket(
  802. sock=sock,
  803. keyfile=key_file,
  804. certfile=cert_file,
  805. key_password=key_password,
  806. ca_certs=ca_certs,
  807. ca_cert_dir=ca_cert_dir,
  808. ca_cert_data=ca_cert_data,
  809. server_hostname=server_hostname,
  810. ssl_context=context,
  811. tls_in_tls=tls_in_tls,
  812. )
  813. try:
  814. if assert_fingerprint:
  815. _assert_fingerprint(
  816. ssl_sock.getpeercert(binary_form=True), assert_fingerprint
  817. )
  818. elif (
  819. context.verify_mode != ssl.CERT_NONE
  820. and not context.check_hostname
  821. and assert_hostname is not False
  822. ):
  823. cert: _TYPE_PEER_CERT_RET_DICT = ssl_sock.getpeercert() # type: ignore[assignment]
  824. # Need to signal to our match_hostname whether to use 'commonName' or not.
  825. # If we're using our own constructed SSLContext we explicitly set 'False'
  826. # because PyPy hard-codes 'True' from SSLContext.hostname_checks_common_name.
  827. if default_ssl_context:
  828. hostname_checks_common_name = False
  829. else:
  830. hostname_checks_common_name = (
  831. getattr(context, "hostname_checks_common_name", False) or False
  832. )
  833. _match_hostname(
  834. cert,
  835. assert_hostname or server_hostname, # type: ignore[arg-type]
  836. hostname_checks_common_name,
  837. )
  838. return _WrappedAndVerifiedSocket(
  839. socket=ssl_sock,
  840. is_verified=context.verify_mode == ssl.CERT_REQUIRED
  841. or bool(assert_fingerprint),
  842. )
  843. except BaseException:
  844. ssl_sock.close()
  845. raise
  846. def _match_hostname(
  847. cert: _TYPE_PEER_CERT_RET_DICT | None,
  848. asserted_hostname: str,
  849. hostname_checks_common_name: bool = False,
  850. ) -> None:
  851. # Our upstream implementation of ssl.match_hostname()
  852. # only applies this normalization to IP addresses so it doesn't
  853. # match DNS SANs so we do the same thing!
  854. stripped_hostname = asserted_hostname.strip("[]")
  855. if is_ipaddress(stripped_hostname):
  856. asserted_hostname = stripped_hostname
  857. try:
  858. match_hostname(cert, asserted_hostname, hostname_checks_common_name)
  859. except CertificateError as e:
  860. log.warning(
  861. "Certificate did not match expected hostname: %s. Certificate: %s",
  862. asserted_hostname,
  863. cert,
  864. )
  865. # Add cert to exception and reraise so client code can inspect
  866. # the cert when catching the exception, if they want to
  867. e._peer_cert = cert # type: ignore[attr-defined]
  868. raise
  869. def _wrap_proxy_error(err: Exception, proxy_scheme: str | None) -> ProxyError:
  870. # Look for the phrase 'wrong version number', if found
  871. # then we should warn the user that we're very sure that
  872. # this proxy is HTTP-only and they have a configuration issue.
  873. error_normalized = " ".join(re.split("[^a-z]", str(err).lower()))
  874. is_likely_http_proxy = (
  875. "wrong version number" in error_normalized
  876. or "unknown protocol" in error_normalized
  877. or "record layer failure" in error_normalized
  878. )
  879. http_proxy_warning = (
  880. ". Your proxy appears to only use HTTP and not HTTPS, "
  881. "try changing your proxy URL to be HTTP. See: "
  882. "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
  883. "#https-proxy-error-http-proxy"
  884. )
  885. new_err = ProxyError(
  886. f"Unable to connect to proxy"
  887. f"{http_proxy_warning if is_likely_http_proxy and proxy_scheme == 'https' else ''}",
  888. err,
  889. )
  890. new_err.__cause__ = err
  891. return new_err
  892. def _get_default_user_agent() -> str:
  893. return f"python-urllib3/{__version__}"
  894. class DummyConnection:
  895. """Used to detect a failed ConnectionCls import."""
  896. if not ssl:
  897. HTTPSConnection = DummyConnection # type: ignore[misc, assignment] # noqa: F811
  898. VerifiedHTTPSConnection = HTTPSConnection
  899. def _url_from_connection(
  900. conn: HTTPConnection | HTTPSConnection, path: str | None = None
  901. ) -> str:
  902. """Returns the URL from a given connection. This is mainly used for testing and logging."""
  903. scheme = "https" if isinstance(conn, HTTPSConnection) else "http"
  904. return Url(scheme=scheme, host=conn.host, port=conn.port, path=path).url