helpers.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958
  1. """Various helper functions"""
  2. import asyncio
  3. import base64
  4. import binascii
  5. import contextlib
  6. import datetime
  7. import enum
  8. import functools
  9. import inspect
  10. import netrc
  11. import os
  12. import platform
  13. import re
  14. import sys
  15. import time
  16. import weakref
  17. from collections import namedtuple
  18. from contextlib import suppress
  19. from email.parser import HeaderParser
  20. from email.utils import parsedate
  21. from math import ceil
  22. from pathlib import Path
  23. from types import MappingProxyType, TracebackType
  24. from typing import (
  25. Any,
  26. Callable,
  27. ContextManager,
  28. Dict,
  29. Generator,
  30. Generic,
  31. Iterable,
  32. Iterator,
  33. List,
  34. Mapping,
  35. Optional,
  36. Protocol,
  37. Tuple,
  38. Type,
  39. TypeVar,
  40. Union,
  41. get_args,
  42. overload,
  43. )
  44. from urllib.parse import quote
  45. from urllib.request import getproxies, proxy_bypass
  46. import attr
  47. from multidict import MultiDict, MultiDictProxy, MultiMapping
  48. from propcache.api import under_cached_property as reify
  49. from yarl import URL
  50. from . import hdrs
  51. from .log import client_logger
  52. if sys.version_info >= (3, 11):
  53. import asyncio as async_timeout
  54. else:
  55. import async_timeout
  56. __all__ = ("BasicAuth", "ChainMapProxy", "ETag", "reify")
  57. IS_MACOS = platform.system() == "Darwin"
  58. IS_WINDOWS = platform.system() == "Windows"
  59. PY_310 = sys.version_info >= (3, 10)
  60. PY_311 = sys.version_info >= (3, 11)
  61. _T = TypeVar("_T")
  62. _S = TypeVar("_S")
  63. _SENTINEL = enum.Enum("_SENTINEL", "sentinel")
  64. sentinel = _SENTINEL.sentinel
  65. NO_EXTENSIONS = bool(os.environ.get("AIOHTTP_NO_EXTENSIONS"))
  66. # https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.1
  67. EMPTY_BODY_STATUS_CODES = frozenset((204, 304, *range(100, 200)))
  68. # https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.1
  69. # https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.2
  70. EMPTY_BODY_METHODS = hdrs.METH_HEAD_ALL
  71. DEBUG = sys.flags.dev_mode or (
  72. not sys.flags.ignore_environment and bool(os.environ.get("PYTHONASYNCIODEBUG"))
  73. )
  74. CHAR = {chr(i) for i in range(0, 128)}
  75. CTL = {chr(i) for i in range(0, 32)} | {
  76. chr(127),
  77. }
  78. SEPARATORS = {
  79. "(",
  80. ")",
  81. "<",
  82. ">",
  83. "@",
  84. ",",
  85. ";",
  86. ":",
  87. "\\",
  88. '"',
  89. "/",
  90. "[",
  91. "]",
  92. "?",
  93. "=",
  94. "{",
  95. "}",
  96. " ",
  97. chr(9),
  98. }
  99. TOKEN = CHAR ^ CTL ^ SEPARATORS
  100. class noop:
  101. def __await__(self) -> Generator[None, None, None]:
  102. yield
  103. class BasicAuth(namedtuple("BasicAuth", ["login", "password", "encoding"])):
  104. """Http basic authentication helper."""
  105. def __new__(
  106. cls, login: str, password: str = "", encoding: str = "latin1"
  107. ) -> "BasicAuth":
  108. if login is None:
  109. raise ValueError("None is not allowed as login value")
  110. if password is None:
  111. raise ValueError("None is not allowed as password value")
  112. if ":" in login:
  113. raise ValueError('A ":" is not allowed in login (RFC 1945#section-11.1)')
  114. return super().__new__(cls, login, password, encoding)
  115. @classmethod
  116. def decode(cls, auth_header: str, encoding: str = "latin1") -> "BasicAuth":
  117. """Create a BasicAuth object from an Authorization HTTP header."""
  118. try:
  119. auth_type, encoded_credentials = auth_header.split(" ", 1)
  120. except ValueError:
  121. raise ValueError("Could not parse authorization header.")
  122. if auth_type.lower() != "basic":
  123. raise ValueError("Unknown authorization method %s" % auth_type)
  124. try:
  125. decoded = base64.b64decode(
  126. encoded_credentials.encode("ascii"), validate=True
  127. ).decode(encoding)
  128. except binascii.Error:
  129. raise ValueError("Invalid base64 encoding.")
  130. try:
  131. # RFC 2617 HTTP Authentication
  132. # https://www.ietf.org/rfc/rfc2617.txt
  133. # the colon must be present, but the username and password may be
  134. # otherwise blank.
  135. username, password = decoded.split(":", 1)
  136. except ValueError:
  137. raise ValueError("Invalid credentials.")
  138. return cls(username, password, encoding=encoding)
  139. @classmethod
  140. def from_url(cls, url: URL, *, encoding: str = "latin1") -> Optional["BasicAuth"]:
  141. """Create BasicAuth from url."""
  142. if not isinstance(url, URL):
  143. raise TypeError("url should be yarl.URL instance")
  144. # Check raw_user and raw_password first as yarl is likely
  145. # to already have these values parsed from the netloc in the cache.
  146. if url.raw_user is None and url.raw_password is None:
  147. return None
  148. return cls(url.user or "", url.password or "", encoding=encoding)
  149. def encode(self) -> str:
  150. """Encode credentials."""
  151. creds = (f"{self.login}:{self.password}").encode(self.encoding)
  152. return "Basic %s" % base64.b64encode(creds).decode(self.encoding)
  153. def strip_auth_from_url(url: URL) -> Tuple[URL, Optional[BasicAuth]]:
  154. """Remove user and password from URL if present and return BasicAuth object."""
  155. # Check raw_user and raw_password first as yarl is likely
  156. # to already have these values parsed from the netloc in the cache.
  157. if url.raw_user is None and url.raw_password is None:
  158. return url, None
  159. return url.with_user(None), BasicAuth(url.user or "", url.password or "")
  160. def netrc_from_env() -> Optional[netrc.netrc]:
  161. """Load netrc from file.
  162. Attempt to load it from the path specified by the env-var
  163. NETRC or in the default location in the user's home directory.
  164. Returns None if it couldn't be found or fails to parse.
  165. """
  166. netrc_env = os.environ.get("NETRC")
  167. if netrc_env is not None:
  168. netrc_path = Path(netrc_env)
  169. else:
  170. try:
  171. home_dir = Path.home()
  172. except RuntimeError as e: # pragma: no cover
  173. # if pathlib can't resolve home, it may raise a RuntimeError
  174. client_logger.debug(
  175. "Could not resolve home directory when "
  176. "trying to look for .netrc file: %s",
  177. e,
  178. )
  179. return None
  180. netrc_path = home_dir / ("_netrc" if IS_WINDOWS else ".netrc")
  181. try:
  182. return netrc.netrc(str(netrc_path))
  183. except netrc.NetrcParseError as e:
  184. client_logger.warning("Could not parse .netrc file: %s", e)
  185. except OSError as e:
  186. netrc_exists = False
  187. with contextlib.suppress(OSError):
  188. netrc_exists = netrc_path.is_file()
  189. # we couldn't read the file (doesn't exist, permissions, etc.)
  190. if netrc_env or netrc_exists:
  191. # only warn if the environment wanted us to load it,
  192. # or it appears like the default file does actually exist
  193. client_logger.warning("Could not read .netrc file: %s", e)
  194. return None
  195. @attr.s(auto_attribs=True, frozen=True, slots=True)
  196. class ProxyInfo:
  197. proxy: URL
  198. proxy_auth: Optional[BasicAuth]
  199. def basicauth_from_netrc(netrc_obj: Optional[netrc.netrc], host: str) -> BasicAuth:
  200. """
  201. Return :py:class:`~aiohttp.BasicAuth` credentials for ``host`` from ``netrc_obj``.
  202. :raises LookupError: if ``netrc_obj`` is :py:data:`None` or if no
  203. entry is found for the ``host``.
  204. """
  205. if netrc_obj is None:
  206. raise LookupError("No .netrc file found")
  207. auth_from_netrc = netrc_obj.authenticators(host)
  208. if auth_from_netrc is None:
  209. raise LookupError(f"No entry for {host!s} found in the `.netrc` file.")
  210. login, account, password = auth_from_netrc
  211. # TODO(PY311): username = login or account
  212. # Up to python 3.10, account could be None if not specified,
  213. # and login will be empty string if not specified. From 3.11,
  214. # login and account will be empty string if not specified.
  215. username = login if (login or account is None) else account
  216. # TODO(PY311): Remove this, as password will be empty string
  217. # if not specified
  218. if password is None:
  219. password = ""
  220. return BasicAuth(username, password)
  221. def proxies_from_env() -> Dict[str, ProxyInfo]:
  222. proxy_urls = {
  223. k: URL(v)
  224. for k, v in getproxies().items()
  225. if k in ("http", "https", "ws", "wss")
  226. }
  227. netrc_obj = netrc_from_env()
  228. stripped = {k: strip_auth_from_url(v) for k, v in proxy_urls.items()}
  229. ret = {}
  230. for proto, val in stripped.items():
  231. proxy, auth = val
  232. if proxy.scheme in ("https", "wss"):
  233. client_logger.warning(
  234. "%s proxies %s are not supported, ignoring", proxy.scheme.upper(), proxy
  235. )
  236. continue
  237. if netrc_obj and auth is None:
  238. if proxy.host is not None:
  239. try:
  240. auth = basicauth_from_netrc(netrc_obj, proxy.host)
  241. except LookupError:
  242. auth = None
  243. ret[proto] = ProxyInfo(proxy, auth)
  244. return ret
  245. def get_env_proxy_for_url(url: URL) -> Tuple[URL, Optional[BasicAuth]]:
  246. """Get a permitted proxy for the given URL from the env."""
  247. if url.host is not None and proxy_bypass(url.host):
  248. raise LookupError(f"Proxying is disallowed for `{url.host!r}`")
  249. proxies_in_env = proxies_from_env()
  250. try:
  251. proxy_info = proxies_in_env[url.scheme]
  252. except KeyError:
  253. raise LookupError(f"No proxies found for `{url!s}` in the env")
  254. else:
  255. return proxy_info.proxy, proxy_info.proxy_auth
  256. @attr.s(auto_attribs=True, frozen=True, slots=True)
  257. class MimeType:
  258. type: str
  259. subtype: str
  260. suffix: str
  261. parameters: "MultiDictProxy[str]"
  262. @functools.lru_cache(maxsize=56)
  263. def parse_mimetype(mimetype: str) -> MimeType:
  264. """Parses a MIME type into its components.
  265. mimetype is a MIME type string.
  266. Returns a MimeType object.
  267. Example:
  268. >>> parse_mimetype('text/html; charset=utf-8')
  269. MimeType(type='text', subtype='html', suffix='',
  270. parameters={'charset': 'utf-8'})
  271. """
  272. if not mimetype:
  273. return MimeType(
  274. type="", subtype="", suffix="", parameters=MultiDictProxy(MultiDict())
  275. )
  276. parts = mimetype.split(";")
  277. params: MultiDict[str] = MultiDict()
  278. for item in parts[1:]:
  279. if not item:
  280. continue
  281. key, _, value = item.partition("=")
  282. params.add(key.lower().strip(), value.strip(' "'))
  283. fulltype = parts[0].strip().lower()
  284. if fulltype == "*":
  285. fulltype = "*/*"
  286. mtype, _, stype = fulltype.partition("/")
  287. stype, _, suffix = stype.partition("+")
  288. return MimeType(
  289. type=mtype, subtype=stype, suffix=suffix, parameters=MultiDictProxy(params)
  290. )
  291. @functools.lru_cache(maxsize=56)
  292. def parse_content_type(raw: str) -> Tuple[str, MappingProxyType[str, str]]:
  293. """Parse Content-Type header.
  294. Returns a tuple of the parsed content type and a
  295. MappingProxyType of parameters.
  296. """
  297. msg = HeaderParser().parsestr(f"Content-Type: {raw}")
  298. content_type = msg.get_content_type()
  299. params = msg.get_params(())
  300. content_dict = dict(params[1:]) # First element is content type again
  301. return content_type, MappingProxyType(content_dict)
  302. def guess_filename(obj: Any, default: Optional[str] = None) -> Optional[str]:
  303. name = getattr(obj, "name", None)
  304. if name and isinstance(name, str) and name[0] != "<" and name[-1] != ">":
  305. return Path(name).name
  306. return default
  307. not_qtext_re = re.compile(r"[^\041\043-\133\135-\176]")
  308. QCONTENT = {chr(i) for i in range(0x20, 0x7F)} | {"\t"}
  309. def quoted_string(content: str) -> str:
  310. """Return 7-bit content as quoted-string.
  311. Format content into a quoted-string as defined in RFC5322 for
  312. Internet Message Format. Notice that this is not the 8-bit HTTP
  313. format, but the 7-bit email format. Content must be in usascii or
  314. a ValueError is raised.
  315. """
  316. if not (QCONTENT > set(content)):
  317. raise ValueError(f"bad content for quoted-string {content!r}")
  318. return not_qtext_re.sub(lambda x: "\\" + x.group(0), content)
  319. def content_disposition_header(
  320. disptype: str, quote_fields: bool = True, _charset: str = "utf-8", **params: str
  321. ) -> str:
  322. """Sets ``Content-Disposition`` header for MIME.
  323. This is the MIME payload Content-Disposition header from RFC 2183
  324. and RFC 7579 section 4.2, not the HTTP Content-Disposition from
  325. RFC 6266.
  326. disptype is a disposition type: inline, attachment, form-data.
  327. Should be valid extension token (see RFC 2183)
  328. quote_fields performs value quoting to 7-bit MIME headers
  329. according to RFC 7578. Set to quote_fields to False if recipient
  330. can take 8-bit file names and field values.
  331. _charset specifies the charset to use when quote_fields is True.
  332. params is a dict with disposition params.
  333. """
  334. if not disptype or not (TOKEN > set(disptype)):
  335. raise ValueError(f"bad content disposition type {disptype!r}")
  336. value = disptype
  337. if params:
  338. lparams = []
  339. for key, val in params.items():
  340. if not key or not (TOKEN > set(key)):
  341. raise ValueError(f"bad content disposition parameter {key!r}={val!r}")
  342. if quote_fields:
  343. if key.lower() == "filename":
  344. qval = quote(val, "", encoding=_charset)
  345. lparams.append((key, '"%s"' % qval))
  346. else:
  347. try:
  348. qval = quoted_string(val)
  349. except ValueError:
  350. qval = "".join(
  351. (_charset, "''", quote(val, "", encoding=_charset))
  352. )
  353. lparams.append((key + "*", qval))
  354. else:
  355. lparams.append((key, '"%s"' % qval))
  356. else:
  357. qval = val.replace("\\", "\\\\").replace('"', '\\"')
  358. lparams.append((key, '"%s"' % qval))
  359. sparams = "; ".join("=".join(pair) for pair in lparams)
  360. value = "; ".join((value, sparams))
  361. return value
  362. def is_ip_address(host: Optional[str]) -> bool:
  363. """Check if host looks like an IP Address.
  364. This check is only meant as a heuristic to ensure that
  365. a host is not a domain name.
  366. """
  367. if not host:
  368. return False
  369. # For a host to be an ipv4 address, it must be all numeric.
  370. # The host must contain a colon to be an IPv6 address.
  371. return ":" in host or host.replace(".", "").isdigit()
  372. _cached_current_datetime: Optional[int] = None
  373. _cached_formatted_datetime = ""
  374. def rfc822_formatted_time() -> str:
  375. global _cached_current_datetime
  376. global _cached_formatted_datetime
  377. now = int(time.time())
  378. if now != _cached_current_datetime:
  379. # Weekday and month names for HTTP date/time formatting;
  380. # always English!
  381. # Tuples are constants stored in codeobject!
  382. _weekdayname = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")
  383. _monthname = (
  384. "", # Dummy so we can use 1-based month numbers
  385. "Jan",
  386. "Feb",
  387. "Mar",
  388. "Apr",
  389. "May",
  390. "Jun",
  391. "Jul",
  392. "Aug",
  393. "Sep",
  394. "Oct",
  395. "Nov",
  396. "Dec",
  397. )
  398. year, month, day, hh, mm, ss, wd, *tail = time.gmtime(now)
  399. _cached_formatted_datetime = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % (
  400. _weekdayname[wd],
  401. day,
  402. _monthname[month],
  403. year,
  404. hh,
  405. mm,
  406. ss,
  407. )
  408. _cached_current_datetime = now
  409. return _cached_formatted_datetime
  410. def _weakref_handle(info: "Tuple[weakref.ref[object], str]") -> None:
  411. ref, name = info
  412. ob = ref()
  413. if ob is not None:
  414. with suppress(Exception):
  415. getattr(ob, name)()
  416. def weakref_handle(
  417. ob: object,
  418. name: str,
  419. timeout: float,
  420. loop: asyncio.AbstractEventLoop,
  421. timeout_ceil_threshold: float = 5,
  422. ) -> Optional[asyncio.TimerHandle]:
  423. if timeout is not None and timeout > 0:
  424. when = loop.time() + timeout
  425. if timeout >= timeout_ceil_threshold:
  426. when = ceil(when)
  427. return loop.call_at(when, _weakref_handle, (weakref.ref(ob), name))
  428. return None
  429. def call_later(
  430. cb: Callable[[], Any],
  431. timeout: float,
  432. loop: asyncio.AbstractEventLoop,
  433. timeout_ceil_threshold: float = 5,
  434. ) -> Optional[asyncio.TimerHandle]:
  435. if timeout is None or timeout <= 0:
  436. return None
  437. now = loop.time()
  438. when = calculate_timeout_when(now, timeout, timeout_ceil_threshold)
  439. return loop.call_at(when, cb)
  440. def calculate_timeout_when(
  441. loop_time: float,
  442. timeout: float,
  443. timeout_ceiling_threshold: float,
  444. ) -> float:
  445. """Calculate when to execute a timeout."""
  446. when = loop_time + timeout
  447. if timeout > timeout_ceiling_threshold:
  448. return ceil(when)
  449. return when
  450. class TimeoutHandle:
  451. """Timeout handle"""
  452. __slots__ = ("_timeout", "_loop", "_ceil_threshold", "_callbacks")
  453. def __init__(
  454. self,
  455. loop: asyncio.AbstractEventLoop,
  456. timeout: Optional[float],
  457. ceil_threshold: float = 5,
  458. ) -> None:
  459. self._timeout = timeout
  460. self._loop = loop
  461. self._ceil_threshold = ceil_threshold
  462. self._callbacks: List[
  463. Tuple[Callable[..., None], Tuple[Any, ...], Dict[str, Any]]
  464. ] = []
  465. def register(
  466. self, callback: Callable[..., None], *args: Any, **kwargs: Any
  467. ) -> None:
  468. self._callbacks.append((callback, args, kwargs))
  469. def close(self) -> None:
  470. self._callbacks.clear()
  471. def start(self) -> Optional[asyncio.TimerHandle]:
  472. timeout = self._timeout
  473. if timeout is not None and timeout > 0:
  474. when = self._loop.time() + timeout
  475. if timeout >= self._ceil_threshold:
  476. when = ceil(when)
  477. return self._loop.call_at(when, self.__call__)
  478. else:
  479. return None
  480. def timer(self) -> "BaseTimerContext":
  481. if self._timeout is not None and self._timeout > 0:
  482. timer = TimerContext(self._loop)
  483. self.register(timer.timeout)
  484. return timer
  485. else:
  486. return TimerNoop()
  487. def __call__(self) -> None:
  488. for cb, args, kwargs in self._callbacks:
  489. with suppress(Exception):
  490. cb(*args, **kwargs)
  491. self._callbacks.clear()
  492. class BaseTimerContext(ContextManager["BaseTimerContext"]):
  493. __slots__ = ()
  494. def assert_timeout(self) -> None:
  495. """Raise TimeoutError if timeout has been exceeded."""
  496. class TimerNoop(BaseTimerContext):
  497. __slots__ = ()
  498. def __enter__(self) -> BaseTimerContext:
  499. return self
  500. def __exit__(
  501. self,
  502. exc_type: Optional[Type[BaseException]],
  503. exc_val: Optional[BaseException],
  504. exc_tb: Optional[TracebackType],
  505. ) -> None:
  506. return
  507. class TimerContext(BaseTimerContext):
  508. """Low resolution timeout context manager"""
  509. __slots__ = ("_loop", "_tasks", "_cancelled", "_cancelling")
  510. def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
  511. self._loop = loop
  512. self._tasks: List[asyncio.Task[Any]] = []
  513. self._cancelled = False
  514. self._cancelling = 0
  515. def assert_timeout(self) -> None:
  516. """Raise TimeoutError if timer has already been cancelled."""
  517. if self._cancelled:
  518. raise asyncio.TimeoutError from None
  519. def __enter__(self) -> BaseTimerContext:
  520. task = asyncio.current_task(loop=self._loop)
  521. if task is None:
  522. raise RuntimeError("Timeout context manager should be used inside a task")
  523. if sys.version_info >= (3, 11):
  524. # Remember if the task was already cancelling
  525. # so when we __exit__ we can decide if we should
  526. # raise asyncio.TimeoutError or let the cancellation propagate
  527. self._cancelling = task.cancelling()
  528. if self._cancelled:
  529. raise asyncio.TimeoutError from None
  530. self._tasks.append(task)
  531. return self
  532. def __exit__(
  533. self,
  534. exc_type: Optional[Type[BaseException]],
  535. exc_val: Optional[BaseException],
  536. exc_tb: Optional[TracebackType],
  537. ) -> Optional[bool]:
  538. enter_task: Optional[asyncio.Task[Any]] = None
  539. if self._tasks:
  540. enter_task = self._tasks.pop()
  541. if exc_type is asyncio.CancelledError and self._cancelled:
  542. assert enter_task is not None
  543. # The timeout was hit, and the task was cancelled
  544. # so we need to uncancel the last task that entered the context manager
  545. # since the cancellation should not leak out of the context manager
  546. if sys.version_info >= (3, 11):
  547. # If the task was already cancelling don't raise
  548. # asyncio.TimeoutError and instead return None
  549. # to allow the cancellation to propagate
  550. if enter_task.uncancel() > self._cancelling:
  551. return None
  552. raise asyncio.TimeoutError from exc_val
  553. return None
  554. def timeout(self) -> None:
  555. if not self._cancelled:
  556. for task in set(self._tasks):
  557. task.cancel()
  558. self._cancelled = True
  559. def ceil_timeout(
  560. delay: Optional[float], ceil_threshold: float = 5
  561. ) -> async_timeout.Timeout:
  562. if delay is None or delay <= 0:
  563. return async_timeout.timeout(None)
  564. loop = asyncio.get_running_loop()
  565. now = loop.time()
  566. when = now + delay
  567. if delay > ceil_threshold:
  568. when = ceil(when)
  569. return async_timeout.timeout_at(when)
  570. class HeadersMixin:
  571. """Mixin for handling headers."""
  572. ATTRS = frozenset(["_content_type", "_content_dict", "_stored_content_type"])
  573. _headers: MultiMapping[str]
  574. _content_type: Optional[str] = None
  575. _content_dict: Optional[Dict[str, str]] = None
  576. _stored_content_type: Union[str, None, _SENTINEL] = sentinel
  577. def _parse_content_type(self, raw: Optional[str]) -> None:
  578. self._stored_content_type = raw
  579. if raw is None:
  580. # default value according to RFC 2616
  581. self._content_type = "application/octet-stream"
  582. self._content_dict = {}
  583. else:
  584. content_type, content_mapping_proxy = parse_content_type(raw)
  585. self._content_type = content_type
  586. # _content_dict needs to be mutable so we can update it
  587. self._content_dict = content_mapping_proxy.copy()
  588. @property
  589. def content_type(self) -> str:
  590. """The value of content part for Content-Type HTTP header."""
  591. raw = self._headers.get(hdrs.CONTENT_TYPE)
  592. if self._stored_content_type != raw:
  593. self._parse_content_type(raw)
  594. assert self._content_type is not None
  595. return self._content_type
  596. @property
  597. def charset(self) -> Optional[str]:
  598. """The value of charset part for Content-Type HTTP header."""
  599. raw = self._headers.get(hdrs.CONTENT_TYPE)
  600. if self._stored_content_type != raw:
  601. self._parse_content_type(raw)
  602. assert self._content_dict is not None
  603. return self._content_dict.get("charset")
  604. @property
  605. def content_length(self) -> Optional[int]:
  606. """The value of Content-Length HTTP header."""
  607. content_length = self._headers.get(hdrs.CONTENT_LENGTH)
  608. return None if content_length is None else int(content_length)
  609. def set_result(fut: "asyncio.Future[_T]", result: _T) -> None:
  610. if not fut.done():
  611. fut.set_result(result)
  612. _EXC_SENTINEL = BaseException()
  613. class ErrorableProtocol(Protocol):
  614. def set_exception(
  615. self,
  616. exc: BaseException,
  617. exc_cause: BaseException = ...,
  618. ) -> None: ... # pragma: no cover
  619. def set_exception(
  620. fut: "asyncio.Future[_T] | ErrorableProtocol",
  621. exc: BaseException,
  622. exc_cause: BaseException = _EXC_SENTINEL,
  623. ) -> None:
  624. """Set future exception.
  625. If the future is marked as complete, this function is a no-op.
  626. :param exc_cause: An exception that is a direct cause of ``exc``.
  627. Only set if provided.
  628. """
  629. if asyncio.isfuture(fut) and fut.done():
  630. return
  631. exc_is_sentinel = exc_cause is _EXC_SENTINEL
  632. exc_causes_itself = exc is exc_cause
  633. if not exc_is_sentinel and not exc_causes_itself:
  634. exc.__cause__ = exc_cause
  635. fut.set_exception(exc)
  636. @functools.total_ordering
  637. class AppKey(Generic[_T]):
  638. """Keys for static typing support in Application."""
  639. __slots__ = ("_name", "_t", "__orig_class__")
  640. # This may be set by Python when instantiating with a generic type. We need to
  641. # support this, in order to support types that are not concrete classes,
  642. # like Iterable, which can't be passed as the second parameter to __init__.
  643. __orig_class__: Type[object]
  644. def __init__(self, name: str, t: Optional[Type[_T]] = None):
  645. # Prefix with module name to help deduplicate key names.
  646. frame = inspect.currentframe()
  647. while frame:
  648. if frame.f_code.co_name == "<module>":
  649. module: str = frame.f_globals["__name__"]
  650. break
  651. frame = frame.f_back
  652. self._name = module + "." + name
  653. self._t = t
  654. def __lt__(self, other: object) -> bool:
  655. if isinstance(other, AppKey):
  656. return self._name < other._name
  657. return True # Order AppKey above other types.
  658. def __repr__(self) -> str:
  659. t = self._t
  660. if t is None:
  661. with suppress(AttributeError):
  662. # Set to type arg.
  663. t = get_args(self.__orig_class__)[0]
  664. if t is None:
  665. t_repr = "<<Unknown>>"
  666. elif isinstance(t, type):
  667. if t.__module__ == "builtins":
  668. t_repr = t.__qualname__
  669. else:
  670. t_repr = f"{t.__module__}.{t.__qualname__}"
  671. else:
  672. t_repr = repr(t)
  673. return f"<AppKey({self._name}, type={t_repr})>"
  674. class ChainMapProxy(Mapping[Union[str, AppKey[Any]], Any]):
  675. __slots__ = ("_maps",)
  676. def __init__(self, maps: Iterable[Mapping[Union[str, AppKey[Any]], Any]]) -> None:
  677. self._maps = tuple(maps)
  678. def __init_subclass__(cls) -> None:
  679. raise TypeError(
  680. "Inheritance class {} from ChainMapProxy "
  681. "is forbidden".format(cls.__name__)
  682. )
  683. @overload # type: ignore[override]
  684. def __getitem__(self, key: AppKey[_T]) -> _T: ...
  685. @overload
  686. def __getitem__(self, key: str) -> Any: ...
  687. def __getitem__(self, key: Union[str, AppKey[_T]]) -> Any:
  688. for mapping in self._maps:
  689. try:
  690. return mapping[key]
  691. except KeyError:
  692. pass
  693. raise KeyError(key)
  694. @overload # type: ignore[override]
  695. def get(self, key: AppKey[_T], default: _S) -> Union[_T, _S]: ...
  696. @overload
  697. def get(self, key: AppKey[_T], default: None = ...) -> Optional[_T]: ...
  698. @overload
  699. def get(self, key: str, default: Any = ...) -> Any: ...
  700. def get(self, key: Union[str, AppKey[_T]], default: Any = None) -> Any:
  701. try:
  702. return self[key]
  703. except KeyError:
  704. return default
  705. def __len__(self) -> int:
  706. # reuses stored hash values if possible
  707. return len(set().union(*self._maps))
  708. def __iter__(self) -> Iterator[Union[str, AppKey[Any]]]:
  709. d: Dict[Union[str, AppKey[Any]], Any] = {}
  710. for mapping in reversed(self._maps):
  711. # reuses stored hash values if possible
  712. d.update(mapping)
  713. return iter(d)
  714. def __contains__(self, key: object) -> bool:
  715. return any(key in m for m in self._maps)
  716. def __bool__(self) -> bool:
  717. return any(self._maps)
  718. def __repr__(self) -> str:
  719. content = ", ".join(map(repr, self._maps))
  720. return f"ChainMapProxy({content})"
  721. # https://tools.ietf.org/html/rfc7232#section-2.3
  722. _ETAGC = r"[!\x23-\x7E\x80-\xff]+"
  723. _ETAGC_RE = re.compile(_ETAGC)
  724. _QUOTED_ETAG = rf'(W/)?"({_ETAGC})"'
  725. QUOTED_ETAG_RE = re.compile(_QUOTED_ETAG)
  726. LIST_QUOTED_ETAG_RE = re.compile(rf"({_QUOTED_ETAG})(?:\s*,\s*|$)|(.)")
  727. ETAG_ANY = "*"
  728. @attr.s(auto_attribs=True, frozen=True, slots=True)
  729. class ETag:
  730. value: str
  731. is_weak: bool = False
  732. def validate_etag_value(value: str) -> None:
  733. if value != ETAG_ANY and not _ETAGC_RE.fullmatch(value):
  734. raise ValueError(
  735. f"Value {value!r} is not a valid etag. Maybe it contains '\"'?"
  736. )
  737. def parse_http_date(date_str: Optional[str]) -> Optional[datetime.datetime]:
  738. """Process a date string, return a datetime object"""
  739. if date_str is not None:
  740. timetuple = parsedate(date_str)
  741. if timetuple is not None:
  742. with suppress(ValueError):
  743. return datetime.datetime(*timetuple[:6], tzinfo=datetime.timezone.utc)
  744. return None
  745. @functools.lru_cache
  746. def must_be_empty_body(method: str, code: int) -> bool:
  747. """Check if a request must return an empty body."""
  748. return (
  749. code in EMPTY_BODY_STATUS_CODES
  750. or method in EMPTY_BODY_METHODS
  751. or (200 <= code < 300 and method in hdrs.METH_CONNECT_ALL)
  752. )
  753. def should_remove_content_length(method: str, code: int) -> bool:
  754. """Check if a Content-Length header should be removed.
  755. This should always be a subset of must_be_empty_body
  756. """
  757. # https://www.rfc-editor.org/rfc/rfc9110.html#section-8.6-8
  758. # https://www.rfc-editor.org/rfc/rfc9110.html#section-15.4.5-4
  759. return code in EMPTY_BODY_STATUS_CODES or (
  760. 200 <= code < 300 and method in hdrs.METH_CONNECT_ALL
  761. )