dump.py 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198
  1. """This module provides functions for dumping information about responses."""
  2. import collections
  3. from requests import compat
  4. __all__ = ('dump_response', 'dump_all')
  5. HTTP_VERSIONS = {
  6. 9: b'0.9',
  7. 10: b'1.0',
  8. 11: b'1.1',
  9. }
  10. _PrefixSettings = collections.namedtuple('PrefixSettings',
  11. ['request', 'response'])
  12. class PrefixSettings(_PrefixSettings):
  13. def __new__(cls, request, response):
  14. request = _coerce_to_bytes(request)
  15. response = _coerce_to_bytes(response)
  16. return super(PrefixSettings, cls).__new__(cls, request, response)
  17. def _get_proxy_information(response):
  18. if getattr(response.connection, 'proxy_manager', False):
  19. proxy_info = {}
  20. request_url = response.request.url
  21. if request_url.startswith('https://'):
  22. proxy_info['method'] = 'CONNECT'
  23. proxy_info['request_path'] = request_url
  24. return proxy_info
  25. return None
  26. def _format_header(name, value):
  27. return (_coerce_to_bytes(name) + b': ' + _coerce_to_bytes(value) +
  28. b'\r\n')
  29. def _build_request_path(url, proxy_info):
  30. uri = compat.urlparse(url)
  31. proxy_url = proxy_info.get('request_path')
  32. if proxy_url is not None:
  33. request_path = _coerce_to_bytes(proxy_url)
  34. return request_path, uri
  35. request_path = _coerce_to_bytes(uri.path)
  36. if uri.query:
  37. request_path += b'?' + _coerce_to_bytes(uri.query)
  38. return request_path, uri
  39. def _dump_request_data(request, prefixes, bytearr, proxy_info=None):
  40. if proxy_info is None:
  41. proxy_info = {}
  42. prefix = prefixes.request
  43. method = _coerce_to_bytes(proxy_info.pop('method', request.method))
  44. request_path, uri = _build_request_path(request.url, proxy_info)
  45. # <prefix><METHOD> <request-path> HTTP/1.1
  46. bytearr.extend(prefix + method + b' ' + request_path + b' HTTP/1.1\r\n')
  47. # <prefix>Host: <request-host> OR host header specified by user
  48. headers = request.headers.copy()
  49. host_header = _coerce_to_bytes(headers.pop('Host', uri.netloc))
  50. bytearr.extend(prefix + b'Host: ' + host_header + b'\r\n')
  51. for name, value in headers.items():
  52. bytearr.extend(prefix + _format_header(name, value))
  53. bytearr.extend(prefix + b'\r\n')
  54. if request.body:
  55. if isinstance(request.body, compat.basestring):
  56. bytearr.extend(prefix + _coerce_to_bytes(request.body))
  57. else:
  58. # In the event that the body is a file-like object, let's not try
  59. # to read everything into memory.
  60. bytearr.extend(b'<< Request body is not a string-like type >>')
  61. bytearr.extend(b'\r\n')
  62. bytearr.extend(b'\r\n')
  63. def _dump_response_data(response, prefixes, bytearr):
  64. prefix = prefixes.response
  65. # Let's interact almost entirely with urllib3's response
  66. raw = response.raw
  67. # Let's convert the version int from httplib to bytes
  68. version_str = HTTP_VERSIONS.get(raw.version, b'?')
  69. # <prefix>HTTP/<version_str> <status_code> <reason>
  70. bytearr.extend(prefix + b'HTTP/' + version_str + b' ' +
  71. str(raw.status).encode('ascii') + b' ' +
  72. _coerce_to_bytes(response.reason) + b'\r\n')
  73. headers = raw.headers
  74. for name in headers.keys():
  75. for value in headers.getlist(name):
  76. bytearr.extend(prefix + _format_header(name, value))
  77. bytearr.extend(prefix + b'\r\n')
  78. bytearr.extend(response.content)
  79. def _coerce_to_bytes(data):
  80. if not isinstance(data, bytes) and hasattr(data, 'encode'):
  81. data = data.encode('utf-8')
  82. # Don't bail out with an exception if data is None
  83. return data if data is not None else b''
  84. def dump_response(response, request_prefix=b'< ', response_prefix=b'> ',
  85. data_array=None):
  86. """Dump a single request-response cycle's information.
  87. This will take a response object and dump only the data that requests can
  88. see for that single request-response cycle.
  89. Example::
  90. import requests
  91. from requests_toolbelt.utils import dump
  92. resp = requests.get('https://api.github.com/users/sigmavirus24')
  93. data = dump.dump_response(resp)
  94. print(data.decode('utf-8'))
  95. :param response:
  96. The response to format
  97. :type response: :class:`requests.Response`
  98. :param request_prefix: (*optional*)
  99. Bytes to prefix each line of the request data
  100. :type request_prefix: :class:`bytes`
  101. :param response_prefix: (*optional*)
  102. Bytes to prefix each line of the response data
  103. :type response_prefix: :class:`bytes`
  104. :param data_array: (*optional*)
  105. Bytearray to which we append the request-response cycle data
  106. :type data_array: :class:`bytearray`
  107. :returns: Formatted bytes of request and response information.
  108. :rtype: :class:`bytearray`
  109. """
  110. data = data_array if data_array is not None else bytearray()
  111. prefixes = PrefixSettings(request_prefix, response_prefix)
  112. if not hasattr(response, 'request'):
  113. raise ValueError('Response has no associated request')
  114. proxy_info = _get_proxy_information(response)
  115. _dump_request_data(response.request, prefixes, data,
  116. proxy_info=proxy_info)
  117. _dump_response_data(response, prefixes, data)
  118. return data
  119. def dump_all(response, request_prefix=b'< ', response_prefix=b'> '):
  120. """Dump all requests and responses including redirects.
  121. This takes the response returned by requests and will dump all
  122. request-response pairs in the redirect history in order followed by the
  123. final request-response.
  124. Example::
  125. import requests
  126. from requests_toolbelt.utils import dump
  127. resp = requests.get('https://httpbin.org/redirect/5')
  128. data = dump.dump_all(resp)
  129. print(data.decode('utf-8'))
  130. :param response:
  131. The response to format
  132. :type response: :class:`requests.Response`
  133. :param request_prefix: (*optional*)
  134. Bytes to prefix each line of the request data
  135. :type request_prefix: :class:`bytes`
  136. :param response_prefix: (*optional*)
  137. Bytes to prefix each line of the response data
  138. :type response_prefix: :class:`bytes`
  139. :returns: Formatted bytes of request and response information.
  140. :rtype: :class:`bytearray`
  141. """
  142. data = bytearray()
  143. history = list(response.history[:])
  144. history.append(response)
  145. for response in history:
  146. dump_response(response, request_prefix, response_prefix, data)
  147. return data