_shims.py 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193
  1. #!/usr/bin/env python
  2. #
  3. # Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
  4. # Author: Anirudh Vegesana (avegesan@cs.stanford.edu)
  5. # Copyright (c) 2021-2024 The Uncertainty Quantification Foundation.
  6. # License: 3-clause BSD. The full license text is available at:
  7. # - https://github.com/uqfoundation/dill/blob/master/LICENSE
  8. """
  9. Provides shims for compatibility between versions of dill and Python.
  10. Compatibility shims should be provided in this file. Here are two simple example
  11. use cases.
  12. Deprecation of constructor function:
  13. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  14. Assume that we were transitioning _import_module in _dill.py to
  15. the builtin function importlib.import_module when present.
  16. @move_to(_dill)
  17. def _import_module(import_name):
  18. ... # code already in _dill.py
  19. _import_module = Getattr(importlib, 'import_module', Getattr(_dill, '_import_module', None))
  20. The code will attempt to find import_module in the importlib module. If not
  21. present, it will use the _import_module function in _dill.
  22. Emulate new Python behavior in older Python versions:
  23. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  24. CellType.cell_contents behaves differently in Python 3.6 and 3.7. It is
  25. read-only in Python 3.6 and writable and deletable in 3.7.
  26. if _dill.OLD37 and _dill.HAS_CTYPES and ...:
  27. @move_to(_dill)
  28. def _setattr(object, name, value):
  29. if type(object) is _dill.CellType and name == 'cell_contents':
  30. _PyCell_Set.argtypes = (ctypes.py_object, ctypes.py_object)
  31. _PyCell_Set(object, value)
  32. else:
  33. setattr(object, name, value)
  34. ... # more cases below
  35. _setattr = Getattr(_dill, '_setattr', setattr)
  36. _dill._setattr will be used when present to emulate Python 3.7 functionality in
  37. older versions of Python while defaulting to the standard setattr in 3.7+.
  38. See this PR for the discussion that lead to this system:
  39. https://github.com/uqfoundation/dill/pull/443
  40. """
  41. import inspect
  42. import sys
  43. _dill = sys.modules['dill._dill']
  44. class Reduce(object):
  45. """
  46. Reduce objects are wrappers used for compatibility enforcement during
  47. unpickle-time. They should only be used in calls to pickler.save and
  48. other Reduce objects. They are only evaluated within unpickler.load.
  49. Pickling a Reduce object makes the two implementations equivalent:
  50. pickler.save(Reduce(*reduction))
  51. pickler.save_reduce(*reduction, obj=reduction)
  52. """
  53. __slots__ = ['reduction']
  54. def __new__(cls, *reduction, **kwargs):
  55. """
  56. Args:
  57. *reduction: a tuple that matches the format given here:
  58. https://docs.python.org/3/library/pickle.html#object.__reduce__
  59. is_callable: a bool to indicate that the object created by
  60. unpickling `reduction` is callable. If true, the current Reduce
  61. is allowed to be used as the function in further save_reduce calls
  62. or Reduce objects.
  63. """
  64. is_callable = kwargs.get('is_callable', False) # Pleases Py2. Can be removed later
  65. if is_callable:
  66. self = object.__new__(_CallableReduce)
  67. else:
  68. self = object.__new__(Reduce)
  69. self.reduction = reduction
  70. return self
  71. def __repr__(self):
  72. return 'Reduce%s' % (self.reduction,)
  73. def __copy__(self):
  74. return self # pragma: no cover
  75. def __deepcopy__(self, memo):
  76. return self # pragma: no cover
  77. def __reduce__(self):
  78. return self.reduction
  79. def __reduce_ex__(self, protocol):
  80. return self.__reduce__()
  81. class _CallableReduce(Reduce):
  82. # A version of Reduce for functions. Used to trick pickler.save_reduce into
  83. # thinking that Reduce objects of functions are themselves meaningful functions.
  84. def __call__(self, *args, **kwargs):
  85. reduction = self.__reduce__()
  86. func = reduction[0]
  87. f_args = reduction[1]
  88. obj = func(*f_args)
  89. return obj(*args, **kwargs)
  90. __NO_DEFAULT = _dill.Sentinel('Getattr.NO_DEFAULT')
  91. def Getattr(object, name, default=__NO_DEFAULT):
  92. """
  93. A Reduce object that represents the getattr operation. When unpickled, the
  94. Getattr will access an attribute 'name' of 'object' and return the value
  95. stored there. If the attribute doesn't exist, the default value will be
  96. returned if present.
  97. The following statements are equivalent:
  98. Getattr(collections, 'OrderedDict')
  99. Getattr(collections, 'spam', None)
  100. Getattr(*args)
  101. Reduce(getattr, (collections, 'OrderedDict'))
  102. Reduce(getattr, (collections, 'spam', None))
  103. Reduce(getattr, args)
  104. During unpickling, the first two will result in collections.OrderedDict and
  105. None respectively because the first attribute exists and the second one does
  106. not, forcing it to use the default value given in the third argument.
  107. """
  108. if default is Getattr.NO_DEFAULT:
  109. reduction = (getattr, (object, name))
  110. else:
  111. reduction = (getattr, (object, name, default))
  112. return Reduce(*reduction, is_callable=callable(default))
  113. Getattr.NO_DEFAULT = __NO_DEFAULT
  114. del __NO_DEFAULT
  115. def move_to(module, name=None):
  116. def decorator(func):
  117. if name is None:
  118. fname = func.__name__
  119. else:
  120. fname = name
  121. module.__dict__[fname] = func
  122. func.__module__ = module.__name__
  123. return func
  124. return decorator
  125. def register_shim(name, default):
  126. """
  127. A easier to understand and more compact way of "softly" defining a function.
  128. These two pieces of code are equivalent:
  129. if _dill.OLD3X:
  130. def _create_class():
  131. ...
  132. _create_class = register_shim('_create_class', types.new_class)
  133. if _dill.OLD3X:
  134. @move_to(_dill)
  135. def _create_class():
  136. ...
  137. _create_class = Getattr(_dill, '_create_class', types.new_class)
  138. Intuitively, it creates a function or object in the versions of dill/python
  139. that require special reimplementations, and use a core library or default
  140. implementation if that function or object does not exist.
  141. """
  142. func = globals().get(name)
  143. if func is not None:
  144. _dill.__dict__[name] = func
  145. func.__module__ = _dill.__name__
  146. if default is Getattr.NO_DEFAULT:
  147. reduction = (getattr, (_dill, name))
  148. else:
  149. reduction = (getattr, (_dill, name, default))
  150. return Reduce(*reduction, is_callable=callable(default))
  151. ######################
  152. ## Compatibility Shims are defined below
  153. ######################
  154. _CELL_EMPTY = register_shim('_CELL_EMPTY', None)
  155. _setattr = register_shim('_setattr', setattr)
  156. _delattr = register_shim('_delattr', delattr)