hstore.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459
  1. # dialects/postgresql/hstore.py
  2. # Copyright (C) 2005-2024 the SQLAlchemy authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of SQLAlchemy and is released under
  6. # the MIT License: https://www.opensource.org/licenses/mit-license.php
  7. import re
  8. from .array import ARRAY
  9. from ... import types as sqltypes
  10. from ... import util
  11. from ...sql import functions as sqlfunc
  12. from ...sql import operators
  13. __all__ = ("HSTORE", "hstore")
  14. idx_precedence = operators._PRECEDENCE[operators.json_getitem_op]
  15. GETITEM = operators.custom_op(
  16. "->",
  17. precedence=idx_precedence,
  18. natural_self_precedent=True,
  19. eager_grouping=True,
  20. )
  21. HAS_KEY = operators.custom_op(
  22. "?",
  23. precedence=idx_precedence,
  24. natural_self_precedent=True,
  25. eager_grouping=True,
  26. )
  27. HAS_ALL = operators.custom_op(
  28. "?&",
  29. precedence=idx_precedence,
  30. natural_self_precedent=True,
  31. eager_grouping=True,
  32. )
  33. HAS_ANY = operators.custom_op(
  34. "?|",
  35. precedence=idx_precedence,
  36. natural_self_precedent=True,
  37. eager_grouping=True,
  38. )
  39. CONTAINS = operators.custom_op(
  40. "@>",
  41. precedence=idx_precedence,
  42. natural_self_precedent=True,
  43. eager_grouping=True,
  44. )
  45. CONTAINED_BY = operators.custom_op(
  46. "<@",
  47. precedence=idx_precedence,
  48. natural_self_precedent=True,
  49. eager_grouping=True,
  50. )
  51. class HSTORE(sqltypes.Indexable, sqltypes.Concatenable, sqltypes.TypeEngine):
  52. """Represent the PostgreSQL HSTORE type.
  53. The :class:`.HSTORE` type stores dictionaries containing strings, e.g.::
  54. data_table = Table('data_table', metadata,
  55. Column('id', Integer, primary_key=True),
  56. Column('data', HSTORE)
  57. )
  58. with engine.connect() as conn:
  59. conn.execute(
  60. data_table.insert(),
  61. data = {"key1": "value1", "key2": "value2"}
  62. )
  63. :class:`.HSTORE` provides for a wide range of operations, including:
  64. * Index operations::
  65. data_table.c.data['some key'] == 'some value'
  66. * Containment operations::
  67. data_table.c.data.has_key('some key')
  68. data_table.c.data.has_all(['one', 'two', 'three'])
  69. * Concatenation::
  70. data_table.c.data + {"k1": "v1"}
  71. For a full list of special methods see
  72. :class:`.HSTORE.comparator_factory`.
  73. .. container:: topic
  74. **Detecting Changes in HSTORE columns when using the ORM**
  75. For usage with the SQLAlchemy ORM, it may be desirable to combine the
  76. usage of :class:`.HSTORE` with :class:`.MutableDict` dictionary now
  77. part of the :mod:`sqlalchemy.ext.mutable` extension. This extension
  78. will allow "in-place" changes to the dictionary, e.g. addition of new
  79. keys or replacement/removal of existing keys to/from the current
  80. dictionary, to produce events which will be detected by the unit of
  81. work::
  82. from sqlalchemy.ext.mutable import MutableDict
  83. class MyClass(Base):
  84. __tablename__ = 'data_table'
  85. id = Column(Integer, primary_key=True)
  86. data = Column(MutableDict.as_mutable(HSTORE))
  87. my_object = session.query(MyClass).one()
  88. # in-place mutation, requires Mutable extension
  89. # in order for the ORM to detect
  90. my_object.data['some_key'] = 'some value'
  91. session.commit()
  92. When the :mod:`sqlalchemy.ext.mutable` extension is not used, the ORM
  93. will not be alerted to any changes to the contents of an existing
  94. dictionary, unless that dictionary value is re-assigned to the
  95. HSTORE-attribute itself, thus generating a change event.
  96. .. seealso::
  97. :class:`.hstore` - render the PostgreSQL ``hstore()`` function.
  98. """
  99. __visit_name__ = "HSTORE"
  100. hashable = False
  101. text_type = sqltypes.Text()
  102. def __init__(self, text_type=None):
  103. """Construct a new :class:`.HSTORE`.
  104. :param text_type: the type that should be used for indexed values.
  105. Defaults to :class:`_types.Text`.
  106. .. versionadded:: 1.1.0
  107. """
  108. if text_type is not None:
  109. self.text_type = text_type
  110. class Comparator(
  111. sqltypes.Indexable.Comparator, sqltypes.Concatenable.Comparator
  112. ):
  113. """Define comparison operations for :class:`.HSTORE`."""
  114. def has_key(self, other):
  115. """Boolean expression. Test for presence of a key. Note that the
  116. key may be a SQLA expression.
  117. """
  118. return self.operate(HAS_KEY, other, result_type=sqltypes.Boolean)
  119. def has_all(self, other):
  120. """Boolean expression. Test for presence of all keys in jsonb"""
  121. return self.operate(HAS_ALL, other, result_type=sqltypes.Boolean)
  122. def has_any(self, other):
  123. """Boolean expression. Test for presence of any key in jsonb"""
  124. return self.operate(HAS_ANY, other, result_type=sqltypes.Boolean)
  125. def contains(self, other, **kwargs):
  126. """Boolean expression. Test if keys (or array) are a superset
  127. of/contained the keys of the argument jsonb expression.
  128. kwargs may be ignored by this operator but are required for API
  129. conformance.
  130. """
  131. return self.operate(CONTAINS, other, result_type=sqltypes.Boolean)
  132. def contained_by(self, other):
  133. """Boolean expression. Test if keys are a proper subset of the
  134. keys of the argument jsonb expression.
  135. """
  136. return self.operate(
  137. CONTAINED_BY, other, result_type=sqltypes.Boolean
  138. )
  139. def _setup_getitem(self, index):
  140. return GETITEM, index, self.type.text_type
  141. def defined(self, key):
  142. """Boolean expression. Test for presence of a non-NULL value for
  143. the key. Note that the key may be a SQLA expression.
  144. """
  145. return _HStoreDefinedFunction(self.expr, key)
  146. def delete(self, key):
  147. """HStore expression. Returns the contents of this hstore with the
  148. given key deleted. Note that the key may be a SQLA expression.
  149. """
  150. if isinstance(key, dict):
  151. key = _serialize_hstore(key)
  152. return _HStoreDeleteFunction(self.expr, key)
  153. def slice(self, array):
  154. """HStore expression. Returns a subset of an hstore defined by
  155. array of keys.
  156. """
  157. return _HStoreSliceFunction(self.expr, array)
  158. def keys(self):
  159. """Text array expression. Returns array of keys."""
  160. return _HStoreKeysFunction(self.expr)
  161. def vals(self):
  162. """Text array expression. Returns array of values."""
  163. return _HStoreValsFunction(self.expr)
  164. def array(self):
  165. """Text array expression. Returns array of alternating keys and
  166. values.
  167. """
  168. return _HStoreArrayFunction(self.expr)
  169. def matrix(self):
  170. """Text array expression. Returns array of [key, value] pairs."""
  171. return _HStoreMatrixFunction(self.expr)
  172. comparator_factory = Comparator
  173. def bind_processor(self, dialect):
  174. if util.py2k:
  175. encoding = dialect.encoding
  176. def process(value):
  177. if isinstance(value, dict):
  178. return _serialize_hstore(value).encode(encoding)
  179. else:
  180. return value
  181. else:
  182. def process(value):
  183. if isinstance(value, dict):
  184. return _serialize_hstore(value)
  185. else:
  186. return value
  187. return process
  188. def result_processor(self, dialect, coltype):
  189. if util.py2k:
  190. encoding = dialect.encoding
  191. def process(value):
  192. if value is not None:
  193. return _parse_hstore(value.decode(encoding))
  194. else:
  195. return value
  196. else:
  197. def process(value):
  198. if value is not None:
  199. return _parse_hstore(value)
  200. else:
  201. return value
  202. return process
  203. class hstore(sqlfunc.GenericFunction):
  204. """Construct an hstore value within a SQL expression using the
  205. PostgreSQL ``hstore()`` function.
  206. The :class:`.hstore` function accepts one or two arguments as described
  207. in the PostgreSQL documentation.
  208. E.g.::
  209. from sqlalchemy.dialects.postgresql import array, hstore
  210. select(hstore('key1', 'value1'))
  211. select(
  212. hstore(
  213. array(['key1', 'key2', 'key3']),
  214. array(['value1', 'value2', 'value3'])
  215. )
  216. )
  217. .. seealso::
  218. :class:`.HSTORE` - the PostgreSQL ``HSTORE`` datatype.
  219. """
  220. type = HSTORE
  221. name = "hstore"
  222. inherit_cache = True
  223. class _HStoreDefinedFunction(sqlfunc.GenericFunction):
  224. type = sqltypes.Boolean
  225. name = "defined"
  226. inherit_cache = True
  227. class _HStoreDeleteFunction(sqlfunc.GenericFunction):
  228. type = HSTORE
  229. name = "delete"
  230. inherit_cache = True
  231. class _HStoreSliceFunction(sqlfunc.GenericFunction):
  232. type = HSTORE
  233. name = "slice"
  234. inherit_cache = True
  235. class _HStoreKeysFunction(sqlfunc.GenericFunction):
  236. type = ARRAY(sqltypes.Text)
  237. name = "akeys"
  238. inherit_cache = True
  239. class _HStoreValsFunction(sqlfunc.GenericFunction):
  240. type = ARRAY(sqltypes.Text)
  241. name = "avals"
  242. inherit_cache = True
  243. class _HStoreArrayFunction(sqlfunc.GenericFunction):
  244. type = ARRAY(sqltypes.Text)
  245. name = "hstore_to_array"
  246. inherit_cache = True
  247. class _HStoreMatrixFunction(sqlfunc.GenericFunction):
  248. type = ARRAY(sqltypes.Text)
  249. name = "hstore_to_matrix"
  250. inherit_cache = True
  251. #
  252. # parsing. note that none of this is used with the psycopg2 backend,
  253. # which provides its own native extensions.
  254. #
  255. # My best guess at the parsing rules of hstore literals, since no formal
  256. # grammar is given. This is mostly reverse engineered from PG's input parser
  257. # behavior.
  258. HSTORE_PAIR_RE = re.compile(
  259. r"""
  260. (
  261. "(?P<key> (\\ . | [^"])* )" # Quoted key
  262. )
  263. [ ]* => [ ]* # Pair operator, optional adjoining whitespace
  264. (
  265. (?P<value_null> NULL ) # NULL value
  266. | "(?P<value> (\\ . | [^"])* )" # Quoted value
  267. )
  268. """,
  269. re.VERBOSE,
  270. )
  271. HSTORE_DELIMITER_RE = re.compile(
  272. r"""
  273. [ ]* , [ ]*
  274. """,
  275. re.VERBOSE,
  276. )
  277. def _parse_error(hstore_str, pos):
  278. """format an unmarshalling error."""
  279. ctx = 20
  280. hslen = len(hstore_str)
  281. parsed_tail = hstore_str[max(pos - ctx - 1, 0) : min(pos, hslen)]
  282. residual = hstore_str[min(pos, hslen) : min(pos + ctx + 1, hslen)]
  283. if len(parsed_tail) > ctx:
  284. parsed_tail = "[...]" + parsed_tail[1:]
  285. if len(residual) > ctx:
  286. residual = residual[:-1] + "[...]"
  287. return "After %r, could not parse residual at position %d: %r" % (
  288. parsed_tail,
  289. pos,
  290. residual,
  291. )
  292. def _parse_hstore(hstore_str):
  293. """Parse an hstore from its literal string representation.
  294. Attempts to approximate PG's hstore input parsing rules as closely as
  295. possible. Although currently this is not strictly necessary, since the
  296. current implementation of hstore's output syntax is stricter than what it
  297. accepts as input, the documentation makes no guarantees that will always
  298. be the case.
  299. """
  300. result = {}
  301. pos = 0
  302. pair_match = HSTORE_PAIR_RE.match(hstore_str)
  303. while pair_match is not None:
  304. key = pair_match.group("key").replace(r"\"", '"').replace("\\\\", "\\")
  305. if pair_match.group("value_null"):
  306. value = None
  307. else:
  308. value = (
  309. pair_match.group("value")
  310. .replace(r"\"", '"')
  311. .replace("\\\\", "\\")
  312. )
  313. result[key] = value
  314. pos += pair_match.end()
  315. delim_match = HSTORE_DELIMITER_RE.match(hstore_str[pos:])
  316. if delim_match is not None:
  317. pos += delim_match.end()
  318. pair_match = HSTORE_PAIR_RE.match(hstore_str[pos:])
  319. if pos != len(hstore_str):
  320. raise ValueError(_parse_error(hstore_str, pos))
  321. return result
  322. def _serialize_hstore(val):
  323. """Serialize a dictionary into an hstore literal. Keys and values must
  324. both be strings (except None for values).
  325. """
  326. def esc(s, position):
  327. if position == "value" and s is None:
  328. return "NULL"
  329. elif isinstance(s, util.string_types):
  330. return '"%s"' % s.replace("\\", "\\\\").replace('"', r"\"")
  331. else:
  332. raise ValueError(
  333. "%r in %s position is not a string." % (s, position)
  334. )
  335. return ", ".join(
  336. "%s=>%s" % (esc(k, "key"), esc(v, "value")) for k, v in val.items()
  337. )