statement_splitter.py 4.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115
  1. #
  2. # Copyright (C) 2009-2020 the sqlparse authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of python-sqlparse and is released under
  6. # the BSD License: https://opensource.org/licenses/BSD-3-Clause
  7. from sqlparse import sql, tokens as T
  8. class StatementSplitter:
  9. """Filter that split stream at individual statements"""
  10. def __init__(self):
  11. self._reset()
  12. def _reset(self):
  13. """Set the filter attributes to its default values"""
  14. self._in_declare = False
  15. self._in_case = False
  16. self._is_create = False
  17. self._begin_depth = 0
  18. self.consume_ws = False
  19. self.tokens = []
  20. self.level = 0
  21. def _change_splitlevel(self, ttype, value):
  22. """Get the new split level (increase, decrease or remain equal)"""
  23. # parenthesis increase/decrease a level
  24. if ttype is T.Punctuation and value == '(':
  25. return 1
  26. elif ttype is T.Punctuation and value == ')':
  27. return -1
  28. elif ttype not in T.Keyword: # if normal token return
  29. return 0
  30. # Everything after here is ttype = T.Keyword
  31. # Also to note, once entered an If statement you are done and basically
  32. # returning
  33. unified = value.upper()
  34. # three keywords begin with CREATE, but only one of them is DDL
  35. # DDL Create though can contain more words such as "or replace"
  36. if ttype is T.Keyword.DDL and unified.startswith('CREATE'):
  37. self._is_create = True
  38. return 0
  39. # can have nested declare inside of being...
  40. if unified == 'DECLARE' and self._is_create and self._begin_depth == 0:
  41. self._in_declare = True
  42. return 1
  43. if unified == 'BEGIN':
  44. self._begin_depth += 1
  45. if self._is_create:
  46. # FIXME(andi): This makes no sense. ## this comment neither
  47. return 1
  48. return 0
  49. # BEGIN and CASE/WHEN both end with END
  50. if unified == 'END':
  51. if not self._in_case:
  52. self._begin_depth = max(0, self._begin_depth - 1)
  53. else:
  54. self._in_case = False
  55. return -1
  56. if (unified in ('IF', 'FOR', 'WHILE', 'CASE')
  57. and self._is_create and self._begin_depth > 0):
  58. if unified == 'CASE':
  59. self._in_case = True
  60. return 1
  61. if unified in ('END IF', 'END FOR', 'END WHILE'):
  62. return -1
  63. # Default
  64. return 0
  65. def process(self, stream):
  66. """Process the stream"""
  67. EOS_TTYPE = T.Whitespace, T.Comment.Single
  68. # Run over all stream tokens
  69. for ttype, value in stream:
  70. # Yield token if we finished a statement and there's no whitespaces
  71. # It will count newline token as a non whitespace. In this context
  72. # whitespace ignores newlines.
  73. # why don't multi line comments also count?
  74. if self.consume_ws and ttype not in EOS_TTYPE:
  75. yield sql.Statement(self.tokens)
  76. # Reset filter and prepare to process next statement
  77. self._reset()
  78. # Change current split level (increase, decrease or remain equal)
  79. self.level += self._change_splitlevel(ttype, value)
  80. # Append the token to the current statement
  81. self.tokens.append(sql.Token(ttype, value))
  82. # Check if we get the end of a statement
  83. # Issue762: Allow GO (or "GO 2") as statement splitter.
  84. # When implementing a language toggle, it's not only to add
  85. # keywords it's also to change some rules, like this splitting
  86. # rule.
  87. if (self.level <= 0 and ttype is T.Punctuation and value == ';') \
  88. or (ttype is T.Keyword and value.split()[0] == 'GO'):
  89. self.consume_ws = True
  90. # Yield pending statement (if any)
  91. if self.tokens and not all(t.is_whitespace for t in self.tokens):
  92. yield sql.Statement(self.tokens)