Edit on GitHub

sqlglot.dialects.clickhouse

  1from __future__ import annotations
  2
  3import typing as t
  4
  5from sqlglot import exp, generator, parser, tokens
  6from sqlglot.dialects.dialect import (
  7    Dialect,
  8    inline_array_sql,
  9    no_pivot_sql,
 10    rename_func,
 11    var_map_sql,
 12)
 13from sqlglot.errors import ParseError
 14from sqlglot.parser import parse_var_map
 15from sqlglot.tokens import Token, TokenType
 16
 17
 18def _lower_func(sql: str) -> str:
 19    index = sql.index("(")
 20    return sql[:index].lower() + sql[index:]
 21
 22
 23class ClickHouse(Dialect):
 24    normalize_functions = None
 25    null_ordering = "nulls_are_last"
 26
 27    class Tokenizer(tokens.Tokenizer):
 28        COMMENTS = ["--", "#", "#!", ("/*", "*/")]
 29        IDENTIFIERS = ['"', "`"]
 30        BIT_STRINGS = [("0b", "")]
 31        HEX_STRINGS = [("0x", ""), ("0X", "")]
 32
 33        KEYWORDS = {
 34            **tokens.Tokenizer.KEYWORDS,
 35            "ASOF": TokenType.ASOF,
 36            "ATTACH": TokenType.COMMAND,
 37            "DATETIME64": TokenType.DATETIME64,
 38            "FINAL": TokenType.FINAL,
 39            "FLOAT32": TokenType.FLOAT,
 40            "FLOAT64": TokenType.DOUBLE,
 41            "GLOBAL": TokenType.GLOBAL,
 42            "INT128": TokenType.INT128,
 43            "INT16": TokenType.SMALLINT,
 44            "INT256": TokenType.INT256,
 45            "INT32": TokenType.INT,
 46            "INT64": TokenType.BIGINT,
 47            "INT8": TokenType.TINYINT,
 48            "MAP": TokenType.MAP,
 49            "TUPLE": TokenType.STRUCT,
 50            "UINT128": TokenType.UINT128,
 51            "UINT16": TokenType.USMALLINT,
 52            "UINT256": TokenType.UINT256,
 53            "UINT32": TokenType.UINT,
 54            "UINT64": TokenType.UBIGINT,
 55            "UINT8": TokenType.UTINYINT,
 56        }
 57
 58    class Parser(parser.Parser):
 59        FUNCTIONS = {
 60            **parser.Parser.FUNCTIONS,  # type: ignore
 61            "ANY": exp.AnyValue.from_arg_list,
 62            "MAP": parse_var_map,
 63            "MATCH": exp.RegexpLike.from_arg_list,
 64            "UNIQ": exp.ApproxDistinct.from_arg_list,
 65        }
 66
 67        FUNCTION_PARSERS = {
 68            **parser.Parser.FUNCTION_PARSERS,
 69            "QUANTILE": lambda self: self._parse_quantile(),
 70        }
 71
 72        FUNCTION_PARSERS.pop("MATCH")
 73
 74        NO_PAREN_FUNCTION_PARSERS = parser.Parser.NO_PAREN_FUNCTION_PARSERS.copy()
 75        NO_PAREN_FUNCTION_PARSERS.pop(TokenType.ANY)
 76
 77        RANGE_PARSERS = {
 78            **parser.Parser.RANGE_PARSERS,
 79            TokenType.GLOBAL: lambda self, this: self._match(TokenType.IN)
 80            and self._parse_in(this, is_global=True),
 81        }
 82
 83        # The PLACEHOLDER entry is popped because 1) it doesn't affect Clickhouse (it corresponds to
 84        # the postgres-specific JSONBContains parser) and 2) it makes parsing the ternary op simpler.
 85        COLUMN_OPERATORS = parser.Parser.COLUMN_OPERATORS.copy()
 86        COLUMN_OPERATORS.pop(TokenType.PLACEHOLDER)
 87
 88        JOIN_KINDS = {
 89            *parser.Parser.JOIN_KINDS,
 90            TokenType.ANY,
 91            TokenType.ASOF,
 92            TokenType.ANTI,
 93            TokenType.SEMI,
 94        }
 95
 96        TABLE_ALIAS_TOKENS = {*parser.Parser.TABLE_ALIAS_TOKENS} - {
 97            TokenType.ANY,
 98            TokenType.ASOF,
 99            TokenType.SEMI,
100            TokenType.ANTI,
101            TokenType.SETTINGS,
102            TokenType.FORMAT,
103        }
104
105        LOG_DEFAULTS_TO_LN = True
106
107        QUERY_MODIFIER_PARSERS = {
108            **parser.Parser.QUERY_MODIFIER_PARSERS,
109            "settings": lambda self: self._parse_csv(self._parse_conjunction)
110            if self._match(TokenType.SETTINGS)
111            else None,
112            "format": lambda self: self._parse_id_var() if self._match(TokenType.FORMAT) else None,
113        }
114
115        def _parse_conjunction(self) -> t.Optional[exp.Expression]:
116            this = super()._parse_conjunction()
117
118            if self._match(TokenType.PLACEHOLDER):
119                return self.expression(
120                    exp.If,
121                    this=this,
122                    true=self._parse_conjunction(),
123                    false=self._match(TokenType.COLON) and self._parse_conjunction(),
124                )
125
126            return this
127
128        def _parse_placeholder(self) -> t.Optional[exp.Expression]:
129            """
130            Parse a placeholder expression like SELECT {abc: UInt32} or FROM {table: Identifier}
131            https://clickhouse.com/docs/en/sql-reference/syntax#defining-and-using-query-parameters
132            """
133            if not self._match(TokenType.L_BRACE):
134                return None
135
136            this = self._parse_id_var()
137            self._match(TokenType.COLON)
138            kind = self._parse_types(check_func=False) or (
139                self._match_text_seq("IDENTIFIER") and "Identifier"
140            )
141
142            if not kind:
143                self.raise_error("Expecting a placeholder type or 'Identifier' for tables")
144            elif not self._match(TokenType.R_BRACE):
145                self.raise_error("Expecting }")
146
147            return self.expression(exp.Placeholder, this=this, kind=kind)
148
149        def _parse_in(
150            self, this: t.Optional[exp.Expression], is_global: bool = False
151        ) -> exp.Expression:
152            this = super()._parse_in(this)
153            this.set("is_global", is_global)
154            return this
155
156        def _parse_table(
157            self, schema: bool = False, alias_tokens: t.Optional[t.Collection[TokenType]] = None
158        ) -> t.Optional[exp.Expression]:
159            this = super()._parse_table(schema=schema, alias_tokens=alias_tokens)
160
161            if self._match(TokenType.FINAL):
162                this = self.expression(exp.Final, this=this)
163
164            return this
165
166        def _parse_position(self, haystack_first: bool = False) -> exp.Expression:
167            return super()._parse_position(haystack_first=True)
168
169        # https://clickhouse.com/docs/en/sql-reference/statements/select/with/
170        def _parse_cte(self) -> exp.Expression:
171            index = self._index
172            try:
173                # WITH <identifier> AS <subquery expression>
174                return super()._parse_cte()
175            except ParseError:
176                # WITH <expression> AS <identifier>
177                self._retreat(index)
178                statement = self._parse_statement()
179
180                if statement and isinstance(statement.this, exp.Alias):
181                    self.raise_error("Expected CTE to have alias")
182
183                return self.expression(exp.CTE, this=statement, alias=statement and statement.this)
184
185        def _parse_join_side_and_kind(
186            self,
187        ) -> t.Tuple[t.Optional[Token], t.Optional[Token], t.Optional[Token]]:
188            is_global = self._match(TokenType.GLOBAL) and self._prev
189            kind_pre = self._match_set(self.JOIN_KINDS, advance=False) and self._prev
190            if kind_pre:
191                kind = self._match_set(self.JOIN_KINDS) and self._prev
192                side = self._match_set(self.JOIN_SIDES) and self._prev
193                return is_global, side, kind
194            return (
195                is_global,
196                self._match_set(self.JOIN_SIDES) and self._prev,
197                self._match_set(self.JOIN_KINDS) and self._prev,
198            )
199
200        def _parse_join(self, skip_join_token: bool = False) -> t.Optional[exp.Expression]:
201            join = super()._parse_join(skip_join_token)
202
203            if join:
204                join.set("global", join.args.pop("natural", None))
205            return join
206
207        def _parse_function(
208            self, functions: t.Optional[t.Dict[str, t.Callable]] = None, anonymous: bool = False
209        ) -> t.Optional[exp.Expression]:
210            func = super()._parse_function(functions, anonymous)
211
212            if isinstance(func, exp.Anonymous):
213                params = self._parse_func_params(func)
214
215                if params:
216                    return self.expression(
217                        exp.ParameterizedAgg,
218                        this=func.this,
219                        expressions=func.expressions,
220                        params=params,
221                    )
222
223            return func
224
225        def _parse_func_params(
226            self, this: t.Optional[exp.Func] = None
227        ) -> t.Optional[t.List[t.Optional[exp.Expression]]]:
228            if self._match_pair(TokenType.R_PAREN, TokenType.L_PAREN):
229                return self._parse_csv(self._parse_lambda)
230            if self._match(TokenType.L_PAREN):
231                params = self._parse_csv(self._parse_lambda)
232                self._match_r_paren(this)
233                return params
234            return None
235
236        def _parse_quantile(self) -> exp.Quantile:
237            this = self._parse_lambda()
238            params = self._parse_func_params()
239            if params:
240                return self.expression(exp.Quantile, this=params[0], quantile=this)
241            return self.expression(exp.Quantile, this=this, quantile=exp.Literal.number(0.5))
242
243        def _parse_wrapped_id_vars(
244            self, optional: bool = False
245        ) -> t.List[t.Optional[exp.Expression]]:
246            return super()._parse_wrapped_id_vars(optional=True)
247
248    class Generator(generator.Generator):
249        STRUCT_DELIMITER = ("(", ")")
250
251        TYPE_MAPPING = {
252            **generator.Generator.TYPE_MAPPING,  # type: ignore
253            exp.DataType.Type.ARRAY: "Array",
254            exp.DataType.Type.BIGINT: "Int64",
255            exp.DataType.Type.DATETIME64: "DateTime64",
256            exp.DataType.Type.DOUBLE: "Float64",
257            exp.DataType.Type.FLOAT: "Float32",
258            exp.DataType.Type.INT: "Int32",
259            exp.DataType.Type.INT128: "Int128",
260            exp.DataType.Type.INT256: "Int256",
261            exp.DataType.Type.MAP: "Map",
262            exp.DataType.Type.NULLABLE: "Nullable",
263            exp.DataType.Type.SMALLINT: "Int16",
264            exp.DataType.Type.STRUCT: "Tuple",
265            exp.DataType.Type.TINYINT: "Int8",
266            exp.DataType.Type.UBIGINT: "UInt64",
267            exp.DataType.Type.UINT: "UInt32",
268            exp.DataType.Type.UINT128: "UInt128",
269            exp.DataType.Type.UINT256: "UInt256",
270            exp.DataType.Type.USMALLINT: "UInt16",
271            exp.DataType.Type.UTINYINT: "UInt8",
272        }
273
274        TRANSFORMS = {
275            **generator.Generator.TRANSFORMS,  # type: ignore
276            exp.AnyValue: rename_func("any"),
277            exp.ApproxDistinct: rename_func("uniq"),
278            exp.Array: inline_array_sql,
279            exp.CastToStrType: rename_func("CAST"),
280            exp.Final: lambda self, e: f"{self.sql(e, 'this')} FINAL",
281            exp.Map: lambda self, e: _lower_func(var_map_sql(self, e)),
282            exp.PartitionedByProperty: lambda self, e: f"PARTITION BY {self.sql(e, 'this')}",
283            exp.Pivot: no_pivot_sql,
284            exp.Quantile: lambda self, e: self.func("quantile", e.args.get("quantile"))
285            + f"({self.sql(e, 'this')})",
286            exp.RegexpLike: lambda self, e: f"match({self.format_args(e.this, e.expression)})",
287            exp.StrPosition: lambda self, e: f"position({self.format_args(e.this, e.args.get('substr'), e.args.get('position'))})",
288            exp.VarMap: lambda self, e: _lower_func(var_map_sql(self, e)),
289        }
290
291        PROPERTIES_LOCATION = {
292            **generator.Generator.PROPERTIES_LOCATION,  # type: ignore
293            exp.VolatileProperty: exp.Properties.Location.UNSUPPORTED,
294            exp.PartitionedByProperty: exp.Properties.Location.POST_SCHEMA,
295        }
296
297        JOIN_HINTS = False
298        TABLE_HINTS = False
299        EXPLICIT_UNION = True
300        GROUPINGS_SEP = ""
301
302        def cte_sql(self, expression: exp.CTE) -> str:
303            if isinstance(expression.this, exp.Alias):
304                return self.sql(expression, "this")
305
306            return super().cte_sql(expression)
307
308        def after_limit_modifiers(self, expression):
309            return super().after_limit_modifiers(expression) + [
310                self.seg("SETTINGS ") + self.expressions(expression, key="settings", flat=True)
311                if expression.args.get("settings")
312                else "",
313                self.seg("FORMAT ") + self.sql(expression, "format")
314                if expression.args.get("format")
315                else "",
316            ]
317
318        def parameterizedagg_sql(self, expression: exp.Anonymous) -> str:
319            params = self.expressions(expression, "params", flat=True)
320            return self.func(expression.name, *expression.expressions) + f"({params})"
321
322        def placeholder_sql(self, expression: exp.Placeholder) -> str:
323            return f"{{{expression.name}: {self.sql(expression, 'kind')}}}"
class ClickHouse(sqlglot.dialects.dialect.Dialect):
 24class ClickHouse(Dialect):
 25    normalize_functions = None
 26    null_ordering = "nulls_are_last"
 27
 28    class Tokenizer(tokens.Tokenizer):
 29        COMMENTS = ["--", "#", "#!", ("/*", "*/")]
 30        IDENTIFIERS = ['"', "`"]
 31        BIT_STRINGS = [("0b", "")]
 32        HEX_STRINGS = [("0x", ""), ("0X", "")]
 33
 34        KEYWORDS = {
 35            **tokens.Tokenizer.KEYWORDS,
 36            "ASOF": TokenType.ASOF,
 37            "ATTACH": TokenType.COMMAND,
 38            "DATETIME64": TokenType.DATETIME64,
 39            "FINAL": TokenType.FINAL,
 40            "FLOAT32": TokenType.FLOAT,
 41            "FLOAT64": TokenType.DOUBLE,
 42            "GLOBAL": TokenType.GLOBAL,
 43            "INT128": TokenType.INT128,
 44            "INT16": TokenType.SMALLINT,
 45            "INT256": TokenType.INT256,
 46            "INT32": TokenType.INT,
 47            "INT64": TokenType.BIGINT,
 48            "INT8": TokenType.TINYINT,
 49            "MAP": TokenType.MAP,
 50            "TUPLE": TokenType.STRUCT,
 51            "UINT128": TokenType.UINT128,
 52            "UINT16": TokenType.USMALLINT,
 53            "UINT256": TokenType.UINT256,
 54            "UINT32": TokenType.UINT,
 55            "UINT64": TokenType.UBIGINT,
 56            "UINT8": TokenType.UTINYINT,
 57        }
 58
 59    class Parser(parser.Parser):
 60        FUNCTIONS = {
 61            **parser.Parser.FUNCTIONS,  # type: ignore
 62            "ANY": exp.AnyValue.from_arg_list,
 63            "MAP": parse_var_map,
 64            "MATCH": exp.RegexpLike.from_arg_list,
 65            "UNIQ": exp.ApproxDistinct.from_arg_list,
 66        }
 67
 68        FUNCTION_PARSERS = {
 69            **parser.Parser.FUNCTION_PARSERS,
 70            "QUANTILE": lambda self: self._parse_quantile(),
 71        }
 72
 73        FUNCTION_PARSERS.pop("MATCH")
 74
 75        NO_PAREN_FUNCTION_PARSERS = parser.Parser.NO_PAREN_FUNCTION_PARSERS.copy()
 76        NO_PAREN_FUNCTION_PARSERS.pop(TokenType.ANY)
 77
 78        RANGE_PARSERS = {
 79            **parser.Parser.RANGE_PARSERS,
 80            TokenType.GLOBAL: lambda self, this: self._match(TokenType.IN)
 81            and self._parse_in(this, is_global=True),
 82        }
 83
 84        # The PLACEHOLDER entry is popped because 1) it doesn't affect Clickhouse (it corresponds to
 85        # the postgres-specific JSONBContains parser) and 2) it makes parsing the ternary op simpler.
 86        COLUMN_OPERATORS = parser.Parser.COLUMN_OPERATORS.copy()
 87        COLUMN_OPERATORS.pop(TokenType.PLACEHOLDER)
 88
 89        JOIN_KINDS = {
 90            *parser.Parser.JOIN_KINDS,
 91            TokenType.ANY,
 92            TokenType.ASOF,
 93            TokenType.ANTI,
 94            TokenType.SEMI,
 95        }
 96
 97        TABLE_ALIAS_TOKENS = {*parser.Parser.TABLE_ALIAS_TOKENS} - {
 98            TokenType.ANY,
 99            TokenType.ASOF,
100            TokenType.SEMI,
101            TokenType.ANTI,
102            TokenType.SETTINGS,
103            TokenType.FORMAT,
104        }
105
106        LOG_DEFAULTS_TO_LN = True
107
108        QUERY_MODIFIER_PARSERS = {
109            **parser.Parser.QUERY_MODIFIER_PARSERS,
110            "settings": lambda self: self._parse_csv(self._parse_conjunction)
111            if self._match(TokenType.SETTINGS)
112            else None,
113            "format": lambda self: self._parse_id_var() if self._match(TokenType.FORMAT) else None,
114        }
115
116        def _parse_conjunction(self) -> t.Optional[exp.Expression]:
117            this = super()._parse_conjunction()
118
119            if self._match(TokenType.PLACEHOLDER):
120                return self.expression(
121                    exp.If,
122                    this=this,
123                    true=self._parse_conjunction(),
124                    false=self._match(TokenType.COLON) and self._parse_conjunction(),
125                )
126
127            return this
128
129        def _parse_placeholder(self) -> t.Optional[exp.Expression]:
130            """
131            Parse a placeholder expression like SELECT {abc: UInt32} or FROM {table: Identifier}
132            https://clickhouse.com/docs/en/sql-reference/syntax#defining-and-using-query-parameters
133            """
134            if not self._match(TokenType.L_BRACE):
135                return None
136
137            this = self._parse_id_var()
138            self._match(TokenType.COLON)
139            kind = self._parse_types(check_func=False) or (
140                self._match_text_seq("IDENTIFIER") and "Identifier"
141            )
142
143            if not kind:
144                self.raise_error("Expecting a placeholder type or 'Identifier' for tables")
145            elif not self._match(TokenType.R_BRACE):
146                self.raise_error("Expecting }")
147
148            return self.expression(exp.Placeholder, this=this, kind=kind)
149
150        def _parse_in(
151            self, this: t.Optional[exp.Expression], is_global: bool = False
152        ) -> exp.Expression:
153            this = super()._parse_in(this)
154            this.set("is_global", is_global)
155            return this
156
157        def _parse_table(
158            self, schema: bool = False, alias_tokens: t.Optional[t.Collection[TokenType]] = None
159        ) -> t.Optional[exp.Expression]:
160            this = super()._parse_table(schema=schema, alias_tokens=alias_tokens)
161
162            if self._match(TokenType.FINAL):
163                this = self.expression(exp.Final, this=this)
164
165            return this
166
167        def _parse_position(self, haystack_first: bool = False) -> exp.Expression:
168            return super()._parse_position(haystack_first=True)
169
170        # https://clickhouse.com/docs/en/sql-reference/statements/select/with/
171        def _parse_cte(self) -> exp.Expression:
172            index = self._index
173            try:
174                # WITH <identifier> AS <subquery expression>
175                return super()._parse_cte()
176            except ParseError:
177                # WITH <expression> AS <identifier>
178                self._retreat(index)
179                statement = self._parse_statement()
180
181                if statement and isinstance(statement.this, exp.Alias):
182                    self.raise_error("Expected CTE to have alias")
183
184                return self.expression(exp.CTE, this=statement, alias=statement and statement.this)
185
186        def _parse_join_side_and_kind(
187            self,
188        ) -> t.Tuple[t.Optional[Token], t.Optional[Token], t.Optional[Token]]:
189            is_global = self._match(TokenType.GLOBAL) and self._prev
190            kind_pre = self._match_set(self.JOIN_KINDS, advance=False) and self._prev
191            if kind_pre:
192                kind = self._match_set(self.JOIN_KINDS) and self._prev
193                side = self._match_set(self.JOIN_SIDES) and self._prev
194                return is_global, side, kind
195            return (
196                is_global,
197                self._match_set(self.JOIN_SIDES) and self._prev,
198                self._match_set(self.JOIN_KINDS) and self._prev,
199            )
200
201        def _parse_join(self, skip_join_token: bool = False) -> t.Optional[exp.Expression]:
202            join = super()._parse_join(skip_join_token)
203
204            if join:
205                join.set("global", join.args.pop("natural", None))
206            return join
207
208        def _parse_function(
209            self, functions: t.Optional[t.Dict[str, t.Callable]] = None, anonymous: bool = False
210        ) -> t.Optional[exp.Expression]:
211            func = super()._parse_function(functions, anonymous)
212
213            if isinstance(func, exp.Anonymous):
214                params = self._parse_func_params(func)
215
216                if params:
217                    return self.expression(
218                        exp.ParameterizedAgg,
219                        this=func.this,
220                        expressions=func.expressions,
221                        params=params,
222                    )
223
224            return func
225
226        def _parse_func_params(
227            self, this: t.Optional[exp.Func] = None
228        ) -> t.Optional[t.List[t.Optional[exp.Expression]]]:
229            if self._match_pair(TokenType.R_PAREN, TokenType.L_PAREN):
230                return self._parse_csv(self._parse_lambda)
231            if self._match(TokenType.L_PAREN):
232                params = self._parse_csv(self._parse_lambda)
233                self._match_r_paren(this)
234                return params
235            return None
236
237        def _parse_quantile(self) -> exp.Quantile:
238            this = self._parse_lambda()
239            params = self._parse_func_params()
240            if params:
241                return self.expression(exp.Quantile, this=params[0], quantile=this)
242            return self.expression(exp.Quantile, this=this, quantile=exp.Literal.number(0.5))
243
244        def _parse_wrapped_id_vars(
245            self, optional: bool = False
246        ) -> t.List[t.Optional[exp.Expression]]:
247            return super()._parse_wrapped_id_vars(optional=True)
248
249    class Generator(generator.Generator):
250        STRUCT_DELIMITER = ("(", ")")
251
252        TYPE_MAPPING = {
253            **generator.Generator.TYPE_MAPPING,  # type: ignore
254            exp.DataType.Type.ARRAY: "Array",
255            exp.DataType.Type.BIGINT: "Int64",
256            exp.DataType.Type.DATETIME64: "DateTime64",
257            exp.DataType.Type.DOUBLE: "Float64",
258            exp.DataType.Type.FLOAT: "Float32",
259            exp.DataType.Type.INT: "Int32",
260            exp.DataType.Type.INT128: "Int128",
261            exp.DataType.Type.INT256: "Int256",
262            exp.DataType.Type.MAP: "Map",
263            exp.DataType.Type.NULLABLE: "Nullable",
264            exp.DataType.Type.SMALLINT: "Int16",
265            exp.DataType.Type.STRUCT: "Tuple",
266            exp.DataType.Type.TINYINT: "Int8",
267            exp.DataType.Type.UBIGINT: "UInt64",
268            exp.DataType.Type.UINT: "UInt32",
269            exp.DataType.Type.UINT128: "UInt128",
270            exp.DataType.Type.UINT256: "UInt256",
271            exp.DataType.Type.USMALLINT: "UInt16",
272            exp.DataType.Type.UTINYINT: "UInt8",
273        }
274
275        TRANSFORMS = {
276            **generator.Generator.TRANSFORMS,  # type: ignore
277            exp.AnyValue: rename_func("any"),
278            exp.ApproxDistinct: rename_func("uniq"),
279            exp.Array: inline_array_sql,
280            exp.CastToStrType: rename_func("CAST"),
281            exp.Final: lambda self, e: f"{self.sql(e, 'this')} FINAL",
282            exp.Map: lambda self, e: _lower_func(var_map_sql(self, e)),
283            exp.PartitionedByProperty: lambda self, e: f"PARTITION BY {self.sql(e, 'this')}",
284            exp.Pivot: no_pivot_sql,
285            exp.Quantile: lambda self, e: self.func("quantile", e.args.get("quantile"))
286            + f"({self.sql(e, 'this')})",
287            exp.RegexpLike: lambda self, e: f"match({self.format_args(e.this, e.expression)})",
288            exp.StrPosition: lambda self, e: f"position({self.format_args(e.this, e.args.get('substr'), e.args.get('position'))})",
289            exp.VarMap: lambda self, e: _lower_func(var_map_sql(self, e)),
290        }
291
292        PROPERTIES_LOCATION = {
293            **generator.Generator.PROPERTIES_LOCATION,  # type: ignore
294            exp.VolatileProperty: exp.Properties.Location.UNSUPPORTED,
295            exp.PartitionedByProperty: exp.Properties.Location.POST_SCHEMA,
296        }
297
298        JOIN_HINTS = False
299        TABLE_HINTS = False
300        EXPLICIT_UNION = True
301        GROUPINGS_SEP = ""
302
303        def cte_sql(self, expression: exp.CTE) -> str:
304            if isinstance(expression.this, exp.Alias):
305                return self.sql(expression, "this")
306
307            return super().cte_sql(expression)
308
309        def after_limit_modifiers(self, expression):
310            return super().after_limit_modifiers(expression) + [
311                self.seg("SETTINGS ") + self.expressions(expression, key="settings", flat=True)
312                if expression.args.get("settings")
313                else "",
314                self.seg("FORMAT ") + self.sql(expression, "format")
315                if expression.args.get("format")
316                else "",
317            ]
318
319        def parameterizedagg_sql(self, expression: exp.Anonymous) -> str:
320            params = self.expressions(expression, "params", flat=True)
321            return self.func(expression.name, *expression.expressions) + f"({params})"
322
323        def placeholder_sql(self, expression: exp.Placeholder) -> str:
324            return f"{{{expression.name}: {self.sql(expression, 'kind')}}}"
class ClickHouse.Tokenizer(sqlglot.tokens.Tokenizer):
28    class Tokenizer(tokens.Tokenizer):
29        COMMENTS = ["--", "#", "#!", ("/*", "*/")]
30        IDENTIFIERS = ['"', "`"]
31        BIT_STRINGS = [("0b", "")]
32        HEX_STRINGS = [("0x", ""), ("0X", "")]
33
34        KEYWORDS = {
35            **tokens.Tokenizer.KEYWORDS,
36            "ASOF": TokenType.ASOF,
37            "ATTACH": TokenType.COMMAND,
38            "DATETIME64": TokenType.DATETIME64,
39            "FINAL": TokenType.FINAL,
40            "FLOAT32": TokenType.FLOAT,
41            "FLOAT64": TokenType.DOUBLE,
42            "GLOBAL": TokenType.GLOBAL,
43            "INT128": TokenType.INT128,
44            "INT16": TokenType.SMALLINT,
45            "INT256": TokenType.INT256,
46            "INT32": TokenType.INT,
47            "INT64": TokenType.BIGINT,
48            "INT8": TokenType.TINYINT,
49            "MAP": TokenType.MAP,
50            "TUPLE": TokenType.STRUCT,
51            "UINT128": TokenType.UINT128,
52            "UINT16": TokenType.USMALLINT,
53            "UINT256": TokenType.UINT256,
54            "UINT32": TokenType.UINT,
55            "UINT64": TokenType.UBIGINT,
56            "UINT8": TokenType.UTINYINT,
57        }
class ClickHouse.Parser(sqlglot.parser.Parser):
 59    class Parser(parser.Parser):
 60        FUNCTIONS = {
 61            **parser.Parser.FUNCTIONS,  # type: ignore
 62            "ANY": exp.AnyValue.from_arg_list,
 63            "MAP": parse_var_map,
 64            "MATCH": exp.RegexpLike.from_arg_list,
 65            "UNIQ": exp.ApproxDistinct.from_arg_list,
 66        }
 67
 68        FUNCTION_PARSERS = {
 69            **parser.Parser.FUNCTION_PARSERS,
 70            "QUANTILE": lambda self: self._parse_quantile(),
 71        }
 72
 73        FUNCTION_PARSERS.pop("MATCH")
 74
 75        NO_PAREN_FUNCTION_PARSERS = parser.Parser.NO_PAREN_FUNCTION_PARSERS.copy()
 76        NO_PAREN_FUNCTION_PARSERS.pop(TokenType.ANY)
 77
 78        RANGE_PARSERS = {
 79            **parser.Parser.RANGE_PARSERS,
 80            TokenType.GLOBAL: lambda self, this: self._match(TokenType.IN)
 81            and self._parse_in(this, is_global=True),
 82        }
 83
 84        # The PLACEHOLDER entry is popped because 1) it doesn't affect Clickhouse (it corresponds to
 85        # the postgres-specific JSONBContains parser) and 2) it makes parsing the ternary op simpler.
 86        COLUMN_OPERATORS = parser.Parser.COLUMN_OPERATORS.copy()
 87        COLUMN_OPERATORS.pop(TokenType.PLACEHOLDER)
 88
 89        JOIN_KINDS = {
 90            *parser.Parser.JOIN_KINDS,
 91            TokenType.ANY,
 92            TokenType.ASOF,
 93            TokenType.ANTI,
 94            TokenType.SEMI,
 95        }
 96
 97        TABLE_ALIAS_TOKENS = {*parser.Parser.TABLE_ALIAS_TOKENS} - {
 98            TokenType.ANY,
 99            TokenType.ASOF,
100            TokenType.SEMI,
101            TokenType.ANTI,
102            TokenType.SETTINGS,
103            TokenType.FORMAT,
104        }
105
106        LOG_DEFAULTS_TO_LN = True
107
108        QUERY_MODIFIER_PARSERS = {
109            **parser.Parser.QUERY_MODIFIER_PARSERS,
110            "settings": lambda self: self._parse_csv(self._parse_conjunction)
111            if self._match(TokenType.SETTINGS)
112            else None,
113            "format": lambda self: self._parse_id_var() if self._match(TokenType.FORMAT) else None,
114        }
115
116        def _parse_conjunction(self) -> t.Optional[exp.Expression]:
117            this = super()._parse_conjunction()
118
119            if self._match(TokenType.PLACEHOLDER):
120                return self.expression(
121                    exp.If,
122                    this=this,
123                    true=self._parse_conjunction(),
124                    false=self._match(TokenType.COLON) and self._parse_conjunction(),
125                )
126
127            return this
128
129        def _parse_placeholder(self) -> t.Optional[exp.Expression]:
130            """
131            Parse a placeholder expression like SELECT {abc: UInt32} or FROM {table: Identifier}
132            https://clickhouse.com/docs/en/sql-reference/syntax#defining-and-using-query-parameters
133            """
134            if not self._match(TokenType.L_BRACE):
135                return None
136
137            this = self._parse_id_var()
138            self._match(TokenType.COLON)
139            kind = self._parse_types(check_func=False) or (
140                self._match_text_seq("IDENTIFIER") and "Identifier"
141            )
142
143            if not kind:
144                self.raise_error("Expecting a placeholder type or 'Identifier' for tables")
145            elif not self._match(TokenType.R_BRACE):
146                self.raise_error("Expecting }")
147
148            return self.expression(exp.Placeholder, this=this, kind=kind)
149
150        def _parse_in(
151            self, this: t.Optional[exp.Expression], is_global: bool = False
152        ) -> exp.Expression:
153            this = super()._parse_in(this)
154            this.set("is_global", is_global)
155            return this
156
157        def _parse_table(
158            self, schema: bool = False, alias_tokens: t.Optional[t.Collection[TokenType]] = None
159        ) -> t.Optional[exp.Expression]:
160            this = super()._parse_table(schema=schema, alias_tokens=alias_tokens)
161
162            if self._match(TokenType.FINAL):
163                this = self.expression(exp.Final, this=this)
164
165            return this
166
167        def _parse_position(self, haystack_first: bool = False) -> exp.Expression:
168            return super()._parse_position(haystack_first=True)
169
170        # https://clickhouse.com/docs/en/sql-reference/statements/select/with/
171        def _parse_cte(self) -> exp.Expression:
172            index = self._index
173            try:
174                # WITH <identifier> AS <subquery expression>
175                return super()._parse_cte()
176            except ParseError:
177                # WITH <expression> AS <identifier>
178                self._retreat(index)
179                statement = self._parse_statement()
180
181                if statement and isinstance(statement.this, exp.Alias):
182                    self.raise_error("Expected CTE to have alias")
183
184                return self.expression(exp.CTE, this=statement, alias=statement and statement.this)
185
186        def _parse_join_side_and_kind(
187            self,
188        ) -> t.Tuple[t.Optional[Token], t.Optional[Token], t.Optional[Token]]:
189            is_global = self._match(TokenType.GLOBAL) and self._prev
190            kind_pre = self._match_set(self.JOIN_KINDS, advance=False) and self._prev
191            if kind_pre:
192                kind = self._match_set(self.JOIN_KINDS) and self._prev
193                side = self._match_set(self.JOIN_SIDES) and self._prev
194                return is_global, side, kind
195            return (
196                is_global,
197                self._match_set(self.JOIN_SIDES) and self._prev,
198                self._match_set(self.JOIN_KINDS) and self._prev,
199            )
200
201        def _parse_join(self, skip_join_token: bool = False) -> t.Optional[exp.Expression]:
202            join = super()._parse_join(skip_join_token)
203
204            if join:
205                join.set("global", join.args.pop("natural", None))
206            return join
207
208        def _parse_function(
209            self, functions: t.Optional[t.Dict[str, t.Callable]] = None, anonymous: bool = False
210        ) -> t.Optional[exp.Expression]:
211            func = super()._parse_function(functions, anonymous)
212
213            if isinstance(func, exp.Anonymous):
214                params = self._parse_func_params(func)
215
216                if params:
217                    return self.expression(
218                        exp.ParameterizedAgg,
219                        this=func.this,
220                        expressions=func.expressions,
221                        params=params,
222                    )
223
224            return func
225
226        def _parse_func_params(
227            self, this: t.Optional[exp.Func] = None
228        ) -> t.Optional[t.List[t.Optional[exp.Expression]]]:
229            if self._match_pair(TokenType.R_PAREN, TokenType.L_PAREN):
230                return self._parse_csv(self._parse_lambda)
231            if self._match(TokenType.L_PAREN):
232                params = self._parse_csv(self._parse_lambda)
233                self._match_r_paren(this)
234                return params
235            return None
236
237        def _parse_quantile(self) -> exp.Quantile:
238            this = self._parse_lambda()
239            params = self._parse_func_params()
240            if params:
241                return self.expression(exp.Quantile, this=params[0], quantile=this)
242            return self.expression(exp.Quantile, this=this, quantile=exp.Literal.number(0.5))
243
244        def _parse_wrapped_id_vars(
245            self, optional: bool = False
246        ) -> t.List[t.Optional[exp.Expression]]:
247            return super()._parse_wrapped_id_vars(optional=True)

Parser consumes a list of tokens produced by the sqlglot.tokens.Tokenizer and produces a parsed syntax tree.

Arguments:
  • error_level: the desired error level. Default: ErrorLevel.RAISE
  • error_message_context: determines the amount of context to capture from a query string when displaying the error message (in number of characters). Default: 50.
  • index_offset: Index offset for arrays eg ARRAY[0] vs ARRAY[1] as the head of a list. Default: 0
  • alias_post_tablesample: If the table alias comes after tablesample. Default: False
  • max_errors: Maximum number of error messages to include in a raised ParseError. This is only relevant if error_level is ErrorLevel.RAISE. Default: 3
  • null_ordering: Indicates the default null ordering method to use if not explicitly set. Options are "nulls_are_small", "nulls_are_large", "nulls_are_last". Default: "nulls_are_small"
class ClickHouse.Generator(sqlglot.generator.Generator):
249    class Generator(generator.Generator):
250        STRUCT_DELIMITER = ("(", ")")
251
252        TYPE_MAPPING = {
253            **generator.Generator.TYPE_MAPPING,  # type: ignore
254            exp.DataType.Type.ARRAY: "Array",
255            exp.DataType.Type.BIGINT: "Int64",
256            exp.DataType.Type.DATETIME64: "DateTime64",
257            exp.DataType.Type.DOUBLE: "Float64",
258            exp.DataType.Type.FLOAT: "Float32",
259            exp.DataType.Type.INT: "Int32",
260            exp.DataType.Type.INT128: "Int128",
261            exp.DataType.Type.INT256: "Int256",
262            exp.DataType.Type.MAP: "Map",
263            exp.DataType.Type.NULLABLE: "Nullable",
264            exp.DataType.Type.SMALLINT: "Int16",
265            exp.DataType.Type.STRUCT: "Tuple",
266            exp.DataType.Type.TINYINT: "Int8",
267            exp.DataType.Type.UBIGINT: "UInt64",
268            exp.DataType.Type.UINT: "UInt32",
269            exp.DataType.Type.UINT128: "UInt128",
270            exp.DataType.Type.UINT256: "UInt256",
271            exp.DataType.Type.USMALLINT: "UInt16",
272            exp.DataType.Type.UTINYINT: "UInt8",
273        }
274
275        TRANSFORMS = {
276            **generator.Generator.TRANSFORMS,  # type: ignore
277            exp.AnyValue: rename_func("any"),
278            exp.ApproxDistinct: rename_func("uniq"),
279            exp.Array: inline_array_sql,
280            exp.CastToStrType: rename_func("CAST"),
281            exp.Final: lambda self, e: f"{self.sql(e, 'this')} FINAL",
282            exp.Map: lambda self, e: _lower_func(var_map_sql(self, e)),
283            exp.PartitionedByProperty: lambda self, e: f"PARTITION BY {self.sql(e, 'this')}",
284            exp.Pivot: no_pivot_sql,
285            exp.Quantile: lambda self, e: self.func("quantile", e.args.get("quantile"))
286            + f"({self.sql(e, 'this')})",
287            exp.RegexpLike: lambda self, e: f"match({self.format_args(e.this, e.expression)})",
288            exp.StrPosition: lambda self, e: f"position({self.format_args(e.this, e.args.get('substr'), e.args.get('position'))})",
289            exp.VarMap: lambda self, e: _lower_func(var_map_sql(self, e)),
290        }
291
292        PROPERTIES_LOCATION = {
293            **generator.Generator.PROPERTIES_LOCATION,  # type: ignore
294            exp.VolatileProperty: exp.Properties.Location.UNSUPPORTED,
295            exp.PartitionedByProperty: exp.Properties.Location.POST_SCHEMA,
296        }
297
298        JOIN_HINTS = False
299        TABLE_HINTS = False
300        EXPLICIT_UNION = True
301        GROUPINGS_SEP = ""
302
303        def cte_sql(self, expression: exp.CTE) -> str:
304            if isinstance(expression.this, exp.Alias):
305                return self.sql(expression, "this")
306
307            return super().cte_sql(expression)
308
309        def after_limit_modifiers(self, expression):
310            return super().after_limit_modifiers(expression) + [
311                self.seg("SETTINGS ") + self.expressions(expression, key="settings", flat=True)
312                if expression.args.get("settings")
313                else "",
314                self.seg("FORMAT ") + self.sql(expression, "format")
315                if expression.args.get("format")
316                else "",
317            ]
318
319        def parameterizedagg_sql(self, expression: exp.Anonymous) -> str:
320            params = self.expressions(expression, "params", flat=True)
321            return self.func(expression.name, *expression.expressions) + f"({params})"
322
323        def placeholder_sql(self, expression: exp.Placeholder) -> str:
324            return f"{{{expression.name}: {self.sql(expression, 'kind')}}}"

Generator interprets the given syntax tree and produces a SQL string as an output.

Arguments:
  • time_mapping (dict): the dictionary of custom time mappings in which the key represents a python time format and the output the target time format
  • time_trie (trie): a trie of the time_mapping keys
  • pretty (bool): if set to True the returned string will be formatted. Default: False.
  • quote_start (str): specifies which starting character to use to delimit quotes. Default: '.
  • quote_end (str): specifies which ending character to use to delimit quotes. Default: '.
  • identifier_start (str): specifies which starting character to use to delimit identifiers. Default: ".
  • identifier_end (str): specifies which ending character to use to delimit identifiers. Default: ".
  • bit_start (str): specifies which starting character to use to delimit bit literals. Default: None.
  • bit_end (str): specifies which ending character to use to delimit bit literals. Default: None.
  • hex_start (str): specifies which starting character to use to delimit hex literals. Default: None.
  • hex_end (str): specifies which ending character to use to delimit hex literals. Default: None.
  • byte_start (str): specifies which starting character to use to delimit byte literals. Default: None.
  • byte_end (str): specifies which ending character to use to delimit byte literals. Default: None.
  • identify (bool | str): 'always': always quote, 'safe': quote identifiers if they don't contain an upcase, True defaults to always.
  • normalize (bool): if set to True all identifiers will lower cased
  • string_escape (str): specifies a string escape character. Default: '.
  • identifier_escape (str): specifies an identifier escape character. Default: ".
  • pad (int): determines padding in a formatted string. Default: 2.
  • indent (int): determines the size of indentation in a formatted string. Default: 4.
  • unnest_column_only (bool): if true unnest table aliases are considered only as column aliases
  • normalize_functions (str): normalize function names, "upper", "lower", or None Default: "upper"
  • alias_post_tablesample (bool): if the table alias comes after tablesample Default: False
  • unsupported_level (ErrorLevel): determines the generator's behavior when it encounters unsupported expressions. Default ErrorLevel.WARN.
  • null_ordering (str): Indicates the default null ordering method to use if not explicitly set. Options are "nulls_are_small", "nulls_are_large", "nulls_are_last". Default: "nulls_are_small"
  • max_unsupported (int): Maximum number of unsupported messages to include in a raised UnsupportedError. This is only relevant if unsupported_level is ErrorLevel.RAISE. Default: 3
  • leading_comma (bool): if the the comma is leading or trailing in select statements Default: False
  • max_text_width: The max number of characters in a segment before creating new lines in pretty mode. The default is on the smaller end because the length only represents a segment and not the true line length. Default: 80
  • comments: Whether or not to preserve comments in the output SQL code. Default: True
def cte_sql(self, expression: sqlglot.expressions.CTE) -> str:
303        def cte_sql(self, expression: exp.CTE) -> str:
304            if isinstance(expression.this, exp.Alias):
305                return self.sql(expression, "this")
306
307            return super().cte_sql(expression)
def after_limit_modifiers(self, expression):
309        def after_limit_modifiers(self, expression):
310            return super().after_limit_modifiers(expression) + [
311                self.seg("SETTINGS ") + self.expressions(expression, key="settings", flat=True)
312                if expression.args.get("settings")
313                else "",
314                self.seg("FORMAT ") + self.sql(expression, "format")
315                if expression.args.get("format")
316                else "",
317            ]
def parameterizedagg_sql(self, expression: sqlglot.expressions.Anonymous) -> str:
319        def parameterizedagg_sql(self, expression: exp.Anonymous) -> str:
320            params = self.expressions(expression, "params", flat=True)
321            return self.func(expression.name, *expression.expressions) + f"({params})"
def placeholder_sql(self, expression: sqlglot.expressions.Placeholder) -> str:
323        def placeholder_sql(self, expression: exp.Placeholder) -> str:
324            return f"{{{expression.name}: {self.sql(expression, 'kind')}}}"
Inherited Members
sqlglot.generator.Generator
Generator
generate
unsupported
sep
seg
pad_comment
maybe_comment
wrap
no_identify
normalize_func
indent
sql
uncache_sql
cache_sql
characterset_sql
column_sql
columnposition_sql
columndef_sql
columnconstraint_sql
autoincrementcolumnconstraint_sql
compresscolumnconstraint_sql
generatedasidentitycolumnconstraint_sql
notnullcolumnconstraint_sql
primarykeycolumnconstraint_sql
uniquecolumnconstraint_sql
create_sql
clone_sql
describe_sql
prepend_ctes
with_sql
tablealias_sql
bitstring_sql
hexstring_sql
bytestring_sql
datatypesize_sql
datatype_sql
directory_sql
delete_sql
drop_sql
except_sql
except_op
fetch_sql
filter_sql
hint_sql
index_sql
identifier_sql
inputoutputformat_sql
national_sql
partition_sql
properties_sql
root_properties
properties
with_properties
locate_properties
property_sql
likeproperty_sql
fallbackproperty_sql
journalproperty_sql
freespaceproperty_sql
afterjournalproperty_sql
checksumproperty_sql
mergeblockratioproperty_sql
datablocksizeproperty_sql
blockcompressionproperty_sql
isolatedloadingproperty_sql
lockingproperty_sql
withdataproperty_sql
insert_sql
intersect_sql
intersect_op
introducer_sql
pseudotype_sql
onconflict_sql
returning_sql
rowformatdelimitedproperty_sql
table_sql
tablesample_sql
pivot_sql
tuple_sql
update_sql
values_sql
var_sql
into_sql
from_sql
group_sql
having_sql
join_sql
lambda_sql
lateral_sql
limit_sql
offset_sql
setitem_sql
set_sql
pragma_sql
lock_sql
literal_sql
loaddata_sql
null_sql
boolean_sql
order_sql
cluster_sql
distribute_sql
sort_sql
ordered_sql
matchrecognize_sql
query_modifiers
after_having_modifiers
select_sql
schema_sql
star_sql
parameter_sql
sessionparameter_sql
subquery_sql
qualify_sql
union_sql
union_op
unnest_sql
where_sql
window_sql
partition_by_sql
windowspec_sql
withingroup_sql
between_sql
bracket_sql
all_sql
any_sql
exists_sql
case_sql
constraint_sql
nextvaluefor_sql
extract_sql
trim_sql
concat_sql
check_sql
foreignkey_sql
primarykey_sql
unique_sql
if_sql
matchagainst_sql
jsonkeyvalue_sql
jsonobject_sql
openjsoncolumndef_sql
openjson_sql
in_sql
in_unnest_op
interval_sql
return_sql
reference_sql
anonymous_sql
paren_sql
neg_sql
not_sql
alias_sql
aliases_sql
attimezone_sql
add_sql
and_sql
connector_sql
bitwiseand_sql
bitwiseleftshift_sql
bitwisenot_sql
bitwiseor_sql
bitwiserightshift_sql
bitwisexor_sql
cast_sql
currentdate_sql
collate_sql
command_sql
comment_sql
mergetreettlaction_sql
mergetreettl_sql
transaction_sql
commit_sql
rollback_sql
altercolumn_sql
renametable_sql
altertable_sql
droppartition_sql
addconstraint_sql
distinct_sql
ignorenulls_sql
respectnulls_sql
intdiv_sql
dpipe_sql
div_sql
overlaps_sql
distance_sql
dot_sql
eq_sql
escape_sql
glob_sql
gt_sql
gte_sql
ilike_sql
ilikeany_sql
is_sql
like_sql
likeany_sql
similarto_sql
lt_sql
lte_sql
mod_sql
mul_sql
neq_sql
nullsafeeq_sql
nullsafeneq_sql
or_sql
slice_sql
sub_sql
trycast_sql
use_sql
binary
function_fallback_sql
func
format_args
text_width
format_time
expressions
op_expressions
naked_property
set_operation
tag_sql
token_sql
userdefinedfunction_sql
joinhint_sql
kwarg_sql
when_sql
merge_sql
tochar_sql