From 3f1a1df7043ed6bd19ee3e751226d4ea4cd324d9 Mon Sep 17 00:00:00 2001 From: Aleandre Ouellet Date: Thu, 18 Oct 2018 08:24:56 -0400 Subject: [PATCH 01/15] Add minimal support for querying multiple catalogs --- pyhive/sqlalchemy_presto.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pyhive/sqlalchemy_presto.py b/pyhive/sqlalchemy_presto.py index a85ebff1..14307c4d 100644 --- a/pyhive/sqlalchemy_presto.py +++ b/pyhive/sqlalchemy_presto.py @@ -12,6 +12,7 @@ from sqlalchemy import exc from sqlalchemy import types from sqlalchemy import util +from sqlalchemy import Table # TODO shouldn't use mysql type from sqlalchemy.databases import mysql from sqlalchemy.engine import default @@ -46,6 +47,27 @@ class PrestoCompiler(SQLCompiler): def visit_char_length_func(self, fn, **kw): return 'length{}'.format(self.function_argspec(fn, **kw)) + def visit_column(self, column, add_to_result_map=None, include_table=True, **kwargs): + sql = super(PrestoCompiler, self).visit_column(column, add_to_result_map, include_table, **kwargs) + table = column.table + return self.__add_catalog(sql, table) + + def visit_table(self, table, asfrom=False, iscrud=False, ashint=False, + fromhints=None, use_schema=True, **kwargs): + sql = super(PrestoCompiler, self).visit_table(table, asfrom, iscrud, ashint, fromhints, use_schema, **kwargs) + return self.__add_catalog(sql, table) + + def __add_catalog(self, sql:str, table: Table) -> str: + if table is None: + return sql + + if "presto" not in table.dialect_options or "catalog" not in table.dialect_options["presto"]._non_defaults: + return sql + + catalog = table.dialect_options["presto"]._non_defaults["catalog"] + sql = f"\"{catalog}\".{sql}" + return sql + class PrestoTypeCompiler(compiler.GenericTypeCompiler): def visit_CLOB(self, type_, **kw): From ce63d813bef73233c37b2556f09f72b0b19af18a Mon Sep 17 00:00:00 2001 From: Aleandre Ouellet Date: Thu, 25 Oct 2018 10:51:59 -0400 Subject: [PATCH 02/15] Add support for date and timestamp in presto --- pyhive/presto.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pyhive/presto.py b/pyhive/presto.py index e90d5bfa..70f5ad3d 100644 --- a/pyhive/presto.py +++ b/pyhive/presto.py @@ -17,6 +17,7 @@ import getpass import logging import requests +import datetime from requests.auth import HTTPBasicAuth try: # Python 3 @@ -31,8 +32,23 @@ paramstyle = 'pyformat' # Python extended format codes, e.g. ...WHERE name=%(name)s _logger = logging.getLogger(__name__) -_escaper = common.ParamEscaper() +class PrestoParamEscaper(common.ParamEscaper): + def escape_item(self, item): + if isinstance(item, datetime.datetime): + return self.escape_datetime(item) + elif isinstance(item, datetime.date): + return self.escape_date(item) + else: + return super().escape_item(item) + + def escape_date(self, item: datetime.date): + return f"date '{item}'" + + def escape_datetime(self, item: datetime.datetime): + return f"timestamp '{item}'" + +_escaper = PrestoParamEscaper() def connect(*args, **kwargs): """Constructor for creating a connection to the database. See class :py:class:`Connection` for From 5d87aec22b2b403b5dc6daba65f98f54c8108517 Mon Sep 17 00:00:00 2001 From: Aleandre Ouellet Date: Mon, 29 Oct 2018 10:35:55 -0400 Subject: [PATCH 03/15] Fix an issue with aliases --- pyhive/sqlalchemy_presto.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyhive/sqlalchemy_presto.py b/pyhive/sqlalchemy_presto.py index 14307c4d..69aa5138 100644 --- a/pyhive/sqlalchemy_presto.py +++ b/pyhive/sqlalchemy_presto.py @@ -61,6 +61,10 @@ def __add_catalog(self, sql:str, table: Table) -> str: if table is None: return sql + if isinstance(table, Alias): + return sql + + if "presto" not in table.dialect_options or "catalog" not in table.dialect_options["presto"]._non_defaults: return sql From b27cc42eb34d0c3594f2f3fcfeec91647dc9b18f Mon Sep 17 00:00:00 2001 From: Aleandre Ouellet Date: Mon, 29 Oct 2018 10:40:58 -0400 Subject: [PATCH 04/15] Fix import of FromClause and Alias --- pyhive/sqlalchemy_presto.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyhive/sqlalchemy_presto.py b/pyhive/sqlalchemy_presto.py index 69aa5138..659c8d36 100644 --- a/pyhive/sqlalchemy_presto.py +++ b/pyhive/sqlalchemy_presto.py @@ -13,11 +13,14 @@ from sqlalchemy import types from sqlalchemy import util from sqlalchemy import Table + # TODO shouldn't use mysql type from sqlalchemy.databases import mysql from sqlalchemy.engine import default from sqlalchemy.sql import compiler from sqlalchemy.sql.compiler import SQLCompiler +from sqlalchemy.sql.selectable import FromClause +from sqlalchemy.sql.expression import Alias from pyhive import presto from pyhive.common import UniversalSet @@ -57,14 +60,13 @@ def visit_table(self, table, asfrom=False, iscrud=False, ashint=False, sql = super(PrestoCompiler, self).visit_table(table, asfrom, iscrud, ashint, fromhints, use_schema, **kwargs) return self.__add_catalog(sql, table) - def __add_catalog(self, sql:str, table: Table) -> str: + def __add_catalog(self, sql:str, table: FromClause) -> str: if table is None: return sql if isinstance(table, Alias): return sql - if "presto" not in table.dialect_options or "catalog" not in table.dialect_options["presto"]._non_defaults: return sql From 9091b4deeda30ff1547caa0927c4ae5ce37a274f Mon Sep 17 00:00:00 2001 From: Alexandre Ouellet Date: Mon, 2 Mar 2020 20:46:32 -0500 Subject: [PATCH 05/15] reformat with flake8's standard of 100 line --- pyhive/presto.py | 3 +++ pyhive/sqlalchemy_presto.py | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/pyhive/presto.py b/pyhive/presto.py index 70f5ad3d..e5756024 100644 --- a/pyhive/presto.py +++ b/pyhive/presto.py @@ -33,6 +33,7 @@ _logger = logging.getLogger(__name__) + class PrestoParamEscaper(common.ParamEscaper): def escape_item(self, item): if isinstance(item, datetime.datetime): @@ -48,8 +49,10 @@ def escape_date(self, item: datetime.date): def escape_datetime(self, item: datetime.datetime): return f"timestamp '{item}'" + _escaper = PrestoParamEscaper() + def connect(*args, **kwargs): """Constructor for creating a connection to the database. See class :py:class:`Connection` for arguments. diff --git a/pyhive/sqlalchemy_presto.py b/pyhive/sqlalchemy_presto.py index 659c8d36..aea652fb 100644 --- a/pyhive/sqlalchemy_presto.py +++ b/pyhive/sqlalchemy_presto.py @@ -12,7 +12,6 @@ from sqlalchemy import exc from sqlalchemy import types from sqlalchemy import util -from sqlalchemy import Table # TODO shouldn't use mysql type from sqlalchemy.databases import mysql @@ -51,28 +50,35 @@ def visit_char_length_func(self, fn, **kw): return 'length{}'.format(self.function_argspec(fn, **kw)) def visit_column(self, column, add_to_result_map=None, include_table=True, **kwargs): - sql = super(PrestoCompiler, self).visit_column(column, add_to_result_map, include_table, **kwargs) + sql = super(PrestoCompiler, self).visit_column( + column, add_to_result_map, include_table, **kwargs + ) table = column.table return self.__add_catalog(sql, table) def visit_table(self, table, asfrom=False, iscrud=False, ashint=False, fromhints=None, use_schema=True, **kwargs): - sql = super(PrestoCompiler, self).visit_table(table, asfrom, iscrud, ashint, fromhints, use_schema, **kwargs) + sql = super(PrestoCompiler, self).visit_table( + table, asfrom, iscrud, ashint, fromhints, use_schema, **kwargs + ) return self.__add_catalog(sql, table) - def __add_catalog(self, sql:str, table: FromClause) -> str: + def __add_catalog(self, sql: str, table: FromClause) -> str: if table is None: return sql if isinstance(table, Alias): return sql - if "presto" not in table.dialect_options or "catalog" not in table.dialect_options["presto"]._non_defaults: + if ( + "presto" not in table.dialect_options + or "catalog" not in table.dialect_options["presto"]._non_defaults + ): return sql catalog = table.dialect_options["presto"]._non_defaults["catalog"] sql = f"\"{catalog}\".{sql}" - return sql + return sql class PrestoTypeCompiler(compiler.GenericTypeCompiler): From c4534d7cde73ecb45718ec2178b91bccabcab6d6 Mon Sep 17 00:00:00 2001 From: Alexandre Ouellet Date: Mon, 2 Mar 2020 20:54:56 -0500 Subject: [PATCH 06/15] use old str.format() for string for python 3.5 --- pyhive/presto.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyhive/presto.py b/pyhive/presto.py index e5756024..18994962 100644 --- a/pyhive/presto.py +++ b/pyhive/presto.py @@ -43,11 +43,11 @@ def escape_item(self, item): else: return super().escape_item(item) - def escape_date(self, item: datetime.date): - return f"date '{item}'" + def escape_date(self, item): + return "date '{}'".format(item) - def escape_datetime(self, item: datetime.datetime): - return f"timestamp '{item}'" + def escape_datetime(self, item): + return "timestamp '{}'".format(item) _escaper = PrestoParamEscaper() From 2d32708d3442452d82c1fae5c5ed1038f5a2fe1c Mon Sep 17 00:00:00 2001 From: Vince Welke Date: Mon, 2 Mar 2020 18:28:50 -0800 Subject: [PATCH 07/15] fix syntax to be compatible with python 3.5 --- pyhive/sqlalchemy_presto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyhive/sqlalchemy_presto.py b/pyhive/sqlalchemy_presto.py index aea652fb..e4218150 100644 --- a/pyhive/sqlalchemy_presto.py +++ b/pyhive/sqlalchemy_presto.py @@ -77,7 +77,7 @@ def __add_catalog(self, sql: str, table: FromClause) -> str: return sql catalog = table.dialect_options["presto"]._non_defaults["catalog"] - sql = f"\"{catalog}\".{sql}" + sql = "\"{catalog}\".{sql}".format(catalog=catalog, sql=sql) return sql From c77de4eeffd203d754fdca09f036b04b37e7cebd Mon Sep 17 00:00:00 2001 From: Vince Welke Date: Mon, 2 Mar 2020 19:12:29 -0800 Subject: [PATCH 08/15] old python 2.7 syntax update --- pyhive/presto.py | 2 +- pyhive/sqlalchemy_presto.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyhive/presto.py b/pyhive/presto.py index 18994962..4ee8489d 100644 --- a/pyhive/presto.py +++ b/pyhive/presto.py @@ -41,7 +41,7 @@ def escape_item(self, item): elif isinstance(item, datetime.date): return self.escape_date(item) else: - return super().escape_item(item) + return super(PrestoParamEscaper, self).escape_item(item) def escape_date(self, item): return "date '{}'".format(item) diff --git a/pyhive/sqlalchemy_presto.py b/pyhive/sqlalchemy_presto.py index e4218150..95ec4814 100644 --- a/pyhive/sqlalchemy_presto.py +++ b/pyhive/sqlalchemy_presto.py @@ -63,7 +63,7 @@ def visit_table(self, table, asfrom=False, iscrud=False, ashint=False, ) return self.__add_catalog(sql, table) - def __add_catalog(self, sql: str, table: FromClause) -> str: + def __add_catalog(self, sql, table): if table is None: return sql From a6c1bf8e0406c208cd4287881d50850b38e6efe8 Mon Sep 17 00:00:00 2001 From: AlexandreOuellet Date: Tue, 3 Mar 2020 09:48:17 -0500 Subject: [PATCH 09/15] remove now-useless FromClause import --- pyhive/sqlalchemy_presto.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyhive/sqlalchemy_presto.py b/pyhive/sqlalchemy_presto.py index 95ec4814..a9823f4b 100644 --- a/pyhive/sqlalchemy_presto.py +++ b/pyhive/sqlalchemy_presto.py @@ -18,7 +18,6 @@ from sqlalchemy.engine import default from sqlalchemy.sql import compiler from sqlalchemy.sql.compiler import SQLCompiler -from sqlalchemy.sql.selectable import FromClause from sqlalchemy.sql.expression import Alias from pyhive import presto From 7e44440a228f8a755298c3ce1dbcd86d17295b9f Mon Sep 17 00:00:00 2001 From: Laser Kaplan Date: Fri, 11 Mar 2022 11:38:05 -0800 Subject: [PATCH 10/15] Check for later version of SQLAlchemy --- pyhive/sqlalchemy_presto.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyhive/sqlalchemy_presto.py b/pyhive/sqlalchemy_presto.py index 1830b932..c1e79470 100644 --- a/pyhive/sqlalchemy_presto.py +++ b/pyhive/sqlalchemy_presto.py @@ -8,10 +8,12 @@ from __future__ import absolute_import from __future__ import unicode_literals +from packaging import version import re from sqlalchemy import exc from sqlalchemy import types from sqlalchemy import util +import sys # TODO shouldn't use mysql type from sqlalchemy.databases import mysql @@ -116,6 +118,8 @@ class PrestoDialect(default.DefaultDialect): returns_unicode_strings = True description_encoding = None supports_native_boolean = True + if version.parse(sys.modules['sqlalchemy'].__version__) >= version.parse('1.4.5'): + supports_statement_cache = False type_compiler = PrestoTypeCompiler @classmethod From bbb491744ec4e17de2e8201b7c225b20caff0f08 Mon Sep 17 00:00:00 2001 From: Laser Kaplan Date: Fri, 11 Mar 2022 11:38:28 -0800 Subject: [PATCH 11/15] Fix test after Decimal change --- pyhive/tests/test_sqlalchemy_presto.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyhive/tests/test_sqlalchemy_presto.py b/pyhive/tests/test_sqlalchemy_presto.py index a01e4a35..0ca28559 100644 --- a/pyhive/tests/test_sqlalchemy_presto.py +++ b/pyhive/tests/test_sqlalchemy_presto.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals from builtins import str +from decimal import Decimal from pyhive.tests.sqlalchemy_test_case import SqlAlchemyTestCase from pyhive.tests.sqlalchemy_test_case import with_engine_connection from sqlalchemy import types @@ -48,7 +49,7 @@ def test_reflect_select(self, engine, connection): {"1": 2, "3": 4}, # Presto converts all keys to strings so that they're valid JSON [1, 2], # struct is returned as a list of elements # '{0:1}', - '0.1', + Decimal('0.1'), ]) # TODO some of these types could be filled in better From e6f8b870514593bb083b7a4974f52665f332ee38 Mon Sep 17 00:00:00 2001 From: Laser Kaplan Date: Fri, 11 Mar 2022 11:38:41 -0800 Subject: [PATCH 12/15] Add new test for changing Presto catalog --- pyhive/tests/test_sqlalchemy_presto.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pyhive/tests/test_sqlalchemy_presto.py b/pyhive/tests/test_sqlalchemy_presto.py index 0ca28559..2b339fe7 100644 --- a/pyhive/tests/test_sqlalchemy_presto.py +++ b/pyhive/tests/test_sqlalchemy_presto.py @@ -86,3 +86,16 @@ def test_reserved_words(self, engine, connection): self.assertIn('"current_timestamp"', query) self.assertNotIn('`select`', query) self.assertNotIn('`current_timestamp`', query) + + @with_engine_connection + def test_multiple_catalogs(self, engine, connection): + system_table = Table( + 'tables', + MetaData(bind=engine), + autoload=True, + schema='information_schema', + presto_catalog='system' + ) + query = str(system_table.select()) + self.assertIn('"system"."information_schema"', query) + self.assertNotIn('"hive"', query) From 75553d063403831544bb59e8214d4ce49952a954 Mon Sep 17 00:00:00 2001 From: Laser Kaplan Date: Wed, 6 Apr 2022 13:07:57 -0700 Subject: [PATCH 13/15] Update for handling SQLA 1.4 changes to selectables --- pyhive/sqlalchemy_presto.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyhive/sqlalchemy_presto.py b/pyhive/sqlalchemy_presto.py index c1e79470..fcca57ab 100644 --- a/pyhive/sqlalchemy_presto.py +++ b/pyhive/sqlalchemy_presto.py @@ -20,7 +20,11 @@ from sqlalchemy.engine import default from sqlalchemy.sql import compiler from sqlalchemy.sql.compiler import SQLCompiler -from sqlalchemy.sql.expression import Alias +from sqlalchemy.sql.expression import ( + Alias, + CTE, + Subquery, +) from pyhive import presto from pyhive.common import UniversalSet @@ -68,7 +72,7 @@ def __add_catalog(self, sql, table): if table is None: return sql - if isinstance(table, Alias): + if isinstance(table, (Alias, CTE, Subquery)): return sql if ( From 5f8a9cd98dd99d98e4f5c36215d72ebe7d1a8b28 Mon Sep 17 00:00:00 2001 From: Laser Kaplan Date: Thu, 7 Apr 2022 14:04:33 -0700 Subject: [PATCH 14/15] Fix for imports for SQLA 1.4 --- pyhive/sqlalchemy_presto.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pyhive/sqlalchemy_presto.py b/pyhive/sqlalchemy_presto.py index fcca57ab..dcd2f2f9 100644 --- a/pyhive/sqlalchemy_presto.py +++ b/pyhive/sqlalchemy_presto.py @@ -20,11 +20,16 @@ from sqlalchemy.engine import default from sqlalchemy.sql import compiler from sqlalchemy.sql.compiler import SQLCompiler -from sqlalchemy.sql.expression import ( - Alias, - CTE, - Subquery, -) +try: + from sqlalchemy.sql.expression import ( + Alias, + CTE, + Subquery, + ) +except ImportError: + from sqlalchemy.sql.expression import Alias + CTE = type(None) + Subquery = type(None) from pyhive import presto from pyhive.common import UniversalSet From 8980976dc6f67f6f56845ff72c732d60d26ad253 Mon Sep 17 00:00:00 2001 From: Laser Kaplan Date: Thu, 7 Apr 2022 14:04:56 -0700 Subject: [PATCH 15/15] Enable important cte_follows_insert flag in dialect since Presto expects queries to be formatted this way --- pyhive/sqlalchemy_presto.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyhive/sqlalchemy_presto.py b/pyhive/sqlalchemy_presto.py index dcd2f2f9..891fa0ab 100644 --- a/pyhive/sqlalchemy_presto.py +++ b/pyhive/sqlalchemy_presto.py @@ -130,6 +130,7 @@ class PrestoDialect(default.DefaultDialect): if version.parse(sys.modules['sqlalchemy'].__version__) >= version.parse('1.4.5'): supports_statement_cache = False type_compiler = PrestoTypeCompiler + cte_follows_insert = True @classmethod def dbapi(cls):