From dbf2028e1a50efbd0bc223fe3c38c5c5eaed65be Mon Sep 17 00:00:00 2001 From: Harshetjain Date: Tue, 28 Jan 2025 13:29:59 +0530 Subject: [PATCH] Add support for python 3.12 version --- example/qubole_presto_api_example.py | 2 +- qds_sdk/commands.py | 30 ++++++++++++++-------------- qds_sdk/role.py | 4 ++-- requirements.txt | 2 +- setup.py | 9 ++++++--- 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/example/qubole_presto_api_example.py b/example/qubole_presto_api_example.py index 6e9e24d6..111a1adb 100644 --- a/example/qubole_presto_api_example.py +++ b/example/qubole_presto_api_example.py @@ -9,7 +9,7 @@ from ConfigParser import SafeConfigParser from qds_sdk.qubole import Qubole from qds_sdk.commands import * -import boto +import boto3 import time diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 0966a9e2..6648d932 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -12,7 +12,7 @@ from qds_sdk.util import GentleOptionParser, OptionParsingError, OptionParsingExit, _is_cloud_url from optparse import SUPPRESS_HELP -import boto +import boto3 import time import logging import sys @@ -299,12 +299,12 @@ def get_results(self, fp=sys.stdout, inline=True, delim=None, fetch=True, qlog=N pass else: if fetch: - if not boto.config.has_section('s3'): - boto.config.add_section('s3') - boto.config.set('s3', 'use-sigv4', 'True') + if not boto3.config.has_section('s3'): + boto3.config.add_section('s3') + boto3.config.set('s3', 'use-sigv4', 'True') storage_credentials = conn.get(Account.credentials_rest_entity_path) host = storage_credentials['region_endpoint'] if storage_credentials['region_endpoint'] else "s3.amazonaws.com" - boto_conn = boto.connect_s3(aws_access_key_id=storage_credentials['storage_access_key'], + boto3_conn = boto3.connect_s3(aws_access_key_id=storage_credentials['storage_access_key'], aws_secret_access_key=storage_credentials['storage_secret_key'], security_token=storage_credentials['session_token'], host=host) @@ -320,11 +320,11 @@ def get_results(self, fp=sys.stdout, inline=True, delim=None, fetch=True, qlog=N # In Python 3, # If the delim is None, fp should be in binary mode because - # boto expects it to be. + # boto3 expects it to be. # If the delim is not None, then both text and binary modes # work. - _download_to_local(boto_conn, s3_path, fp, num_result_dir, delim=delim) + _download_to_local(boto3_conn, s3_path, fp, num_result_dir, delim=delim) else: fp.write(",".join(r['result_location'])) @@ -1121,7 +1121,7 @@ def parse(cls, args): raise ParseError("dbtap_id and db_table are required", cls.optparser.format_help()) - if options.mode is "1": + if options.mode == "1": if options.hive_table is None: raise ParseError("hive_table is required for mode 1", cls.optparser.format_help()) @@ -1134,7 +1134,7 @@ def parse(cls, args): raise ParseError("db_update_mode should either be left blank for append " "mode or be 'updateonly' or 'allowinsert'", cls.optparser.format_help()) - if options.db_update_mode is "updateonly": + if options.db_update_mode == "updateonly": if options.db_update_keys is None: raise ParseError("db_update_keys is required when db_update_mode " "is 'updateonly'", @@ -1501,12 +1501,12 @@ def write_headers(qlog,fp): fp.write(col_names.encode()) -def _download_to_local(boto_conn, s3_path, fp, num_result_dir, delim=None): +def _download_to_local(boto3_conn, s3_path, fp, num_result_dir, delim=None): ''' Downloads the contents of all objects in s3_path into fp Args: - `boto_conn`: S3 connection object + `boto3_conn`: S3 connection object `s3_path`: S3 path to be downloaded @@ -1521,7 +1521,7 @@ def _callback(downloaded, total): `total`: Total file size to be downloaded (int) ''' - if (total is 0) or (downloaded == total): + if (total == 0) or (downloaded == total): return progress = downloaded*100/total sys.stderr.write('\r[{0}] {1}%'.format('#'*progress, progress)) @@ -1529,7 +1529,7 @@ def _callback(downloaded, total): m = _URI_RE.match(s3_path) bucket_name = m.group(1) - bucket = boto_conn.get_bucket(bucket_name) + bucket = boto3_conn.get_bucket(bucket_name) retries = 6 if s3_path.endswith('/') is False: #It is a file @@ -1546,9 +1546,9 @@ def _callback(downloaded, total): if delim is None: try: key_instance.get_contents_to_file(fp) # cb=_callback - except boto.exception.S3ResponseError as e: + except boto3.exception.S3ResponseError as e: if (e.status == 403): - # SDK-191, boto gives an error while fetching the objects using versions which happens by default + # SDK-191, boto3 gives an error while fetching the objects using versions which happens by default # in the get_contents_to_file() api. So attempt one without specifying version. log.warn("Access denied while fetching the s3 object. Retrying without specifying the version....") key_instance.open() diff --git a/qds_sdk/role.py b/qds_sdk/role.py index ad7fc295..893f86cb 100644 --- a/qds_sdk/role.py +++ b/qds_sdk/role.py @@ -20,7 +20,7 @@ def parsers(): create = subparsers.add_parser("create", help="Create a new Role") create.add_argument("--name", dest="name", required=True, help="Name of the new Role") create.add_argument("--policy", dest="policy", required=True, - help="Policy Statement example '[{\"access\":\"deny\", \"resource\": \"all\", \"action\": \"[\"create\",\"update\",\"delete\"\]\"}]'") + help="Policy Statement example '[{\"access\":\"deny\", \"resource\": \"all\", \"action\": \"[\"create\",\"update\",\"delete\"]\"}]'") create.set_defaults(func=RoleCmdLine.create) #List @@ -39,7 +39,7 @@ def parsers(): update.add_argument("id", help="Numeric id of the Role") update.add_argument("--name", dest="name", help="New name of the Role") update.add_argument("--policy", dest="policy", - help="Policy Statement example '[{\"access\":\"deny\", \"resource\": \"all\", \"action\": \"[\"create\",\"update\",\"delete\"\]\"}]'") + help="Policy Statement example '[{\"access\":\"deny\", \"resource\": \"all\", \"action\": \"[\"create\",\"update\",\"delete\"]\"}]'") update.set_defaults(func=RoleCmdLine.update) #Delete diff --git a/requirements.txt b/requirements.txt index 54393b3e..c7dc351a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -boto==2.45.0 +boto3==1.15.0 certifi==2019.6.16 chardet==3.0.4 idna==2.8 diff --git a/setup.py b/setup.py index cc3b343d..ad89459e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import sys from setuptools import setup -INSTALL_REQUIRES = ['requests >=2.21.0', 'boto >=2.45.0', 'six >=1.12.0', +INSTALL_REQUIRES = ['requests >=2.21.0', 'boto3 >=1.15.0', 'six >=1.12.0', 'urllib3 >= 1.24.3'] if sys.version_info < (2, 7, 0): INSTALL_REQUIRES.append('argparse>=1.1') @@ -17,7 +17,7 @@ def read(fname): setup( name="qds_sdk", - version="1.13.2", + version="1.13.3", author="Qubole", author_email="dev@qubole.com", description=("Python SDK for coding to the Qubole Data Service API"), @@ -38,6 +38,9 @@ def read(fname): "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4" + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12" ] )