Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

new:dev:SDK-387:Add hive version support #303

Merged
merged 32 commits into from
Apr 1, 2020
Merged

new:dev:SDK-387:Add hive version support #303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
95fb8e2
add hive version option
Feb 17, 2020
9f7e48a
indentation
Feb 17, 2020
336c0ac
change
Feb 17, 2020
5bf32a0
add space as build is failing
Feb 17, 2020
cbe297a
resolve conflicts
Feb 17, 2020
682135d
resolve conflicts
Feb 17, 2020
286bf21
Merge branch 'unreleased' into SDK-387
satyabolnedi Feb 17, 2020
4c4db16
indentation
Feb 17, 2020
c7ad46d
Merge branch 'SDK-387' of https://github.com/satyabolnedi/qds-sdk-py …
Feb 17, 2020
144e110
space
Feb 17, 2020
78664aa
space
Feb 17, 2020
b8bcaf6
space
Feb 17, 2020
6dd29db
space
Feb 17, 2020
fb910a6
space
Feb 17, 2020
bb61935
Merge branch 'unreleased' into SDK-387
satyabolnedi Feb 21, 2020
3fbfb8e
change
Mar 6, 2020
d50a793
Merge branch 'SDK-387' of https://github.com/satyabolnedi/qds-sdk-py …
Mar 6, 2020
aa0e0a7
change
Mar 30, 2020
1a859a8
Merge branch 'unreleased' into SDK-387
satyabolnedi Mar 30, 2020
d093057
change
Mar 30, 2020
14b181b
change
Mar 30, 2020
fe79f38
change
Mar 30, 2020
0ce9e2c
Merge branch 'unreleased' into SDK-387
satyabolnedi Mar 30, 2020
7373279
change
Mar 30, 2020
94d7c9c
Merge branch 'SDK-387' of https://github.com/satyabolnedi/qds-sdk-py …
Mar 30, 2020
181dd76
change
Mar 30, 2020
b743e39
change
Mar 30, 2020
f18259f
change
Mar 30, 2020
f063773
change
Mar 30, 2020
9cc18c4
change
Mar 30, 2020
b35d595
change
Mar 30, 2020
4638f53
change
Mar 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion qds_sdk/engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ class Engine:
def __init__(self, flavour=None):
self.flavour = flavour
self.hadoop_settings = {}
self.hive_settings = {}
self.presto_settings = {}
self.spark_settings = {}
self.airflow_settings = {}
Expand All @@ -26,6 +27,7 @@ def set_engine_config(self,
custom_presto_config=None,
spark_version=None,
custom_spark_config=None,
hive_version=None,
dbtap_id=None,
fernet_key=None,
overrides=None,
Expand Down Expand Up @@ -56,6 +58,8 @@ def set_engine_config(self,

custom_spark_config: Specify the custom Spark configuration overrides

hive_version: Version of hive to be used in cluster

dbtap_id: ID of the data store inside QDS

fernet_key: Encryption key for sensitive information inside airflow database.
Expand All @@ -75,8 +79,10 @@ def set_engine_config(self,

'''

self.set_hadoop_settings(custom_hadoop_config, use_qubole_placement_policy, is_ha, fairscheduler_config_xml,
self.set_hadoop_settings(custom_hadoop_config, use_qubole_placement_policy,
is_ha, fairscheduler_config_xml,
default_pool, enable_rubix)
self.set_hive_settings(hive_version)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Verify the value of hive_version column in hive_cluster_configs with and without this change.

Copy link
Contributor Author

@satyabolnedi satyabolnedi Mar 30, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tarrygo verified from DB with the updated sdk changes and hive version being updated with 3.1

self.set_presto_settings(presto_version, custom_presto_config)
self.set_spark_settings(spark_version, custom_spark_config)
self.set_airflow_settings(dbtap_id, fernet_key, overrides, airflow_version, airflow_python_version)
Expand All @@ -103,6 +109,10 @@ def set_hadoop_settings(self,
self.set_fairscheduler_settings(fairscheduler_config_xml, default_pool)
self.hadoop_settings['enable_rubix'] = enable_rubix

def set_hive_settings(self,
hive_version=None):
self.hive_settings['hive_version'] = hive_version

def set_presto_settings(self,
presto_version=None,
custom_presto_config=None):
Expand Down Expand Up @@ -147,6 +157,7 @@ def set_engine_config_settings(self, arguments):
custom_presto_config=custom_presto_config,
spark_version=arguments.spark_version,
custom_spark_config=arguments.custom_spark_config,
hive_version=arguments.hive_version,
dbtap_id=arguments.dbtap_id,
fernet_key=arguments.fernet_key,
overrides=arguments.overrides,
Expand Down Expand Up @@ -218,6 +229,11 @@ def engine_parser(argparser):
dest="presto_custom_config_file",
help="location of file containg custom" +
" presto configuration overrides")
hive_settings_group = argparser.add_argument_group("hive version settings")
hive_settings_group.add_argument("--hive_version",
dest="hive_version",
default=None,
help="Version of hive for the cluster",)

spark_settings_group = argparser.add_argument_group("spark settings")
spark_settings_group.add_argument("--spark-version",
Expand Down
20 changes: 20 additions & 0 deletions tests/test_clusterv2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -562,6 +562,26 @@ def test_mlflow_engine_config(self):
}},
'cluster_info': {'label': ['test_label'], }})

def test_hive_engine_config(self):
with tempfile.NamedTemporaryFile() as temp:
temp.write("config.properties:\na=1\nb=2".encode("utf8"))
temp.flush()
sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label',
'--flavour', 'hadoop2', '--hive_version', '2.3']
Qubole.cloud = None
print_command()
Connection._api_call = Mock(return_value={})
qds.main()
Connection._api_call.assert_called_with('POST', 'clusters',
{'engine_config':
{'flavour': 'hadoop2',
'hive_settings': {
'hive_version': '2.3'
}},
'cluster_info': {'label': ['test_label'],}})



def test_persistent_security_groups_v2(self):
sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label',
'--persistent-security-groups', 'sg1, sg2']
Expand Down