From c46125e4b64cb39c3c35b1521b2fe8edb3ccbceb Mon Sep 17 00:00:00 2001 From: Tanish Gupta Date: Mon, 27 Jun 2016 11:29:46 -0700 Subject: [PATCH 1/2] fix: dev: SDK-121: Add branding changes (#159) Squashed commit of the following: commit a4cfc6bfa1c51d0affc5f353bc5116e6880603e9 commit 259b80630d4f342d3422847e7327fd30d9d0b741 commit f492883528708a1125cd07624579ca3f098b85f0 --- qds_sdk/account.py | 34 +++++++++++++++++++++++++++++ tests/test_account.py | 51 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/qds_sdk/account.py b/qds_sdk/account.py index c604bf20..560521d0 100644 --- a/qds_sdk/account.py +++ b/qds_sdk/account.py @@ -43,6 +43,15 @@ def parsers(): choices=["true", "false"], default="false", help="Use previous account plan, default: false") create.set_defaults(func=AccountCmdLine.create) + + #branding + branding = subparsers.add_parser( + "branding", help="Branding logo and link") + branding.add_argument("--account-id", dest = "account_id", required=True, help = "Account ID of the Qubole account for which branding has to be done") + branding.add_argument("--logo-uri", dest = "logo_uri", help = "Publicly accessible logo URI image in jpg/gif/svg/jpeg format.Image size must be less than 100 KB.") + branding.add_argument("--link-url", dest = "link_url", help = "Specify the documentation URL.") + branding.add_argument("--link-label", dest = "link_label", help = "Add a label to describe the documentation URL.") + branding.set_defaults(func=AccountCmdLine.branding) return argparser @staticmethod @@ -64,6 +73,25 @@ def create(args): result = Account.create(**v) return result + @staticmethod + def branding(args): + v= {} + print(args) + v['account_id'] = args.account_id + if args.logo_uri is not None: + v['logo'] = {'logo_uri' : args.logo_uri } + + link = {} + if args.link_url is not None: + link['link_url'] = args.link_url + if args.link_label is not None: + link['link_label'] = args.link_label + + if bool(link): + v['link'] = link + + result = Account.branding(**v) + return result class Account(SingletonResource): credentials_rest_entity_path = "accounts/get_creds" @@ -73,3 +101,9 @@ class Account(SingletonResource): def create(cls, **kwargs): conn = Qubole.agent() return cls(conn.post(cls.rest_entity_path, data=kwargs)) + + @classmethod + def branding(cls, **kwargs): + conn = Qubole.agent() + url_path = "accounts/branding" + return cls(conn.put(url_path, data=kwargs)) diff --git a/tests/test_account.py b/tests/test_account.py index 60b1bebd..d1e7e5e6 100644 --- a/tests/test_account.py +++ b/tests/test_account.py @@ -183,5 +183,56 @@ def test_default_previous_account_plan(self): 'defloc': 's3://bucket/path'}}) +class TestAccountBranding(QdsCliTestCase): + def test_logo(self): + sys.argv = ['qds.py', 'account', 'branding', + '--account-id', '4', + '--logo-uri', 'https://www.xyz.com/image.jpg'] + print_command() + Connection._api_call = Mock(return_value={}) + qds.main() + Connection._api_call.assert_called_with("PUT", "accounts/branding", {'logo': { + 'logo_uri' : 'https://www.xyz.com/image.jpg'}, + 'account_id' : '4'}) + + def test_link(self): + sys.argv = ['qds.py', 'account', 'branding', + '--account-id', '4', + '--link-url', 'https://www.xyz.com', + '--link-label', 'Documentation'] + print_command() + Connection._api_call = Mock(return_value={}) + qds.main() + Connection._api_call.assert_called_with("PUT", "accounts/branding", {'link': { + 'link_url' : 'https://www.xyz.com', + 'link_label' : 'Documentation'}, + 'account_id' : '4'}) + + def test_logo_link(self): + sys.argv = ['qds.py', 'account', 'branding', + '--account-id', '4', + '--logo-uri', 'https://www.xyz.com/image.jpg', + '--link-url', 'https://www.xyz.com', + '--link-label', 'Documentation'] + print_command() + Connection._api_call = Mock(return_value={}) + qds.main() + Connection._api_call.assert_called_with("PUT", "accounts/branding", {'logo': { + 'logo_uri' : 'https://www.xyz.com/image.jpg'}, + 'link': {'link_url' : 'https://www.xyz.com', + 'link_label' : 'Documentation'}, + 'account_id' : '4'}) + + def test_without_account_id(self): + sys.argv = ['qds.py', 'account', 'branding', + '--logo-uri', 'https://www.xyz.com/image.jpg', + '--link-url', 'https://www.xyz.com', + '--link-label', 'Documentation'] + print_command() + Connection._api_call = Mock(return_value={}) + with self.assertRaises(SystemExit): + qds.main() + + if __name__ == '__main__': unittest.main() From 83ec6b30862843acb38d966687111a89d15414ab Mon Sep 17 00:00:00 2001 From: Sumit Maheshwari Date: Tue, 28 Jun 2016 10:58:58 -0700 Subject: [PATCH 2/2] fix: dev: SDK-117: Live logging for command run option (#117) Add changes for printing the logs live while running the commands. Squashed commit of the following: commit dc3e08eea1cb6c1cee0d4e529a7480dac863373e commit 60c6c08fe3c65b94af8e40f109414e7bfc01a0ac commit b927745902c222669f16577ce88bb332d4c094de commit d78a8890d3e43147aad96e660f4cc1221eb1ab9a commit 1ed0cfd8937b62f3480e2465ae704606272bcf63 commit 259b80630d4f342d3422847e7327fd30d9d0b741 commit f492883528708a1125cd07624579ca3f098b85f0 --- bin/qds.py | 1 + qds_sdk/commands.py | 61 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/bin/qds.py b/bin/qds.py index 08301349..94eae585 100755 --- a/bin/qds.py +++ b/bin/qds.py @@ -107,6 +107,7 @@ def submitaction(cmdclass, args): args = cmdclass.parse(args) if args is not None: args.pop("print_logs") # This is only useful while using the 'run' action. + args.pop("print_logs_live") # This is only useful while using the 'run' action. cmd = cmdclass.create(**args) print("Submitted %s, Id: %s" % (cmdclass.__name__, cmd.id)) return 0 diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index d09e9fc7..247fbe68 100644 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -4,6 +4,7 @@ the specific commands """ +from __future__ import print_function from qds_sdk.qubole import Qubole from qds_sdk.resource import Resource from qds_sdk.exception import ParseError @@ -12,8 +13,8 @@ from qds_sdk.util import OptionParsingError from qds_sdk.util import OptionParsingExit from optparse import SUPPRESS_HELP -import boto +import boto import time import logging import sys @@ -87,10 +88,30 @@ def run(cls, **kwargs): Returns: Command object """ + + # vars to keep track of actual logs bytes (err, tmp) and new bytes seen in each iteration + err_pointer, tmp_pointer, new_bytes = 0, 0, 0 + print_logs_live = kwargs.pop("print_logs_live") # We don't want to send this to the API. + cmd = cls.create(**kwargs) while not Command.is_done(cmd.status): time.sleep(Qubole.poll_interval) cmd = cls.find(cmd.id) + if print_logs_live is True: + log, err_length, tmp_length = cmd.get_log_partial(err_pointer, tmp_pointer) + + # if err length is non zero, then tmp_pointer needs to be reset to the current tmp_length as the + # err_length will contain the full set of logs from last seen non-zero err_length. + if err_length != "0": + err_pointer += int(err_length) + new_bytes = int(err_length) + int(tmp_length) - tmp_pointer + tmp_pointer = int(tmp_length) + else: + tmp_pointer += int(tmp_length) + new_bytes = int(tmp_length) + + if len(log) > 0 and new_bytes > 0: + print(log[-new_bytes:], file=sys.stderr) return cmd @@ -136,6 +157,23 @@ def get_log(self): r = conn.get_raw(log_path) return r.text + def get_log_partial(self, err_pointer=0, tmp_pointer=0): + """ + Fetches log (full or partial) for the command represented by this object + Accepts: + err_pointer(int): Pointer to err text bytes we've received so far, which will be passed to next api call + to indicate pointer to fetch logs. + tmp_pointer(int): Same as err_pointer except it indicates the bytes of tmp file processed. + Returns: + An array where the first field is actual log (string), while 2nd & 3rd are counts of err and tmp bytes + which have been returned by api in addition to the given pointers. + """ + log_path = self.meta_data['logs_resource'] + conn = Qubole.agent() + r = conn.get_raw(log_path, params={'err_file_processed':err_pointer, 'tmp_file_processed':tmp_pointer}) + if 'err_length' in r.headers.keys() and 'tmp_length' in r.headers.keys(): + return [r.text, r.headers['err_length'], r.headers['tmp_length']] + return [r.text, 0, 0] @classmethod def get_jobs_id(cls, id): @@ -243,6 +281,8 @@ class HiveCommand(Command): optparser.add_option("--print-logs", action="store_true", dest="print_logs", default=False, help="Fetch logs and print them to stderr.") + optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", + default=False, help="Fetch logs and print them to stderr while command is running.") optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job") @classmethod @@ -328,6 +368,8 @@ class SqlCommand(Command): optparser.add_option("--print-logs", action="store_true", dest="print_logs", default=False, help="Fetch logs and print them to stderr.") + optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", + default=False, help="Fetch logs and print them to stderr while command is running.") @classmethod def parse(cls, args): @@ -419,6 +461,8 @@ class SparkCommand(Command): optparser.add_option("--print-logs", action="store_true", dest="print_logs", default=False, help="Fetch logs and print them to stderr.") + optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", + default=False, help="Fetch logs and print them to stderr while command is running.") optparser.add_option("--retry", dest="retry", default=0, help="Number of retries") @classmethod @@ -576,6 +620,8 @@ class PrestoCommand(Command): optparser.add_option("--print-logs", action="store_true", dest="print_logs", default=False, help="Fetch logs and print them to stderr.") + optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", + default=False, help="Fetch logs and print them to stderr while command is running.") optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job") @classmethod @@ -650,6 +696,8 @@ class HadoopCommand(Command): optparser.add_option("--print-logs", action="store_true", dest="print_logs", default=False, help="Fetch logs and print them to stderr.") + optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", + default=False, help="Fetch logs and print them to stderr while command is running.") optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job") optparser.disable_interspersed_args() @@ -684,6 +732,7 @@ def parse(cls, args): parsed['tags'] = options.tags parsed["command_type"] = "HadoopCommand" parsed['print_logs'] = options.print_logs + parsed['print_logs_live'] = options.print_logs_live if len(args) < 2: raise ParseError("Need at least two arguments", cls.usage) @@ -728,6 +777,8 @@ class ShellCommand(Command): optparser.add_option("--print-logs", action="store_true", dest="print_logs", default=False, help="Fetch logs and print them to stderr.") + optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", + default=False, help="Fetch logs and print them to stderr while command is running.") @classmethod def parse(cls, args): @@ -821,6 +872,8 @@ class PigCommand(Command): optparser.add_option("--print-logs", action="store_true", dest="print_logs", default=False, help="Fetch logs and print them to stderr.") + optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", + default=False, help="Fetch logs and print them to stderr while command is running.") optparser.add_option("--retry", dest="retry", choices=[1,2,3], default=0, help="Number of retries for a job") @classmethod @@ -937,6 +990,8 @@ class DbExportCommand(Command): optparser.add_option("--print-logs", action="store_true", dest="print_logs", default=False, help="Fetch logs and print them to stderr.") + optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", + default=False, help="Fetch logs and print them to stderr while command is running.") optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job") @classmethod @@ -1038,6 +1093,8 @@ class DbImportCommand(Command): optparser.add_option("--print-logs", action="store_true", dest="print_logs", default=False, help="Fetch logs and print them to stderr.") + optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", + default=False, help="Fetch logs and print them to stderr while command is running.") optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job") @classmethod @@ -1126,6 +1183,8 @@ class DbTapQueryCommand(Command): optparser.add_option("--print-logs", action="store_true", dest="print_logs", default=False, help="Fetch logs and print them to stderr.") + optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", + default=False, help="Fetch logs and print them to stderr while command is running.") @classmethod def parse(cls, args):