V1.9.3

bin/qds.py

@@ -107,6 +107,7 @@ def submitaction(cmdclass, args):
     args = cmdclass.parse(args)
     if args is not None:
         args.pop("print_logs") # This is only useful while using the 'run' action.
+        args.pop("print_logs_live") # This is only useful while using the 'run' action.
         cmd = cmdclass.create(**args)
         print("Submitted %s, Id: %s" % (cmdclass.__name__, cmd.id))
         return 0

qds_sdk/account.py

@@ -43,6 +43,15 @@ def parsers():
             choices=["true", "false"], default="false",
             help="Use previous account plan, default: false")
         create.set_defaults(func=AccountCmdLine.create)
+
+        #branding
+        branding = subparsers.add_parser(
+            "branding", help="Branding logo and link")
+        branding.add_argument("--account-id", dest = "account_id", required=True, help = "Account ID of the Qubole account for which branding has to be done")
+        branding.add_argument("--logo-uri", dest = "logo_uri", help = "Publicly accessible logo URI image in jpg/gif/svg/jpeg format.Image size must be less than 100 KB.")
+        branding.add_argument("--link-url", dest = "link_url", help = "Specify the documentation URL.")
+        branding.add_argument("--link-label", dest = "link_label", help = "Add a label to describe the documentation URL.")
+        branding.set_defaults(func=AccountCmdLine.branding)
         return argparser
 
     @staticmethod
@@ -64,6 +73,25 @@ def create(args):
         result = Account.create(**v)
         return result
 
+    @staticmethod
+    def branding(args):
+        v= {}
+        print(args)
+        v['account_id'] = args.account_id
+        if args.logo_uri is not None:
+            v['logo'] = {'logo_uri' : args.logo_uri }
+
+        link = {}
+        if args.link_url is not None:
+            link['link_url'] = args.link_url
+        if args.link_label is not None:
+            link['link_label'] = args.link_label
+
+        if bool(link):
+            v['link'] = link
+
+        result = Account.branding(**v)
+        return result
 
 class Account(SingletonResource):
     credentials_rest_entity_path = "accounts/get_creds"
@@ -73,3 +101,9 @@ class Account(SingletonResource):
     def create(cls, **kwargs):
         conn = Qubole.agent()
         return cls(conn.post(cls.rest_entity_path, data=kwargs))
+
+    @classmethod
+    def branding(cls, **kwargs):
+        conn = Qubole.agent()
+        url_path = "accounts/branding"
+        return  cls(conn.put(url_path, data=kwargs))

qds_sdk/commands.py

@@ -4,6 +4,7 @@
 the specific commands
 """
 
+from __future__ import print_function
 from qds_sdk.qubole import Qubole
 from qds_sdk.resource import Resource
 from qds_sdk.exception import ParseError
@@ -12,8 +13,8 @@
 from qds_sdk.util import OptionParsingError
 from qds_sdk.util import OptionParsingExit
 from optparse import SUPPRESS_HELP
-import boto
 
+import boto
 import time
 import logging
 import sys
@@ -87,10 +88,30 @@ def run(cls, **kwargs):
         Returns:
             Command object
         """
+
+        # vars to keep track of actual logs bytes (err, tmp) and new bytes seen in each iteration
+        err_pointer, tmp_pointer, new_bytes = 0, 0, 0
+        print_logs_live = kwargs.pop("print_logs_live") # We don't want to send this to the API.
+
         cmd = cls.create(**kwargs)
         while not Command.is_done(cmd.status):
             time.sleep(Qubole.poll_interval)
             cmd = cls.find(cmd.id)
+            if print_logs_live is True:
+                log, err_length, tmp_length = cmd.get_log_partial(err_pointer, tmp_pointer)
+
+                # if err length is non zero, then tmp_pointer needs to be reset to the current tmp_length as the
+                # err_length will contain the full set of logs from last seen non-zero err_length.
+                if err_length != "0":
+                    err_pointer += int(err_length)
+                    new_bytes = int(err_length) + int(tmp_length) - tmp_pointer
+                    tmp_pointer = int(tmp_length)
+                else:
+                    tmp_pointer += int(tmp_length)
+                    new_bytes = int(tmp_length)
+
+                if len(log) > 0 and new_bytes > 0:
+                    print(log[-new_bytes:], file=sys.stderr)
 
         return cmd
 
@@ -136,6 +157,23 @@ def get_log(self):
         r = conn.get_raw(log_path)
         return r.text
 
+    def get_log_partial(self, err_pointer=0, tmp_pointer=0):
+        """
+        Fetches log (full or partial) for the command represented by this object
+        Accepts:
+            err_pointer(int): Pointer to err text bytes we've received so far, which will be passed to next api call
+                to indicate pointer to fetch logs.
+            tmp_pointer(int): Same as err_pointer except it indicates the bytes of tmp file processed.
+        Returns:
+            An array where the first field is actual log (string), while 2nd & 3rd are counts of err and tmp bytes
+                which have been returned by api in addition to the given pointers.
+        """
+        log_path = self.meta_data['logs_resource']
+        conn = Qubole.agent()
+        r = conn.get_raw(log_path, params={'err_file_processed':err_pointer, 'tmp_file_processed':tmp_pointer})
+        if 'err_length' in r.headers.keys() and 'tmp_length' in r.headers.keys():
+            return [r.text, r.headers['err_length'], r.headers['tmp_length']]
+        return [r.text, 0, 0]
 
     @classmethod
     def get_jobs_id(cls, id):
@@ -243,6 +281,8 @@ class HiveCommand(Command):
 
     optparser.add_option("--print-logs", action="store_true", dest="print_logs",
                          default=False, help="Fetch logs and print them to stderr.")
+    optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live",
+                         default=False, help="Fetch logs and print them to stderr while command is running.")
     optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job")
 
     @classmethod
@@ -328,6 +368,8 @@ class SqlCommand(Command):
 
     optparser.add_option("--print-logs", action="store_true", dest="print_logs",
                          default=False, help="Fetch logs and print them to stderr.")
+    optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live",
+                         default=False, help="Fetch logs and print them to stderr while command is running.")
 
     @classmethod
     def parse(cls, args):
@@ -419,6 +461,8 @@ class SparkCommand(Command):
 
     optparser.add_option("--print-logs", action="store_true", dest="print_logs",
                          default=False, help="Fetch logs and print them to stderr.")
+    optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live",
+                         default=False, help="Fetch logs and print them to stderr while command is running.")
     optparser.add_option("--retry", dest="retry", default=0, help="Number of retries")
 
     @classmethod
@@ -576,6 +620,8 @@ class PrestoCommand(Command):
 
     optparser.add_option("--print-logs", action="store_true", dest="print_logs",
                          default=False, help="Fetch logs and print them to stderr.")
+    optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live",
+                         default=False, help="Fetch logs and print them to stderr while command is running.")
     optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job")
 
     @classmethod
@@ -650,6 +696,8 @@ class HadoopCommand(Command):
 
     optparser.add_option("--print-logs", action="store_true", dest="print_logs",
                          default=False, help="Fetch logs and print them to stderr.")
+    optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live",
+                         default=False, help="Fetch logs and print them to stderr while command is running.")
     optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job")
 
     optparser.disable_interspersed_args()
@@ -684,6 +732,7 @@ def parse(cls, args):
         parsed['tags'] = options.tags
         parsed["command_type"] = "HadoopCommand"
         parsed['print_logs'] = options.print_logs
+        parsed['print_logs_live'] = options.print_logs_live
 
         if len(args) < 2:
             raise ParseError("Need at least two arguments", cls.usage)
@@ -728,6 +777,8 @@ class ShellCommand(Command):
 
     optparser.add_option("--print-logs", action="store_true", dest="print_logs",
                          default=False, help="Fetch logs and print them to stderr.")
+    optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live",
+                         default=False, help="Fetch logs and print them to stderr while command is running.")
 
     @classmethod
     def parse(cls, args):
@@ -821,6 +872,8 @@ class PigCommand(Command):
 
     optparser.add_option("--print-logs", action="store_true", dest="print_logs",
                          default=False, help="Fetch logs and print them to stderr.")
+    optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live",
+                         default=False, help="Fetch logs and print them to stderr while command is running.")
     optparser.add_option("--retry", dest="retry", choices=[1,2,3], default=0, help="Number of retries for a job")
 
     @classmethod
@@ -937,6 +990,8 @@ class DbExportCommand(Command):
 
     optparser.add_option("--print-logs", action="store_true", dest="print_logs",
                          default=False, help="Fetch logs and print them to stderr.")
+    optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live",
+                         default=False, help="Fetch logs and print them to stderr while command is running.")
     optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job")
 
     @classmethod
@@ -1038,6 +1093,8 @@ class DbImportCommand(Command):
 
     optparser.add_option("--print-logs", action="store_true", dest="print_logs",
                          default=False, help="Fetch logs and print them to stderr.")
+    optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live",
+                         default=False, help="Fetch logs and print them to stderr while command is running.")
     optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job")
 
     @classmethod
@@ -1126,6 +1183,8 @@ class DbTapQueryCommand(Command):
 
     optparser.add_option("--print-logs", action="store_true", dest="print_logs",
                          default=False, help="Fetch logs and print them to stderr.")
+    optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live",
+                         default=False, help="Fetch logs and print them to stderr while command is running.")
 
     @classmethod
     def parse(cls, args):

setup.py

@@ -12,7 +12,7 @@ def read(fname):
 
 setup(
     name="qds_sdk",
-    version="1.9.2",
+    version="1.9.3",
     author="Qubole",
     author_email="[email protected]",
     description=("Python SDK for coding to the Qubole Data Service API"),

tests/test_account.py

@@ -183,5 +183,56 @@ def test_default_previous_account_plan(self):
             'defloc': 's3://bucket/path'}})
 
 
+class TestAccountBranding(QdsCliTestCase):
+    def test_logo(self):
+        sys.argv = ['qds.py', 'account', 'branding',
+                    '--account-id', '4',
+                    '--logo-uri', 'https://www.xyz.com/image.jpg']
+        print_command()
+        Connection._api_call = Mock(return_value={})
+        qds.main()
+        Connection._api_call.assert_called_with("PUT", "accounts/branding", {'logo': {
+            'logo_uri' : 'https://www.xyz.com/image.jpg'},
+            'account_id' : '4'})
+
+    def test_link(self):
+        sys.argv = ['qds.py', 'account', 'branding',
+                    '--account-id', '4',
+                    '--link-url', 'https://www.xyz.com',
+                    '--link-label', 'Documentation']
+        print_command()
+        Connection._api_call = Mock(return_value={})
+        qds.main()
+        Connection._api_call.assert_called_with("PUT", "accounts/branding", {'link': {
+            'link_url' : 'https://www.xyz.com',
+            'link_label' : 'Documentation'},
+            'account_id' : '4'})
+
+    def test_logo_link(self):
+        sys.argv = ['qds.py', 'account', 'branding',
+                    '--account-id', '4',
+                    '--logo-uri', 'https://www.xyz.com/image.jpg',
+                    '--link-url', 'https://www.xyz.com',
+                    '--link-label', 'Documentation']
+        print_command()
+        Connection._api_call = Mock(return_value={})
+        qds.main()
+        Connection._api_call.assert_called_with("PUT", "accounts/branding", {'logo': {
+            'logo_uri' : 'https://www.xyz.com/image.jpg'},
+            'link': {'link_url' : 'https://www.xyz.com',
+            'link_label' : 'Documentation'},
+            'account_id' : '4'})
+
+    def test_without_account_id(self):
+        sys.argv = ['qds.py', 'account', 'branding',
+                    '--logo-uri', 'https://www.xyz.com/image.jpg',
+                    '--link-url', 'https://www.xyz.com',
+                    '--link-label', 'Documentation']
+        print_command()
+        Connection._api_call = Mock(return_value={})
+        with self.assertRaises(SystemExit):
+            qds.main()
+
+
 if __name__ == '__main__':
     unittest.main()