feat(sdk): Support to print the cli command output in json format (#4566

)

* Add support for json print format on command output

* Remove the additional log messages from cli command output

sdk/python/kfp/cli/cli.py

@@ -20,21 +20,28 @@
 from .pipeline import pipeline
 from .diagnose_me_cli import diagnose_me
 from .experiment import experiment
+from .output import OutputFormat
 
 @click.group()
 @click.option('--endpoint', help='Endpoint of the KFP API service to connect.')
 @click.option('--iap-client-id', help='Client ID for IAP protected endpoint.')
-@click.option('-n', '--namespace', default='kubeflow', help='Kubernetes namespace to connect to the KFP API.')
+@click.option('-n', '--namespace', default='kubeflow', show_default=True,
+              help='Kubernetes namespace to connect to the KFP API.')
 @click.option('--other-client-id', help='Client ID for IAP protected endpoint to obtain the refresh token.')
 @click.option('--other-client-secret', help='Client ID for IAP protected endpoint to obtain the refresh token.')
+@click.option('--output', type=click.Choice(list(map(lambda x: x.name, OutputFormat))),
+              default=OutputFormat.table.name, show_default=True,
+              help='The formatting style for command output.')
 @click.pass_context
-def cli(ctx, endpoint, iap_client_id, namespace, other_client_id, other_client_secret):
+def cli(ctx, endpoint, iap_client_id, namespace, other_client_id, other_client_secret, output):
     """kfp is the command line interface to KFP service."""
     if ctx.invoked_subcommand == 'diagnose_me':
           # Do not create a client for diagnose_me
           return
     ctx.obj['client'] = Client(endpoint, iap_client_id, namespace, other_client_id, other_client_secret)
-    ctx.obj['namespace']= namespace
+    ctx.obj['namespace'] = namespace
+    ctx.obj['output'] = output
+
 
 def main():
     logging.basicConfig(format='%(message)s', level=logging.INFO)

sdk/python/kfp/cli/experiment.py

@@ -1,6 +1,7 @@
 import click
 import logging
-from tabulate import tabulate
+
+from .output import print_output, OutputFormat
 
 
 @click.group()
@@ -19,10 +20,10 @@ def experiment():
 def create(ctx, description, name):
     """Create an experiment"""
     client = ctx.obj["client"]
+    output_format = ctx.obj["output"]
 
     response = client.create_experiment(name, description=description)
-    logging.info("Experiment {} has been submitted\n".format(name))
-    _display_experiment(response)
+    _display_experiment(response, output_format)
 
 
 @experiment.command()
@@ -35,13 +36,14 @@ def create(ctx, description, name):
 def list(ctx, max_size):
     """List experiments"""
     client = ctx.obj['client']
+    output_format = ctx.obj['output']
 
     response = client.experiments.list_experiment(
         page_size=max_size,
         sort_by="created_at desc"
     )
     if response.experiments:
-        _display_experiments(response.experiments)
+        _display_experiments(response.experiments, output_format)
     else:
         logging.info("No experiments found")
 
@@ -52,9 +54,10 @@ def list(ctx, max_size):
 def get(ctx, experiment_id):
     """Get detailed information about an experiment"""
     client = ctx.obj["client"]
+    output_format = ctx.obj["output"]
 
     response = client.get_experiment(experiment_id)
-    _display_experiment(response)
+    _display_experiment(response, output_format)
 
 
 @experiment.command()
@@ -75,22 +78,25 @@ def delete(ctx, experiment_id):
     print("{} is deleted.".format(experiment_id))
 
 
-def _display_experiments(experiments):
+def _display_experiments(experiments, output_format):
     headers = ["Experiment ID", "Name", "Created at"]
     data = [[
         exp.id,
         exp.name,
         exp.created_at.isoformat()
     ] for exp in experiments]
-    print(tabulate(data, headers=headers, tablefmt="grid"))
+    print_output(data, headers, output_format, table_format="grid")
 
 
-def _display_experiment(exp):
-    print(tabulate([], headers=["Experiment Details"]))
+def _display_experiment(exp, output_format):
     table = [
         ["ID", exp.id],
         ["Name", exp.name],
         ["Description", exp.description],
         ["Created at", exp.created_at.isoformat()],
     ]
-    print(tabulate(table, tablefmt="plain"))
+    if output_format == OutputFormat.table.name:
+        print_output([], ["Experiment Details"], output_format)
+        print_output(table, [], output_format, table_format="plain")
+    elif output_format == OutputFormat.json.name:
+        print_output(dict(table), [], output_format)

sdk/python/kfp/cli/output.py

@@ -0,0 +1,57 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from enum import Enum, unique
+from typing import Union
+
+from tabulate import tabulate
+
+
+@unique
+class OutputFormat(Enum):
+    """Enumerated class with the allowed output format constants."""
+    table = "table"
+    json = "json"
+
+
+def print_output(data: Union[list, dict], headers: list, output_format: str, table_format: str = "simple"):
+    """Prints the output from the cli command execution based on the specified format.
+
+    Args:
+        data (Union[list, dict]): Nested list of values representing the rows to be printed.
+        headers (list): List of values representing the column names to be printed
+            for the ``data``.
+        output_format (str): The desired formatting of the text from the command output.
+        table_format (str): The formatting for the table ``output_format``.
+            Default value set to ``simple``.
+
+    Returns:
+        None: Prints the output.
+
+    Raises:
+        NotImplementedError: If the ``output_format`` is unknown.
+    """
+    if output_format == OutputFormat.table.name:
+        print(tabulate(data, headers=headers, tablefmt=table_format))
+    elif output_format == OutputFormat.json.name:
+        if not headers:
+            output = data
+        else:
+            output = []
+            for row in data:
+                output.append(dict(zip(headers, row)))
+        print(json.dumps(output, indent=4))
+    else:
+        raise NotImplementedError("Unknown Output Format: {}".format(output_format))

sdk/python/kfp/cli/pipeline.py

@@ -14,7 +14,8 @@
 
 import click
 import logging
-from tabulate import tabulate
+
+from .output import print_output, OutputFormat
 
 
 @click.group()
@@ -34,12 +35,12 @@ def pipeline():
 def upload(ctx, pipeline_name, package_file):
     """Upload a KFP pipeline"""
     client = ctx.obj["client"]
+    output_format = ctx.obj["output"]
     if not pipeline_name:
         pipeline_name = package_file.split(".")[0]
 
     pipeline = client.upload_pipeline(package_file, pipeline_name)
-    logging.info("Pipeline {} has been submitted\n".format(pipeline.id))
-    _display_pipeline(pipeline)
+    _display_pipeline(pipeline, output_format)
 
 
 @pipeline.command()
@@ -66,19 +67,16 @@ def upload(ctx, pipeline_name, package_file):
 def upload_version(ctx, package_file, pipeline_version, pipeline_id=None, pipeline_name=None):
     """Upload a version of the KFP pipeline"""
     client = ctx.obj["client"]
+    output_format = ctx.obj["output"]
     if bool(pipeline_id) == bool(pipeline_name):
         raise ValueError("Need to suppy 'pipeline-name' or 'pipeline-id'")
     if pipeline_name!=None: 
         pipeline_id = client.get_pipeline_id(name=pipeline_name)
         if pipeline_id==None: 
             raise ValueError("Can't find a pipeline with name: %s" % pipeline_name)
-    logging.info("The pipeline id is: %s" % pipeline_id)
     version = client.pipeline_uploads.upload_pipeline_version(
         package_file, name=pipeline_version, pipelineid=pipeline_id)
-    logging.info(
-        "The %s version of the pipeline %s has been submitted\n" %
-            (pipeline_version, pipeline_id))
-    _display_pipeline_version(version)
+    _display_pipeline_version(version, output_format)
 
 
 @pipeline.command()
@@ -91,13 +89,14 @@ def upload_version(ctx, package_file, pipeline_version, pipeline_id=None, pipeli
 def list(ctx, max_size):
     """List uploaded KFP pipelines"""
     client = ctx.obj["client"]
+    output_format = ctx.obj["output"]
 
     response = client.list_pipelines(
         page_size=max_size,
         sort_by="created_at desc"
     )
     if response.pipelines:
-        _print_pipelines(response.pipelines)
+        _print_pipelines(response.pipelines, output_format)
     else:
         logging.info("No pipelines found")
 
@@ -113,14 +112,15 @@ def list(ctx, max_size):
 def list_versions(ctx, pipeline_id, max_size):
     """List versions of an uploaded KFP pipeline"""
     client = ctx.obj["client"]
+    output_format = ctx.obj["output"]
 
     response = client.list_pipeline_versions(
         pipeline_id=pipeline_id,
         page_size=max_size,
         sort_by="created_at desc"
     )
     if response.versions:
-        _print_pipeline_versions(response.versions)
+        _print_pipeline_versions(response.versions, output_format)
     else:
         logging.info("No pipeline or version found")
 
@@ -130,9 +130,10 @@ def list_versions(ctx, pipeline_id, max_size):
 def get(ctx, pipeline_id):
     """Get detailed information about an uploaded KFP pipeline"""
     client = ctx.obj["client"]
+    output_format = ctx.obj["output"]
 
     pipeline = client.get_pipeline(pipeline_id)
-    _display_pipeline(pipeline)
+    _display_pipeline(pipeline, output_format)
 
 
 @pipeline.command()
@@ -146,49 +147,63 @@ def delete(ctx, pipeline_id):
     print("{} is deleted".format(pipeline_id))
 
 
-def _print_pipelines(pipelines):
+def _print_pipelines(pipelines, output_format):
     headers = ["Pipeline ID", "Name", "Uploaded at"]
     data = [[
         pipeline.id,
         pipeline.name,
         pipeline.created_at.isoformat()
     ] for pipeline in pipelines]
-    print(tabulate(data, headers=headers, tablefmt="grid"))
+    print_output(data, headers, output_format, table_format="grid")
 
 
-def _print_pipeline_versions(versions):
+def _print_pipeline_versions(versions, output_format):
     headers = ["Version ID", "Version name", "Uploaded at"]
     data = [[
         version.id,
         version.name,
         version.created_at.isoformat()
     ] for version in versions]
-    print(tabulate(data, headers=headers, tablefmt="grid"))
+    print_output(data, headers, output_format, table_format="grid")
 
 
-def _display_pipeline(pipeline):
-    # Print pipeline information
-    print(tabulate([], headers=["Pipeline Details"]))
+def _display_pipeline(pipeline, output_format):
+    # Pipeline information
     table = [
         ["ID", pipeline.id],
         ["Name", pipeline.name],
         ["Description", pipeline.description],
         ["Uploaded at", pipeline.created_at.isoformat()],
     ]
-    print(tabulate(table, tablefmt="plain"))
 
-    # Print pipeline parameter details
+    # Pipeline parameter details
     headers = ["Parameter Name", "Default Value"]
     data = [[param.name, param.value] for param in pipeline.parameters]
-    print(tabulate(data, headers=headers, tablefmt="grid"))
-
 
-def _display_pipeline_version(version):
-    print(tabulate([], headers=["Pipeline Version Details"]))
+    if output_format == OutputFormat.table.name:
+        print_output([], ["Pipeline Details"], output_format)
+        print_output(table, [], output_format, table_format="plain")
+        print_output(data, headers, output_format, table_format="grid")
+    elif output_format == OutputFormat.json.name:
+        output = dict()
+        output["Pipeline Details"] = dict(table)
+        params = []
+        for item in data:
+            params.append(dict(zip(headers, item)))
+        output["Pipeline Parameters"] = params
+        print_output(output, [], output_format)
+
+
+def _display_pipeline_version(version, output_format):
     pipeline_id = version.resource_references[0].key.id
     table = [
         ["Pipeline ID", pipeline_id],
         ["Version Name", version.name],
         ["Uploaded at", version.created_at.isoformat()],
     ]
-    print(tabulate(table, tablefmt="plain"))
+
+    if output_format == OutputFormat.table.name:
+        print_output([], ["Pipeline Version Details"], output_format)
+        print_output(table, [], output_format, table_format="plain")
+    elif output_format == OutputFormat.json.name:
+        print_output(dict(table), [], output_format)