[Kernel] Add examples for Delta Kernel API usage

kernel/examples/run-kernel-examples.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (2021) The Delta Lake Project Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import subprocess
+from os import path
+import shutil
+import argparse
+
+def run_single_threaded_examples(version, maven_repo, examples_root_dir, golden_tables_dir):
+    main_class = "io.delta.kernel.examples.SingleThreadedTableReader"
+    test_cases = [
+        f"--table={golden_tables_dir}/data-reader-primitives --columns=as_int,as_long --limit=5",
+        f"--table={golden_tables_dir}/data-reader-primitives --columns=as_int,as_long,as_double,as_string --limit=20",
+        f"--table={golden_tables_dir}/data-reader-partition-values --columns=as_string,as_byte,as_list_of_records,as_nested_struct --limit=20"
+    ]
+    project_dir = path.join(examples_root_dir, "table-reader")
+
+    run_example(version, maven_repo, project_dir, main_class, test_cases)
+
+
+def run_multi_threaded_examples(version, maven_repo, examples_root_dir, golden_tables_dir):
+    main_class = "io.delta.kernel.examples.MultiThreadedTableReader"
+    test_cases = [
+        f"--table={golden_tables_dir}/data-reader-primitives --columns=as_int,as_long --limit=5 --parallelism=5",
+        f"--table={golden_tables_dir}/data-reader-primitives --columns=as_int,as_long,as_double,as_string --limit=20 --parallelism=20",
+        f"--table={golden_tables_dir}/data-reader-partition-values --columns=as_string,as_byte,as_list_of_records,as_nested_struct --limit=20 --parallelism=2"
+    ]
+    project_dir = path.join(examples_root_dir, "table-reader")
+
+    run_example(version, maven_repo, project_dir, main_class, test_cases)
+
+
+def run_example(version, maven_repo, project_dir, main_class, test_cases):
+    with WorkingDirectory(project_dir):
+        for test in test_cases:
+            cmd = ["mvn", "package", "exec:java", f"-Dexec.mainClass={main_class}",
+                   f"-Dstaging-repo={maven_repo}",
+                   f"-Ddelta-kernel.version={version}",
+                   f"-Dexec.args={test}"]
+            run_cmd(cmd, stream_output=True)
+
+
+def clear_artifact_cache():
+    print("Clearing Delta Kernel artifacts from ivy2 and mvn cache")
+    delete_if_exists(os.path.expanduser("~/.ivy2/cache/io.delta.kernel"))
+    delete_if_exists(os.path.expanduser("~/.ivy2/local/io.delta.kernel"))
+    delete_if_exists(os.path.expanduser("~/.m2/repository/io/delta/kernel/"))
+
+
+def delete_if_exists(path):
+    # if path exists, delete it.
+    if os.path.exists(path):
+        shutil.rmtree(path)
+        print("Deleted %s " % path)
+
+
+# pylint: disable=too-few-public-methods
+class WorkingDirectory(object):
+    def __init__(self, working_directory):
+        self.working_directory = working_directory
+        self.old_workdir = os.getcwd()
+
+    def __enter__(self):
+        os.chdir(self.working_directory)
+
+    def __exit__(self, tpe, value, traceback):
+        os.chdir(self.old_workdir)
+
+
+def run_cmd(cmd, throw_on_error=True, env=None, stream_output=False, **kwargs):
+    cmd_env = os.environ.copy()
+    if env:
+        cmd_env.update(env)
+
+    if stream_output:
+        child = subprocess.Popen(cmd, env=cmd_env, **kwargs)
+        exit_code = child.wait()
+        if throw_on_error and exit_code != 0:
+            raise Exception("Non-zero exitcode: %s" % (exit_code))
+        return exit_code
+    else:
+        child = subprocess.Popen(
+            cmd,
+            env=cmd_env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            **kwargs)
+        (stdout, stderr) = child.communicate()
+        exit_code = child.wait()
+        if throw_on_error and exit_code != 0:
+            raise Exception(
+                "Non-zero exitcode: %s\n\nSTDOUT:\n%s\n\nSTDERR:%s" %
+                (exit_code, stdout, stderr))
+        return (exit_code, stdout, stderr)
+
+
+if __name__ == "__main__":
+    """
+        Script to run Delta Kernel examples which are located in the kernel/examples directory.
+        call this by running `python run-kernel-examples.py`
+        additionally the version can be provided as a command line argument.
+    """
+
+    # get the version of the package
+    examples_root_dir = path.dirname(__file__)
+    with open(path.join(examples_root_dir, "../../version.sbt")) as fd:
+        default_version = fd.readline().split('"')[1]
+
+        parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--version",
+        required=False,
+        default=default_version,
+        help="Delta Kernel version to use to run the examples")
+
+    parser.add_argument(
+        "--maven-repo",
+        required=False,
+        default=None,
+        help="Additional Maven repo to resolve staged new release artifacts")
+
+    parser.add_argument(
+        "--use-local",
+        required=False,
+        default=False,
+        action="store_true",
+        help="Generate JARs from local source code and use to run tests")
+
+    args = parser.parse_args()
+
+    if args.use_local and (args.version != default_version):
+        raise Exception("Cannot specify --use-local with a --version different than in version.sbt")
+
+    clear_artifact_cache()
+
+    if args.use_local:
+        project_root = path.join(examples_root_dir, "../../")
+        with WorkingDirectory(project_root):
+            run_cmd([f"{project_root}/build/sbt", "kernelGroup/publishM2"], stream_output=True)
+
+    golden_file_dir = path.join(
+        examples_root_dir,
+        "../../connectors//golden-tables/src/main/resources/golden/")
+
+    run_single_threaded_examples(args.version, args.maven_repo, examples_root_dir, golden_file_dir)
+    run_multi_threaded_examples(args.version, args.maven_repo, examples_root_dir, golden_file_dir)

kernel/examples/table-reader/pom.xml

@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--Copyright (2021) The Delta Lake Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.example</groupId>
+    <artifactId>table-reader</artifactId>
+    <version>0.1-SNAPSHOT</version>
+
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+        <staging.repo.url>""</staging.repo.url>
+        <delta-kernel.version>3.0.0-SNAPSHOT</delta-kernel.version>
+        <hadoop.version>3.3.1</hadoop.version>
+    </properties>
+
+    <repositories>
+        <repository>
+            <id>staging-repo</id>
+            <url>${staging.repo.url}</url>
+        </repository>
+    </repositories>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.delta</groupId>
+            <artifactId>delta-kernel-api</artifactId>
+            <version>${delta-kernel.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>io.delta</groupId>
+            <artifactId>delta-kernel-default</artifactId>
+            <version>${delta-kernel.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client-runtime</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client-api</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+            <version>1.5.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.13.5</version>
+        </dependency>
+
+    </dependencies>
+</project>