Configure 14.3 support at runtime [databricks] (#11997)

Contributes to #10661, implements #11745 for DBR 14.3

- Add switch spark.rapids.shims.spark350db143.enabled

<!--

Thank you for contributing to RAPIDS Accelerator for Apache Spark!

Here are some guidelines to help the review process go smoothly.

1. Please write a description in this text box of the changes that are
being
   made.

2. Please ensure that you have written units tests for the changes
made/features
   added.

3. If you are closing an issue please use one of the automatic closing
words as
noted here:
https://help.github.com/articles/closing-issues-using-keywords/

4. If your pull request is not ready for review but you want to make use
of the
continuous integration testing facilities please label it with `[WIP]`.

5. If your pull request is ready to be reviewed without requiring
additional
   work on top of it, then remove the `[WIP]` label (if present).

6. Once all work has been done and review has taken place please do not
add
features or make changes out of the scope of those requested by the
reviewer
(doing this just add delays as already reviewed code ends up having to
be
re-reviewed/it is hard to tell what is new etc!). Further, please avoid
rebasing your branch during the review process, as this causes the
context
of any comments made by reviewers to be lost. If conflicts occur during
review then they should be resolved by merging into the branch used for
   making the pull request.

Many thanks in advance for your cooperation!

-->

---------

Signed-off-by: Gera Shegalov <[email protected]>

jenkins/databricks/test.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -78,6 +78,8 @@ IS_SPARK_321_OR_LATER=0
 export PYSP_TEST_spark_eventLog_enabled=true
 mkdir -p /tmp/spark-events
 
+export PYSP_TEST_spark_rapids_shims_spark350db143_enabled=${PYSP_TEST_spark_rapids_shims_spark350db143_enabled:-true}
+
 rapids_shuffle_smoke_test() {
     echo "Run rapids_shuffle_smoke_test..."
 

sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/DatabricksShimServiceProvider.scala

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2024, NVIDIA CORPORATION.
+ * Copyright (c) 2024-2025, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,7 +24,12 @@ package com.nvidia.spark.rapids
 
 object DatabricksShimServiceProvider {
   val log = org.slf4j.LoggerFactory.getLogger(getClass().getName().stripSuffix("$"))
-  def matchesVersion(dbrVersion: String): Boolean = {
+
+  def matchesVersion(dbrVersion: String,
+    shimMatchEnabled: Boolean = true,
+    disclaimer: String = ""
+  ): Boolean = {
+    var ignoreExceptions = true
     try {
       val sparkBuildInfo = org.apache.spark.BuildInfo
       val databricksBuildInfo = com.databricks.BuildInfo
@@ -39,12 +44,20 @@ object DatabricksShimServiceProvider {
            .stripMargin
       if (matchRes) {
         log.warn(logMessage)
+        if (shimMatchEnabled) {
+          if (disclaimer.nonEmpty) {
+            log.warn(disclaimer)
+          }
+        } else {
+          ignoreExceptions = false
+          sys.error(disclaimer)
+        }
       } else {
         log.debug(logMessage)
       }
       matchRes
     } catch {
-      case x: Throwable =>
+      case x: Throwable if ignoreExceptions =>
         log.debug("Databricks detection failed: " + x, x)
         false
     }

sql-plugin/src/main/spark350db143/scala/com/nvidia/spark/rapids/shims/spark350db143/SparkShimServiceProvider.scala

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023-2024, NVIDIA CORPORATION.
+ * Copyright (c) 2023-2025, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,6 +21,8 @@ package com.nvidia.spark.rapids.shims.spark350db143
 
 import com.nvidia.spark.rapids._
 
+import org.apache.spark.SparkEnv
+
 object SparkShimServiceProvider {
   val VERSION = DatabricksShimVersion(3, 5, 0, "14.3")
 }
@@ -30,6 +32,19 @@ class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceP
   override def getShimVersion: ShimVersion = SparkShimServiceProvider.VERSION
 
   def matchesVersion(version: String): Boolean = {
-    DatabricksShimServiceProvider.matchesVersion("14.3.x")
+    val shimEnabledProp = "spark.rapids.shims.spark350db143" + ".enabled"
+    // disabled by default
+    val shimEnabled = Option(SparkEnv.get)
+      .flatMap(_.conf.getOption(shimEnabledProp).map(_.toBoolean))
+      .getOrElse(false)
+
+    DatabricksShimServiceProvider.matchesVersion(
+      dbrVersion = "14.3.x",
+      shimMatchEnabled = shimEnabled,
+      // scalastyle:off line.size.limit
+      disclaimer = s"""|!!!! Databricks 14.3.x support is incomplete: https://github.com/NVIDIA/spark-rapids/issues/10661
+                       |!!!! It can be experimentally enabled by configuring ${shimEnabledProp}=true.""".stripMargin
+      // scalastyle:on line.size.limit
+    )
   }
 }