[RPD-278] Bug Fixes and Improvements

.pre-commit-config.yaml

@@ -7,6 +7,7 @@ repos:
       - id: check-toml
       - id: check-yaml
         args: ["--unsafe"] # only check syntax for yaml files
+        exclude: ^src/matcha_ml/infrastructure/llm/zen_server
       - id: check-json
       - id: mixed-line-ending
         files: "\\.(py|txt|yaml|json|md|toml|lock|cfg|html|sh|js|yml)$"

docs/data-version-control.md

@@ -1,10 +1,10 @@
 # Data Version Control
 
 One of the resources that matcha provisions by default is a storage container for data. The storage container is
-tool-agnostic, so whether you want to use the `dvc` package, `LakeFS` or some other data version control package is 
+tool-agnostic, so whether you want to use the `dvc` package, `LakeFS` or some other data version control package is
 up to you.
 
-You can find all the specifications for the data version control resource in the 
+You can find all the specifications for the data version control resource in the
 `infrastructure/data_version_control_storage` folder inside `matcha_ml`.
 
 ## Using the provisioned resources with the `dvc` package
@@ -29,15 +29,15 @@ This will print something like the following to your terminal:
         - account-name: ACCOUNT_NAME
         - container-name: CONTAINER_NAME
 
-Now that we have our connection string (you should keep this a secret), assuming you have followed the steps from the 
-dvc docs for initializing and adding files for `dvc` to track, we can tell the `dvc` package where to look for historic 
+Now that we have our connection string (you should keep this a secret), assuming you have followed the steps from the
+dvc docs for initializing and adding files for `dvc` to track, we can tell the `dvc` package where to look for historic
 data, and where to push new, versionable data. This is done as below:
 
     $ dvc remote modify --local my_dvc connection_string CONNECTION_STRING
     $ dvc remote modify my_dvc url azure://CONTAINER_NAME
     $ dvc remote modify my_dvc account_name STORAGE_ACCOUNT_NAME
 
-It's important to make sure your connection string is stored in `config.local`, so your connection string never appears 
+It's important to make sure your connection string is stored in `config.local`, so your connection string never appears
 in any public repository.
 
 That's it! Whenever you run `dvc push`, `dvc pull` or `dvc checkout`, you or whoever you grant access to your storage

docs/getting-started.md

@@ -124,7 +124,7 @@ Pipeline
    - server-url: <url>
    - server-username: ********
    - storage-path: az://<path>
-  
+
 Data version control
     - flavor: FLAVOR
     - connection-string: ********
@@ -241,5 +241,5 @@ matcha destroy
 ```
 
 > Note: that this command is irreversible will remove all the resources deployed by `matcha provision` including the resource group, so make sure you save any data you wish to keep before running this command.
-> 
+>
 > You may also notice that an additional resource has appeared in Azure called 'NetworkWatcherRG' (if it wasn't already there). This is a resource that is automatically provisioned by Azure in each region when there is in-coming traffic to a provisioned resource and isn't controlled by Matcha. More information can be found [here](https://learn.microsoft.com/en-us/azure/network-watcher/network-watcher-monitoring-overview) on how to manage or remove this resource.

src/matcha_ml/config/matcha_config.py

@@ -24,7 +24,9 @@ class MatchaConfigComponent:
     name: str
     properties: List[MatchaConfigComponentProperty]
 
-    def find_property(self, property_name: str) -> MatchaConfigComponentProperty:
+    def find_property(
+        self, property_name: str
+    ) -> Optional[MatchaConfigComponentProperty]:
         """Given a property name, find the property that matches it.
 
         Note: this only works under the assumption of none-duplicated properties.
@@ -43,10 +45,10 @@ def find_property(self, property_name: str) -> MatchaConfigComponentProperty:
             None,
         )
 
-        if property is None:
-            raise MatchaError(
-                f"The property with the name '{property_name}' could not be found."
-            )
+        # if property is None:
+        #     raise MatchaError(
+        #         f"The property with the name '{property_name}' could not be found."
+        #     )
 
         return property
 
@@ -57,17 +59,14 @@ class MatchaConfig:
 
     components: List[MatchaConfigComponent]
 
-    def find_component(self, component_name: str) -> MatchaConfigComponent:
+    def find_component(self, component_name: str) -> Optional[MatchaConfigComponent]:
         """Given a component name, find the component that matches it.
 
         Note: this only works under the assumption of none-duplicated properties.
 
         Args:
             component_name (str): the name of the component.
 
-        Raises:
-            MatchaError: if the component could not be found.
-
         Returns:
             MatchaConfigComponent: the component that matches the component_name parameter.
         """
@@ -76,11 +75,6 @@ def find_component(self, component_name: str) -> MatchaConfigComponent:
             None,
         )
 
-        if component is None:
-            raise MatchaError(
-                f"The component with the name '{component_name}' could not be found."
-            )
-
         return component
 
     def to_dict(self) -> Dict[str, Dict[str, str]]:
@@ -134,15 +128,19 @@ def get_stack() -> Optional[MatchaConfigComponentProperty]:
             Optional[MatchaConfigComponentProperty]: The name of the current stack being used as a config component object.
         """
         try:
-            stack = (
-                MatchaConfigService.read_matcha_config()
-                .find_component("stack")
-                .find_property("name")
-            )
+            stack = MatchaConfigService.read_matcha_config().find_component("stack")
         except MatchaError:
             stack = None
 
-        return stack
+        if stack is None:
+            return None
+
+        name = stack.find_property("name")
+
+        if name is None:
+            return None
+
+        return name
 
     @staticmethod
     def write_matcha_config(matcha_config: MatchaConfig) -> None:

src/matcha_ml/core/core.py

@@ -4,8 +4,15 @@
 from typing import Optional
 
 from matcha_ml.cli._validation import get_command_validation
-from matcha_ml.cli.ui.print_messages import print_status
-from matcha_ml.cli.ui.status_message_builders import build_warning_status
+from matcha_ml.cli.ui.print_messages import (
+    print_json,
+    print_status,
+)
+from matcha_ml.cli.ui.resource_message_builders import (
+    dict_to_json,
+    hide_sensitive_in_output,
+)
+from matcha_ml.cli.ui.status_message_builders import build_status, build_warning_status
 from matcha_ml.config import (
     MatchaConfigComponent,
     MatchaConfigComponentProperty,
@@ -69,6 +76,18 @@ def infer_zenml_version() -> str:
     return version
 
 
+def _show_terraform_outputs(matcha_state: MatchaState) -> None:
+    """Print the formatted Terraform outputs.
+
+    Args:
+        matcha_state (MatchaState): Terraform outputs in a MatchaState format.
+    """
+    print_status(build_status("Here are the endpoints for what's been provisioned"))
+    resources_dict = hide_sensitive_in_output(matcha_state.to_dict())
+    resources_json = dict_to_json(resources_dict)
+    print_json(resources_json)
+
+
 @track(event_name=AnalyticsEvent.GET)
 def get(
     resource_name: Optional[str],
@@ -307,9 +326,10 @@ def provision(
         )
         azure_template.build_template(config, template, destination, verbose)
 
-        template_runner.provision()
+        matcha_state_service = template_runner.provision()
 
-        matcha_state_service = MatchaStateService()
+        if verbose:
+            _show_terraform_outputs(matcha_state_service._state)
 
         return matcha_state_service.fetch_resources_from_state_file()
 

src/matcha_ml/infrastructure/llm/main.tf

@@ -79,6 +79,8 @@ module "zenserver" {
   # ZenServer credentials
   username = var.username
   password = var.password
+
+  zenmlserver_version = var.zenmlserver_version
 }
 
 

src/matcha_ml/infrastructure/llm/variables.tf

@@ -21,6 +21,12 @@ variable "password" {
   sensitive   = true
 }
 
+variable "zenmlserver_version" {
+  description = "The tag to use for the zenmlserver docker image."
+  default     = "latest"
+  type        = string
+}
+
 # seldon variables
 variable "seldon_name" {
   description = "Name of the Seldon deployment"

src/matcha_ml/infrastructure/llm/zen_server/main.tf

@@ -86,6 +86,10 @@ resource "helm_release" "zen_server" {
     name  = "zenml.database.sslVerifyServerCert"
     value = var.deploy_db ? false : var.database_ssl_verify_server_cert
   }
+  set {
+    name = "zenml.image.tag"
+    value = var.zenmlserver_version
+  }
   depends_on = [
     resource.kubernetes_namespace.zen_server
   ]

src/matcha_ml/infrastructure/llm/zen_server/variables.tf

@@ -161,8 +161,7 @@ variable "zenmlserver_image_repo" {
   default     = "zenmldocker/zenml-server"
   type        = string
 }
-variable "zenmlserver_image_tag" {
+variable "zenmlserver_version" {
   description = "The tag to use for the zenmlserver docker image."
-  default     = "latest"
   type        = string
 }

src/matcha_ml/infrastructure/llm/zen_server/zenml_helm/Chart.yaml

@@ -9,5 +9,5 @@ keywords:
 home: https://zenml.io
 sources:
 - https://github.com/zenml-io/zenml
-icon: https://raw.githubusercontent.com/zenml-io/zenml/main/docs/book/assets/zenml_logo.png
-appVersion: "0.36.1"
+icon: https://raw.githubusercontent.com/zenml-io/zenml/main/docs/book/.gitbook/assets/zenml_logo.png
+appVersion: "0.42.1"

src/matcha_ml/infrastructure/llm/zen_server/zenml_helm/templates/server-deployment.yaml

@@ -55,13 +55,8 @@ spec:
             - name: ZENML_LOGGING_VERBOSITY
               value: "DEBUG"
             {{- end }}
-            {{- if .Values.zenml.analyticsOptIn }}
-            - name: ZENML_ANALYTICS_OPT_IN
-              value: "True"
-            {{- else if not .Values.zenml.analyticsOptIn }}
             - name: ZENML_ANALYTICS_OPT_IN
               value: "False"
-            {{- end }}
             - name: ZENML_DEFAULT_PROJECT_NAME
               value: {{ .Values.zenml.defaultProject | quote }}
             - name: ZENML_DEFAULT_USER_NAME
@@ -153,6 +148,10 @@ spec:
               value: {{ .Values.zenml.defaultProject | quote }}
             - name: ZENML_DEFAULT_USER_NAME
               value: {{ .Values.zenml.defaultUsername | quote }}
+            {{- if .Values.zenml.enableImplicitAuthMethods }}
+            - name: ZENML_ENABLE_IMPLICIT_AUTH_METHODS
+              value: "True"
+            {{- end }}
             {{- if .Values.zenml.database.url }}
             - name: ZENML_STORE_TYPE
               value: sql

src/matcha_ml/infrastructure/llm/zen_server/zenml_helm/templates/server-ingress.yaml

@@ -6,7 +6,7 @@
   {{- $_ := set .Values.zenml.ingress.annotations "kubernetes.io/ingress.class" .Values.zenml.ingress.className}}
   {{- end }}
 {{- end }}
-{{- if $.Values.zenml.ingress.tls.enabled }}
+{{- if and $.Values.zenml.ingress.tls.enabled (eq .Values.zenml.ingress.className "nginx") }}
   {{- $_ := set .Values.zenml.ingress.annotations "nginx.ingress.kubernetes.io/ssl-redirect" "true"}}
 {{- end }}