Merge pull request #450 from harsh97/main

Updating AQUA stack

.github/workflows/stack.yml

@@ -0,0 +1,50 @@
+# Copyright (c) 2022 Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
+# 
+
+# Creates and Publishes the Oracle Resource Manager stack - v0.0.5
+
+name: Generate stacks and publish release
+
+on:
+  push:
+    branches: [ main ]
+    paths: ['VERSION']
+
+jobs:
+
+  publish_stack:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Create stacks
+      id: create_stacks
+      run: |
+        
+        STACKNAME=oci-ods-aqua
+        STACK_FILES="ai-quick-actions/policies/terraform/*"
+        RELEASE=$(cat VERSION)
+        ASSETS+="${STACKNAME}.zip"
+        echo "::group::Processing $STACKNAME"
+        zip -r ${STACKNAME}-stack.zip $STACK_FILES || { printf '\n⛔ Unable to create %s stack.\n'; exit 1;  }
+        cp ${STACKNAME}-stack.zip ${STACKNAME}.zip || { printf '\n⛔ Unable to create %s stack.\n'; exit 1;  }
+        echo "::endgroup::"
+        echo "::set-output name=assets::$ASSETS"
+        echo "::set-output name=release::$RELEASE"
+        echo "::set-output name=prefix::$STACKNAME"
+
+    - name: Prepare Release Notes
+      run: |
+        #
+        printf '%s\n' '${{ steps.create_stacks.outputs.prefix }} Stack - v${{ steps.create_stacks.outputs.release }}' >release.md
+        printf '%s\n' '' '## [![Deploy to Oracle Cloud][magic_button]][magic_stack]' >>release.md
+        printf '%s\n' '' '' >>release.md
+        printf '%s\n' '' '[magic_button]: https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg' >>release.md
+        printf '%s\n' '' '[magic_stack]: https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/${{ github.repository }}/releases/download/${{ steps.create_stacks.outputs.release }}/${{ steps.create_stacks.outputs.prefix }}.zip' >>release.md
+
+    - name: Create Release
+      run: gh release create ${{ steps.create_stacks.outputs.release }} --generate-notes -F release.md ${{ steps.create_stacks.outputs.assets }}
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

VERSION

@@ -0,0 +1 @@
+1.0

ai-quick-actions/policies/README.md

@@ -28,11 +28,15 @@ allow group <your_admin_group> to manage policies in TENANCY
 allow group <your_admin_group> to read compartments in TENANCY
 ```
 
- Download terraform configuration file [oci-ods-aqua-orm.zip](./oci-ods-aqua-orm.zip) with the infrastructure instructions for the dynamic groups and polices. For steps on creating stacks, see [Creating a Stack from a Zip File](https://docs.oracle.com/en-us/iaas/Content/ResourceManager/Tasks/create-stack-local.htm#top).
+Click to deploy the stack  [![Deploy to Oracle Cloud][magic_button]][magic_stack]
 
-![Setup 1](../web_assets/policies1.png)
 
-![Setup 2](../web_assets/policies2.png)
+After clicking the button, you will be redirected to the Oracle Cloud Infrastructure Console.  You will need to sign in if you are not already signed in.  You can select the kind of policies that need to be deployed for AQUA:
+1. All policies - This will deploy all the policies needed for AQUA in one go.
+2. Only admin policies - This will deploy only the minimal set of policies that are required to be defined at the root compartment by the tenancy administrator for AQUA.
+3. Only resource policies - This will deploy the required policies that are required to be defined at the compartment level provided that the tenancy administrator has already defined the admin policies for AQUA.
+
+![Setup 1](../web_assets/policies1.png)
 
 > **Note:** To save fine-tuned models, versioning has to be enabled in the selected Object Storage bucket. See [here](https://docs.oracle.com/iaas/data-science/using/ai-quick-actions-fine-tuning.htm) for more information.
 
@@ -115,7 +119,8 @@ These policies and dynamic groups set up the necessary permissions to enable AI
 > **Note:** To save fine-tuned models, versioning has to be enabled in the selected Object Storage bucket. See [here](https://docs.oracle.com/iaas/data-science/using/ai-quick-actions-fine-tuning.htm) for more information.
 
 ![Setup 3](../web_assets/policies3.png)
-
+- [magic_button]: https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg
+- [magic_stack]: https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-samples/oci-data-science-ai-samples/releases/latest/download/oci-ods-aqua.zip
 - [Home](../README.md)
 - [CLI](../cli-tips.md)
 - [Model Deployment](../model-deployment-tips.md)

ai-quick-actions/policies/terraform/iam.tf

@@ -0,0 +1,117 @@
+resource "oci_identity_dynamic_group" "aqua-dynamic-group" {
+  count = local.is_admin_policies_only ? 0 : 1
+  compartment_id = var.tenancy_ocid
+  description    = "Data Science Aqua Dynamic Group"
+  name           = var.aqua_dg_name
+  matching_rule =  local.is_resource_policy_required? local.aqua_dg_match: local.aqua_admin_only_dg_match
+}
+
+resource "oci_identity_dynamic_group" "distributed_training_job_runs" {
+  count          = local.is_resource_policy_required ? 1 : 0
+  compartment_id = var.tenancy_ocid
+  description    = "Data Science Distributed Training Job Runs Group"
+  name           = var.distributed_training_dg_name
+  matching_rule = "any {all {resource.type='datasciencejobrun',resource.compartment.id='${var.compartment_ocid}'}}"
+}
+
+
+locals {
+  is_admin_policies_only = var.deployment_type == "Only admin policies"
+  is_resource_policy_only = var.deployment_type == "Only resource policies"
+  is_all_policies = var.deployment_type == "All policies"
+  is_resource_policy_required = var.deployment_type != "Only admin policies"
+  // Aqua dg matching rules
+  aqua_admin_only_dg_match = "all {resource.type='datasciencenotebooksession'}"
+  aqua_dg_match = "any {all {resource.type='datasciencenotebooksession',resource.compartment.id='${var.compartment_ocid}'}, all {resource.type='datasciencemodeldeployment',resource.compartment.id='${var.compartment_ocid}'}, all {resource.type='datasciencejobrun',resource.compartment.id='${var.compartment_ocid}'}}"
+  is_compartment_tenancy = length(regexall(".*tenancy.*", var.compartment_ocid)) > 0
+  compartment_policy_string = local.is_compartment_tenancy ? "tenancy" :  "compartment id ${var.compartment_ocid}"
+  policy_tenancy = local.is_resource_policy_only? var.compartment_ocid : var.tenancy_ocid
+  // Contains only necessary admin policies. These policies will be created in the tenancy. When the user selects "Only admin policies" these policies will be created.
+  aqua_admin_only_policies = local.is_admin_policies_only?[
+    "Define tenancy datascience as ocid1.tenancy.oc1..aaaaaaaax5hdic7ya6r5rxsgpifff4l6xdxzltnrncdzp3m75ubbvzqqzn3q",
+    "Endorse any-user to read data-science-models in tenancy datascience where ALL {target.compartment.name='service-managed-models'}",
+    "Endorse any-user to inspect data-science-models in tenancy datascience where ALL {target.compartment.name='service-managed-models'}",
+    "Endorse any-user to read object in tenancy datascience where ALL {target.compartment.name='service-managed-models', target.bucket.name='service-managed-models'}",
+  ]:[]
+
+  tenancy_map = ({
+    oc1: "ocid1.tenancy.oc1..aaaaaaaax5hdic7ya6r5rxsgpifff4l6xdxzltnrncdzp3m75ubbvzqqzn3q"
+    oc8: "ocid1.tenancy.oc8..aaaaaaaa2nxkxxix6ngdcifswvrezlmuylejvse4x6oa2ub4wfaduyz547wa"
+  })
+  user_realm = element(split(".", var.tenancy_ocid), 2)
+  service_tenancy_ocid = lookup(local.tenancy_map, local.user_realm, "UNKNOWN")
+
+  // These are encompassing policies that will be created in the tenancy. When the user selects "All policies" these policies will be created.
+  aqua_all_policies = local.is_all_policies? [
+    "Define tenancy datascience as ${local.service_tenancy_ocid}",
+    "Endorse any-user to read data-science-models in tenancy datascience where ALL {target.compartment.name='service-managed-models'}",
+    "Endorse any-user to inspect data-science-models in tenancy datascience where ALL {target.compartment.name='service-managed-models'}",
+    "Endorse any-user to read object in tenancy datascience where ALL {target.compartment.name='service-managed-models', target.bucket.name='service-managed-models'}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage data-science-model-deployments in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage data-science-models in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to use logging-family in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage data-science-jobs in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage data-science-job-runs in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to use virtual-network-family in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to read resource-availability in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage data-science-projects in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage data-science-notebook-sessions in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage data-science-modelversionsets in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to read buckets in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to read objectstorage-namespaces in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to inspect compartments in tenancy"
+    ]:[]
+
+  // Aqua resource only policies. These policies will be created in a specific compartment. When the user selects "Only resource policies" these policies will be created.
+  aqua_resource_only_policies = local.is_resource_policy_only? [
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage data-science-model-deployments in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage data-science-models in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to use logging-family in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage data-science-jobs in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage data-science-job-runs in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to use virtual-network-family in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to read resource-availability in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage data-science-projects in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage data-science-notebook-sessions in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage data-science-modelversionsets in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to read buckets in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to read objectstorage-namespaces in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to inspect compartments in ${local.compartment_policy_string}"
+  ]:[]
+
+  policies_to_use = local.is_admin_policies_only ? local.aqua_admin_only_policies : local.is_resource_policy_only ? local.aqua_resource_only_policies : local.aqua_all_policies
+
+  all_buckets = concat(var.user_model_buckets, var.user_data_buckets)
+  bucket_names = join(", ", formatlist("target.bucket.name='%s'", local.all_buckets))
+  bucket_names_oss = join(", ", formatlist("all{target.bucket.name='%s', any {request.permission='OBJECT_CREATE', request.permission='OBJECT_INSPECT'}}", local.all_buckets))
+  dt_jr_policies = local.is_resource_policy_required?[
+    "Allow dynamic-group id ${oci_identity_dynamic_group.distributed_training_job_runs[0].id} to use logging-family in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.distributed_training_job_runs[0].id} to manage data-science-models in ${local.compartment_policy_string}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.distributed_training_job_runs[0].id} to read data-science-jobs in ${local.compartment_policy_string}"
+  ]: []
+  dt_jr_policies_target_buckets = local.is_resource_policy_required? [
+    "Allow dynamic-group id ${oci_identity_dynamic_group.distributed_training_job_runs[0].id} to manage objects in ${local.compartment_policy_string} where any {${local.bucket_names}}",
+    "Allow dynamic-group id ${oci_identity_dynamic_group.distributed_training_job_runs[0].id} to read buckets in ${local.compartment_policy_string} where any {${local.bucket_names}}"
+  ]: []
+  aqua_policies_target_buckets = local.is_resource_policy_required?[
+    "Allow dynamic-group id ${oci_identity_dynamic_group.aqua-dynamic-group[0].id} to manage object-family in ${local.compartment_policy_string} where any {${local.bucket_names_oss}}"
+  ]:[]
+
+}
+
+resource "oci_identity_policy" "aqua-policy" {
+  compartment_id = local.policy_tenancy
+  description    = "Data Science Aqua Policies"
+  name           = var.aqua_policy_name
+  statements     = length(local.bucket_names) > 0 ? concat(local.policies_to_use, local.aqua_policies_target_buckets): local.policies_to_use
+}
+
+resource "oci_identity_policy" "distributed_training_job_runs_policy" {
+  count          = local.is_resource_policy_required ? 1 : 0
+  compartment_id = local.policy_tenancy
+  description    = "Distributed Training Job Runs Policies"
+  name           = var.distributed_training_policy_name
+  statements     = length(local.bucket_names) > 0 ? concat(local.dt_jr_policies, local.dt_jr_policies_target_buckets) : local.dt_jr_policies
+}
+
+

ai-quick-actions/policies/terraform/outputs.tf

@@ -0,0 +1,7 @@
+output "deployment_type" {
+  value = var.deployment_type
+}
+
+output "aqua_info" {
+  value = "https://docs.oracle.com/en-us/iaas/data-science/using/ai-quick-actions.htm"
+}

ai-quick-actions/policies/terraform/provider.tf

@@ -0,0 +1,12 @@
+
+terraform {
+  required_version = ">= 1.0"
+}
+
+provider "oci" {
+  region       = var.region
+  tenancy_ocid = var.tenancy_ocid
+#  auth = "SecurityToken"
+#  config_file_profile = "DEFAULT"
+}
+

ai-quick-actions/policies/terraform/variables.tf

@@ -0,0 +1,47 @@
+#*************************************
+#          IAM Specific
+#*************************************
+variable "aqua_policy_name" {
+  default = "DataScienceAquaPolicies"
+}
+
+variable "aqua_dg_name" {
+  default = "DataScienceAquaDynamicGroup"
+}
+
+variable "distributed_training_dg_name" {
+  default = "DistributedTrainingJobRunsDynamicGroup"
+}
+
+variable "distributed_training_policy_name" {
+  default = "DistributedTrainingJobRunsPolicies"
+}
+
+#*************************************
+#           TF Requirements
+#*************************************
+variable "tenancy_ocid" {
+}
+variable "region" {
+}
+variable "compartment_ocid" {
+}
+variable "user_model_buckets" {
+  default = []
+  type = list(string)
+  description = "List buckets for storing fine tuning models and evaluation. Important: To save fine-tuned models, versioning has to be enabled in the selected Object Storage bucket."
+}
+variable "user_data_buckets" {
+  default = []
+  type = list(string)
+  description = "List buckets for storing dataset used for fine tuning and evaluation."
+}
+
+variable "deployment_type" {
+  type = string
+  description = "Type of deployment"
+  validation {
+    condition     = contains(["All policies", "Only admin policies", "Only resource policies"], var.deployment_type)
+    error_message = "The deployment_type must be one of: 'All policies', 'Only admin policies', 'Only resource policies'."
+  }
+}