Merge branch 'main' into lishakur/1889-tag-gitea

.github/workflows/pr_comment_bot.yml

@@ -144,9 +144,9 @@ jobs:
       prRef: ${{ needs.pr_comment.outputs.prRef }}
       prHeadSha: ${{ needs.pr_comment.outputs.prHeadSha }}
       ciGitRef: ${{ needs.pr_comment.outputs.ciGitRef }}
-      e2eTestsCustomSelector: |
+      e2eTestsCustomSelector: >-
         ${{ (needs.pr_comment.outputs.command == 'run-tests-extended' && 'extended') ||
-        (needs.pr_comment.outputs.command == 'run-tests-shared-services' && 'shared_sevices')}}
+        (needs.pr_comment.outputs.command == 'run-tests-shared-services' && 'shared_sevices') }}
     secrets:
       AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }}
       ACR_NAME: ${{ format('tre{0}', needs.pr_comment.outputs.prRefId) }}

Makefile

@@ -10,13 +10,14 @@ FULL_IMAGE_NAME_PREFIX:=`echo "${FULL_CONTAINER_REGISTRY_NAME}/${IMAGE_NAME_PREF
 target_title = @echo -e "\n\e[34m»»» 🧩 \e[96m$(1)\e[0m..."
 
 all: bootstrap mgmt-deploy images tre-deploy
-images: build-and-push-api build-and-push-resource-processor build-and-push-gitea build-and-push-guacamole build-and-push-mlflow
+images: build-and-push-api build-and-push-resource-processor build-and-push-gitea build-and-push-guacamole build-and-push-mlflow build-and-push-airlock-processor
 
 build-and-push-api: build-api-image push-api-image
 build-and-push-resource-processor: build-resource-processor-vm-porter-image push-resource-processor-vm-porter-image
 build-and-push-gitea: build-gitea-image push-gitea-image
 build-and-push-guacamole: build-guacamole-image push-guacamole-image
 build-and-push-mlflow: build-mlflow-image push-mlflow-image
+build-and-push-airlock-processor: build-airlock-processor push-airlock-processor
 tre-deploy: deploy-core deploy-shared-services db-migrate show-core-output
 deploy-shared-services:
 	$(MAKE) firewall-install \
@@ -88,6 +89,9 @@ build-guacamole-image:
 build-mlflow-image:
 	$(call build_image,"mlflow-server","${MAKEFILE_DIR}/templates/workspace_services/mlflow/mlflow-server/version.txt","${MAKEFILE_DIR}/templates/workspace_services/mlflow/mlflow-server/docker/Dockerfile","${MAKEFILE_DIR}/templates/workspace_services/mlflow/mlflow-server")
 
+build-airlock-processor:
+	$(call build_image,"airlock-processor","${MAKEFILE_DIR}/airlock_processor/_version.py","${MAKEFILE_DIR}/airlock_processor/Dockerfile","${MAKEFILE_DIR}/airlock_processor/")
+
 firewall-install:
 	$(MAKE) bundle-build DIR=${MAKEFILE_DIR}/templates/shared_services/firewall/ \
 	&& $(MAKE) bundle-publish DIR=${MAKEFILE_DIR}/templates/shared_services/firewall/ \
@@ -138,6 +142,9 @@ push-guacamole-image:
 push-mlflow-image:
 	$(call push_image,"mlflow-server","${MAKEFILE_DIR}/templates/workspace_services/mlflow/mlflow-server/version.txt")
 
+push-airlock-processor:
+	$(call push_image,"airlock-processor","${MAKEFILE_DIR}/airlock_processor/_version.py")
+
 # # These targets are for a graceful migration of Firewall
 # # from terraform state in Core to a Shared Service.
 # # See https://github.com/microsoft/AzureTRE/issues/1177

airlock_processor/.dockerignore

@@ -0,0 +1 @@
+local.settings.json

airlock_processor/.gitignore

@@ -0,0 +1,48 @@
+bin
+obj
+csx
+.vs
+edge
+Publish
+
+*.user
+*.suo
+*.cscfg
+*.Cache
+project.lock.json
+
+/packages
+/TestResults
+
+/tools/NuGet.exe
+/App_Data
+/secrets
+/data
+.secrets
+appsettings.json
+local.settings.json
+
+node_modules
+dist
+
+# Local python packages
+.python_packages/
+
+# Python Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Azurite artifacts
+__blobstorage__
+__queuestorage__
+__azurite_db*__.json

airlock_processor/Dockerfile

@@ -0,0 +1,11 @@
+# To enable ssh & remote debugging on app service change the base image to the one below
+# FROM mcr.microsoft.com/azure-functions/python:3.0-python3.8-appservice
+FROM mcr.microsoft.com/azure-functions/python:3.0-python3.8
+
+ENV AzureWebJobsScriptRoot=/home/site/wwwroot \
+    AzureFunctionsJobHost__Logging__Console__IsEnabled=true
+
+COPY requirements.txt /
+RUN pip install --no-cache-dir -r /requirements.txt
+
+COPY . /home/site/wwwroot

airlock_processor/StatusChangedQueueTrigger/__init__.py

@@ -0,0 +1,18 @@
+import logging
+
+import azure.functions as func
+import datetime
+
+
+def main(msg: func.ServiceBusMessage,
+         outputEvent: func.Out[func.EventGridOutputEvent]):
+
+    logging.info('Python ServiceBus queue trigger processed message: %s', msg.get_body().decode('utf-8'))
+    outputEvent.set(
+        func.EventGridOutputEvent(
+            id="step-result-id",
+            data={"tag1": "value1", "tag2": "value2"},
+            subject="test-subject",
+            event_type="test-event-1",
+            event_time=datetime.datetime.utcnow(),
+            data_version="1.0"))

airlock_processor/StatusChangedQueueTrigger/function.json

@@ -0,0 +1,19 @@
+{
+  "scriptFile": "__init__.py",
+  "bindings": [
+    {
+      "name": "msg",
+      "type": "serviceBusTrigger",
+      "direction": "in",
+      "queueName": "%AIRLOCK_STATUS_CHANGED_QUEUE_NAME%",
+      "connection": "SB_CONNECTION_STRING"
+    },
+    {
+      "type": "eventGrid",
+      "name": "outputEvent",
+      "topicEndpointUri": "EVENT_GRID_TOPIC_URI_SETTING",
+      "topicKeySetting": "EVENT_GRID_TOPIC_KEY_SETTING",
+      "direction": "out"
+    }
+  ]
+}

airlock_processor/_version.py

@@ -0,0 +1 @@
+__version__ = "0.0.2"

airlock_processor/host.json

@@ -0,0 +1,15 @@
+{
+  "version": "2.0",
+  "logging": {
+    "applicationInsights": {
+      "samplingSettings": {
+        "isEnabled": true,
+        "excludedTypes": "Request"
+      }
+    }
+  },
+  "extensionBundle": {
+    "id": "Microsoft.Azure.Functions.ExtensionBundle",
+    "version": "[3.3.0, 4.0.0)"
+  }
+}

airlock_processor/requirements.txt

@@ -0,0 +1,3 @@
+# Do not include azure-functions-worker as it may conflict with the Azure Functions platform
+
+azure-functions

templates/core/terraform/airlock/airlock_processor.tf

@@ -0,0 +1,72 @@
+data "local_file" "airlock_processor_version" {
+  filename = "${path.root}/../../../airlock_processor/_version.py"
+}
+
+locals {
+  version = replace(replace(replace(data.local_file.airlock_processor_version.content, "__version__ = \"", ""), "\"", ""), "\n", "")
+}
+
+# re-using the web api app plan
+data "azurerm_app_service_plan" "core" {
+  name                = "plan-${var.tre_id}"
+  resource_group_name = var.resource_group_name
+}
+
+data "azurerm_application_insights" "core" {
+  name                = "appi-${var.tre_id}"
+  resource_group_name = var.resource_group_name
+}
+
+
+resource "azurerm_storage_account" "sa_airlock_processor_func_app" {
+  name                     = local.airlock_function_sa_name
+  resource_group_name      = var.resource_group_name
+  location                 = var.location
+  account_tier             = "Standard"
+  account_replication_type = "LRS"
+  tags                     = local.tre_core_tags
+
+  lifecycle { ignore_changes = [tags] }
+}
+
+resource "azurerm_linux_function_app" "airlock_function_app" {
+  name                = local.airlock_function_app_name
+  resource_group_name = var.resource_group_name
+  location            = var.location
+
+  storage_account_name = azurerm_storage_account.sa_airlock_processor_func_app.name
+  service_plan_id      = data.azurerm_app_service_plan.core.id
+
+  storage_account_access_key = azurerm_storage_account.sa_airlock_processor_func_app.primary_access_key
+  tags                       = local.tre_core_tags
+
+  identity {
+    type         = "UserAssigned"
+    identity_ids = [azurerm_user_assigned_identity.airlock_id.id]
+  }
+
+  app_settings = {
+    "SB_CONNECTION_STRING"                = data.azurerm_servicebus_namespace.airlock_sb.default_primary_connection_string
+    "EVENT_GRID_TOPIC_URI_SETTING"        = azurerm_eventgrid_topic.step_result.endpoint
+    "EVENT_GRID_TOPIC_KEY_SETTING"        = azurerm_eventgrid_topic.step_result.primary_access_key
+    "WEBSITES_ENABLE_APP_SERVICE_STORAGE" = false
+    "AIRLOCK_STATUS_CHANGED_QUEUE_NAME"   = "airlock-status-changed"
+    "APPINSIGHTS_INSTRUMENTATIONKEY"      = data.azurerm_application_insights.core.instrumentation_key
+  }
+
+  site_config {
+    always_on                                     = var.enable_local_debugging ? true : false
+    container_registry_managed_identity_client_id = azurerm_user_assigned_identity.airlock_id.client_id
+    container_registry_use_managed_identity       = true
+    application_stack {
+      docker {
+        registry_url = var.docker_registry_server
+        image_name   = var.airlock_processor_image_repository
+        image_tag    = local.version
+      }
+    }
+  }
+
+  lifecycle { ignore_changes = [tags] }
+}
+

templates/core/terraform/airlock/identity.tf

@@ -0,0 +1,43 @@
+data "azurerm_cosmosdb_account" "tre-db-account" {
+  name                = "cosmos-${var.tre_id}"
+  resource_group_name = var.resource_group_name
+}
+
+data "azurerm_container_registry" "mgmt_acr" {
+  name                = var.mgmt_acr_name
+  resource_group_name = var.mgmt_resource_group_name
+}
+
+resource "azurerm_user_assigned_identity" "airlock_id" {
+  resource_group_name = var.resource_group_name
+  location            = var.location
+  tags                = local.tre_core_tags
+
+  name = "id-airlock-${var.tre_id}"
+
+  lifecycle { ignore_changes = [tags] }
+}
+
+resource "azurerm_role_assignment" "acrpull_role" {
+  scope                = data.azurerm_container_registry.mgmt_acr.id
+  role_definition_name = "AcrPull"
+  principal_id         = azurerm_user_assigned_identity.airlock_id.principal_id
+}
+
+resource "azurerm_role_assignment" "servicebus_sender" {
+  scope                = data.azurerm_servicebus_namespace.airlock_sb.id
+  role_definition_name = "Azure Service Bus Data Sender"
+  principal_id         = azurerm_user_assigned_identity.airlock_id.principal_id
+}
+
+resource "azurerm_role_assignment" "servicebus_receiver" {
+  scope                = data.azurerm_servicebus_namespace.airlock_sb.id
+  role_definition_name = "Azure Service Bus Data Receiver"
+  principal_id         = azurerm_user_assigned_identity.airlock_id.principal_id
+}
+
+resource "azurerm_role_assignment" "cosmos_contributor" {
+  scope                = data.azurerm_cosmosdb_account.tre-db-account.id
+  role_definition_name = "Contributor"
+  principal_id         = azurerm_user_assigned_identity.airlock_id.principal_id
+}

templates/core/terraform/airlock/locals.tf

@@ -33,4 +33,11 @@ locals {
   import_rejected_eventgrid_subscription_name   = "evgs-airlock-import-rejected-blob-created"
   export_approved_eventgrid_subscription_name   = "evgs-airlock-export-approved-blob-created"
 
+  airlock_function_app_name = "func-airlock-processor-${var.tre_id}"
+  airlock_function_sa_name  = "saairlockp${var.tre_id}"
+
+  tre_core_tags = {
+    tre_id              = var.tre_id
+    tre_core_service_id = var.tre_id
+  }
 }

templates/core/terraform/airlock/service_bus.tf

@@ -2,7 +2,6 @@
 data "azurerm_servicebus_namespace" "airlock_sb" {
   name                = "sb-${var.tre_id}"
   resource_group_name = var.resource_group_name
-
 }
 
 resource "azurerm_servicebus_queue" "step_result" {

templates/core/terraform/airlock/storage_accounts.tf

@@ -125,8 +125,6 @@ resource "azurerm_private_endpoint" "stgipimportpe" {
   resource_group_name = var.resource_group_name
   subnet_id           = var.shared_subnet_id
 
-  lifecycle { ignore_changes = [tags] }
-
   private_dns_zone_group {
     name                 = "private-dns-zone-group-stg-import-rej"
     private_dns_zone_ids = [data.azurerm_private_dns_zone.blobcore.id]
@@ -138,4 +136,8 @@ resource "azurerm_private_endpoint" "stgipimportpe" {
     is_manual_connection           = false
     subresource_names              = ["Blob"]
   }
+
+  tags = local.tre_core_tags
+
+  lifecycle { ignore_changes = [tags] }
 }

templates/core/terraform/airlock/variables.tf

@@ -3,3 +3,24 @@ variable "location" {}
 variable "resource_group_name" {}
 variable "shared_subnet_id" {}
 variable "enable_local_debugging" {}
+
+variable "docker_registry_server" {
+  type        = string
+  description = "Docker registry server"
+}
+
+variable "airlock_processor_image_repository" {
+  type        = string
+  description = "Repository for Airlock processor image"
+  default     = "microsoft/azuretre/airlock-processor"
+}
+
+variable "mgmt_resource_group_name" {
+  type        = string
+  description = "Shared management resource group"
+}
+
+variable "mgmt_acr_name" {
+  type        = string
+  description = "Management ACR name"
+}

templates/core/terraform/appgateway/staticweb.tf

@@ -10,8 +10,7 @@ resource "azurerm_storage_account" "staticweb" {
   account_replication_type        = "LRS"
   enable_https_traffic_only       = true
   allow_nested_items_to_be_public = false
-
-  tags = local.tre_core_tags
+  tags                            = local.tre_core_tags
 
   static_website {
     index_document     = "index.html"

templates/core/terraform/main.tf

@@ -73,13 +73,15 @@ module "appgateway" {
 }
 
 module "airlock_resources" {
-  source                 = "./airlock"
-  tre_id                 = var.tre_id
-  location               = var.location
-  resource_group_name    = azurerm_resource_group.core.name
-  shared_subnet_id       = module.network.shared_subnet_id
-  enable_local_debugging = var.enable_local_debugging
-
+  source                   = "./airlock"
+  tre_id                   = var.tre_id
+  location                 = var.location
+  resource_group_name      = azurerm_resource_group.core.name
+  shared_subnet_id         = module.network.shared_subnet_id
+  enable_local_debugging   = var.enable_local_debugging
+  docker_registry_server   = var.docker_registry_server
+  mgmt_resource_group_name = var.mgmt_resource_group_name
+  mgmt_acr_name            = var.acr_name
   depends_on = [
     azurerm_servicebus_namespace.sb,
     module.network