Merge pull request #41 from davidkydd/master

AAD and Batch Container Image support, prep for v0.23.0 release

AzureBatchMaya.pyproj

@@ -10,8 +10,10 @@
     <OutputPath>.</OutputPath>
     <Name>Maya.Client</Name>
     <RootNamespace>Maya.Client</RootNamespace>
-    <InterpreterId>{a76b797f-9426-4cd7-a078-9af9c9e4b437}</InterpreterId>
-    <InterpreterVersion>2.7</InterpreterVersion>
+    <InterpreterId>
+    </InterpreterId>
+    <InterpreterVersion>
+    </InterpreterVersion>
     <SearchPath>azure_batch_maya\scripts\;azure_batch_maya\scripts\ui\;tests\data\;tests\data\modules\</SearchPath>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
@@ -34,9 +36,18 @@
     <Compile Include="azure_batch_maya\scripts\assets.py">
       <SubType>Code</SubType>
     </Compile>
+    <Compile Include="azure_batch_maya\scripts\aadEnvironmentProvider.py">
+      <SubType>Code</SubType>
+    </Compile>
+    <Compile Include="azure_batch_maya\scripts\poolImageProvider.py">
+      <SubType>Code</SubType>
+    </Compile>
     <Compile Include="azure_batch_maya\scripts\config.py">
       <SubType>Code</SubType>
     </Compile>
+    <Compile Include="azure_batch_maya\scripts\poolImageFilter.py">
+      <SubType>Code</SubType>
+    </Compile>
     <Compile Include="azure_batch_maya\scripts\environment.py">
       <SubType>Code</SubType>
     </Compile>
@@ -59,9 +70,15 @@
     </Compile>
     <Compile Include="azure_batch_maya\scripts\tools\getpip.py" />
     <Compile Include="azure_batch_maya\scripts\tools\job_watcher.py" />
+    <Compile Include="azure_batch_maya\scripts\tools\refreshsession.py">
+      <SubType>Code</SubType>
+    </Compile>
     <Compile Include="azure_batch_maya\scripts\ui\ui_assets.py">
       <SubType>Code</SubType>
     </Compile>
+    <Compile Include="azure_batch_maya\scripts\ui\ui_containerImage.py">
+      <SubType>Code</SubType>
+    </Compile>
     <Compile Include="azure_batch_maya\scripts\ui\ui_config.py">
       <SubType>Code</SubType>
     </Compile>
@@ -84,6 +101,10 @@
       <SubType>Code</SubType>
     </Compile>
     <Compile Include="package.py" />
+    <Compile Include="tests\data\modules\default.py" />
+    <Compile Include="tests\data\modules\render_module_a.py" />
+    <Compile Include="tests\data\modules\render_module_b.py" />
+    <Compile Include="tests\data\modules\render_module_c.py" />
     <Compile Include="tests\test_assets.py">
       <SubType>Code</SubType>
     </Compile>
@@ -96,6 +117,9 @@
     <Compile Include="tests\test_jobwatcher.py">
       <SubType>Code</SubType>
     </Compile>
+    <Compile Include="tests\test_poolImages.py">
+      <SubType>Code</SubType>
+    </Compile>
     <Compile Include="tests\test_pools.py">
       <SubType>Code</SubType>
     </Compile>
@@ -113,10 +137,16 @@
     <Folder Include="azure_batch_maya\scripts\tools\" />
     <Folder Include="azure_batch_maya\scripts\ui\" />
     <Folder Include="azure_batch_maya\templates\" />
+    <Folder Include="azure_batch_maya\templates\containers\" />
     <Folder Include="docs\" />
     <Folder Include="docs\images\" />
     <Folder Include="tests\" />
-    <Folder Include="tests\test_scene\" />
+    <Folder Include="tests\basic_test_scene\" />
+    <Folder Include="tests\data\modules\" />
+    <Folder Include="tests\json_data\" />
+    <Folder Include="tests\json_data\file_tests\" />
+    <Folder Include="tests\json_data\file_tests\sample_data\" />
+    <Folder Include="tests\data\" />
   </ItemGroup>
   <ItemGroup>
     <Content Include="azure_batch_maya\icons\btn_background.png" />
@@ -143,6 +173,18 @@
     <Content Include="azure_batch_maya\templates\arnold-2017-windows.json" />
     <Content Include="azure_batch_maya\templates\arnold-2018-linux.json" />
     <Content Include="azure_batch_maya\templates\arnold-2018-windows.json" />
+    <Content Include="azure_batch_maya\templates\containers\arnold-2017-linux.json" />
+    <Content Include="azure_batch_maya\templates\containers\arnold-2017-windows.json" />
+    <Content Include="azure_batch_maya\templates\containers\arnold-2018-linux.json" />
+    <Content Include="azure_batch_maya\templates\containers\arnold-2018-windows.json" />
+    <Content Include="azure_batch_maya\templates\containers\mayaSoftware-2017-linux.json" />
+    <Content Include="azure_batch_maya\templates\containers\mayaSoftware-2017-windows.json" />
+    <Content Include="azure_batch_maya\templates\containers\mayaSoftware-2018-linux.json" />
+    <Content Include="azure_batch_maya\templates\containers\mayaSoftware-2018-windows.json" />
+    <Content Include="azure_batch_maya\templates\containers\vray-2017-linux.json" />
+    <Content Include="azure_batch_maya\templates\containers\vray-2017-windows.json" />
+    <Content Include="azure_batch_maya\templates\containers\vray-2018-linux.json" />
+    <Content Include="azure_batch_maya\templates\containers\vray-2018-windows.json" />
     <Content Include="azure_batch_maya\templates\mayaSoftware-2017-linux.json" />
     <Content Include="azure_batch_maya\templates\mayaSoftware-2017-windows.json" />
     <Content Include="azure_batch_maya\templates\mayaSoftware-2018-linux.json" />
@@ -162,19 +204,28 @@
     <Content Include="requirements.txt">
       <SubType>Code</SubType>
     </Content>
-  </ItemGroup>
-  <ItemGroup>
-    <Interpreter Include="env\">
-      <Id>{a76b797f-9426-4cd7-a078-9af9c9e4b437}</Id>
-      <BaseInterpreter>{9a7a9026-48c1-4688-9d5d-e5699d47d074}</BaseInterpreter>
-      <Version>2.7</Version>
-      <Description>env (Python 64-bit 2.7)</Description>
-      <InterpreterPath>Scripts\python.exe</InterpreterPath>
-      <WindowsInterpreterPath>Scripts\pythonw.exe</WindowsInterpreterPath>
-      <LibraryPath>Lib\</LibraryPath>
-      <PathEnvironmentVariable>PYTHONPATH</PathEnvironmentVariable>
-      <Architecture>Amd64</Architecture>
-    </Interpreter>
+    <Content Include="tests\basic_test_scene\asset_map.mel" />
+    <Content Include="tests\basic_test_scene\test_scene.mb" />
+    <Content Include="tests\basic_test_scene\workspace.mel" />
+    <Content Include="tests\data\azure_batch_test.ini" />
+    <Content Include="tests\data\empty.mb" />
+    <Content Include="tests\data\star.png" />
+    <Content Include="tests\json_data\batch-applicationTemplate-parameters.json" />
+    <Content Include="tests\json_data\batch-applicationTemplate-prohibitedApplicationTemplateInfo.json" />
+    <Content Include="tests\json_data\batch-applicationTemplate-prohibitedId.json" />
+    <Content Include="tests\json_data\batch-applicationTemplate-prohibitedPoolInfo.json" />
+    <Content Include="tests\json_data\batch-applicationTemplate-prohibitedPriority.json" />
+    <Content Include="tests\json_data\batch-applicationTemplate-static.json" />
+    <Content Include="tests\json_data\batch-applicationTemplate-unsupportedProperty.json" />
+    <Content Include="tests\json_data\batch-applicationTemplate-untypedParameter.json" />
+    <Content Include="tests\json_data\batch.job.parameters.json" />
+    <Content Include="tests\json_data\batch.job.parametricsweep.json" />
+    <Content Include="tests\json_data\batch.job.simple.json" />
+    <Content Include="tests\json_data\batch.pool.parameters.json" />
+    <Content Include="tests\json_data\batch.pool.simple.json" />
+    <Content Include="tests\json_data\file_tests\bar.bmp" />
+    <Content Include="tests\json_data\file_tests\foo.txt" />
+    <Content Include="tests\json_data\file_tests\sample_data\test.txt" />
   </ItemGroup>
   <PropertyGroup>
     <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">10.0</VisualStudioVersion>

CHANGES.txt

@@ -1,3 +1,10 @@
+2018.11.07 v0.23.0
+------------------
+- Change authentication mode to AAD, including support for private cloud
+- Support for Batch Container Images + initial set of images for Maya2017 and 2018
+- Updated VM SKU list to match what is available for Maya in BatchExplorer
+
+
 2018.09.24 v0.22.0
 ------------------
 - Revert template change which broke thumbnailing of outputs in arnold maya 2018 jobs

README.md

@@ -1,13 +1,11 @@
 # Azure Batch for Maya
 
-This project demonstrates cloud rendering using the Azure Batch service with integrated licensing for Maya, VRay and Arnold.
+This project demonstrates cloud rendering using the Azure Batch service with integrated licensing for Maya, VRay and Arnold. It is supported for Windows and Mac.
 
-Please note that the Azure Batch licensing service for Maya is currently in preview.
-For more information and to register your interest, please see [rendering.azure.com](rendering.azure.com).
+For more information, please see [rendering.azure.com](https://rendering.azure.com).
 
-In order to run this sample, you will need to [create an Azure Batch account](https://docs.microsoft.com/azure/batch/batch-account-create-portal).
-You will also need a storage account. You will have the option to create a storage account as part of the Batch account setup for use as `Auto Storage`.
-You can use this, or you can [set up a storage account independently](https://docs.microsoft.com/azure/storage/storage-create-storage-account).
+In order to run this plugin, you will need to [create an Azure Batch account](https://docs.microsoft.com/azure/batch/batch-account-create-portal).
+You will also need an `Auto Storage` account configured, which you will have the option to create as part of the Batch account setup.
 
 
 ## Loading the plug-in in Maya and installing dependencies
@@ -27,28 +25,30 @@ The first time the plug-in is loaded, you will be prompted to agree to some term
 The downloading and installing of the Python dependencies may take a few minutes, after which you will need to close and reopen Maya to
 ensure the updated dependencies are loaded correctly.
 
-Any errors in the dependency install will be logged to the file "AzureBatchInstall.log" in the "azure-batch-libs" folder, which is created for holding dependencies. In windows this is located at: Users\<username>\Documents\maya\<version>\scripts\azure-batch-libs
+Any errors in the dependency install will be logged to the file "AzureBatchInstall.log" in the "azure-batch-libs" folder, which is created for holding dependencies. 
+On Windows this is located at: Users\<username>\Documents\maya\<version>\scripts\azure-batch-libs
+On Mac this is located at: /Users/<username>/Library/Preferences/Autodesk/maya/<version>/scripts/azure-batch-libs
 
 
 ![](./docs/images/install_dependencies.png)
 
 
 ## Authentication
 
-Before using the plug-in, it will need to be authenticated using your Azure Batch and Azure Storage account keys.
-In order to retrieve this information:
+Before using the plug-in, it will need to be authenticated using your Azure Active Directory (AAD) credentials.
+In order to retrieve your AAD tenant:
 
-1. Open the Azure management portal (portal.azure.com).
-2. Select Azure Batch Accounts in the left-hand menu. This can be found under `More Services` in the `Compute` category.
-3. Select your account in the list. Copy and paste the account URL into `Service` field of the plug-in UI. Paste the account name into the `Batch Account` field.
-4. In the portal, select `Keys` on the left-hand menu. Copy and paste one of the access keys into the `Batch Key` field in the plug-in.
-5. Return to the management portal home, and select Storage Accounts from the left-hand menu. This can be found under `More Services` in the `Storage` category.
-6. Select your account from the list. Copy and paste the account name into the `Storage Account` field.
-7. In the portal, select `Access Keys` on the left-hand menu. Copy and paste one of the access keys into the `Storage Key` field.
-8. Click `Authenticate`.
+1. Open the Azure management portal (for public cloud this is [portal.azure.com](https://portal.azure.com) ).
+2. Hover over your account name on the upper right-hand side, your AAD Domain will be the bottom value shown, e.g. `contoso.onmicrosoft.com`
+3. Open the plugin, select Cloud Environment using the dropdown and copy your AAD domain into the appropriate field
+4. Follow the instructions to sign-in by entering the device code provided at [aka.ms/devicelogin](https://aka.ms/devicelogin)
 
 ![](./docs/images/authentication.png)
 
+5. Select a Subscription and Batch Account from the dropdown menus in order to complete Batch Account configuration
+
+![](./docs/images/accountConfiguration.png)
+
 ## Using the Azure Batch plug-in
 
 - [Job configuration](./docs/submitting_jobs.md#job-configuration)
@@ -69,8 +69,9 @@ To uninstall the plug-in:
 5. Confirm the removal of the shelf 
 6. Confirm to remove the dependencies if you wish a clean uninstall
 
-To remove config which is retained for a reinstall, delete the config file "azure_batch.ini". For Windows this is located in:
-Users\<username>\Documents\maya\2017\prefs\AzureBatchData
+To remove config which is retained for a reinstall, delete the config file "azure_batch.ini". 
+On Windows this is located in: Users\<username>\Documents\maya\2017\prefs\AzureBatchData
+On Mac this is located at: /Users/<username>/Library/Preferences/Autodesk/maya/<version>/prefs/AzureBatchData
 
 ##  Upgrading or reinstalling
 

azure_batch_maya/mel/create_shelf.mel

@@ -1,9 +1,11 @@
-python("from shared import AzureBatchSettings");
+python("import refreshsession");
 
 $batch_path = `getenv "AZUREBATCH_ICONS"`;
 
 global proc run_guiStarter()
 {
+    python("refreshsession.refreshSessionModules()");
+    python("from shared import AzureBatchSettings");
     python("AzureBatchSettings.starter()");
 }
 

azure_batch_maya/plug-in/AzureBatch.py

@@ -38,20 +38,23 @@
 REQUIREMENTS = {
     "pathlib==1.0.1": "pathlib",
     "futures==3.1.1": "concurrent.futures",
+    "keyring==5.6": "keyring",
+    "msrestazure==0.4.16": "msrestazure",
+    "azure-common==1.1.8": "azure.common",
     "configparser==3.5.0": "backports.configparser",
-    "msrestazure==0.4.14": "msrestazure",
-    "azure-common==1.1.8": "azure.common"
+    "python-dateutil==2.6.1": "dateutil",
 }
 
 NAMESPACE_PACKAGES = {
     "azure-mgmt-batch==4.0.0": "azure.mgmt.batch",
     "azure-mgmt-storage==1.0.0": "azure.mgmt.storage",
+    "azure-mgmt-resource==1.2.2": "azure.mgmt.resource",
     "azure-batch==4.0.0": "azure.batch",
     "azure-storage==0.32.0": "azure.storage",
     "azure-batch-extensions==1.0.1": "azure.batch_extensions"
 }
 
-VERSION = "0.22.0"
+VERSION = "0.23.0"
 EULA_PREF = "AzureBatch_EULA"
 SHELF_FILE = "shelf_AzureBatch.mel"
 cmd_name = "AzureBatch"

azure_batch_maya/scripts/aadEnvironmentProvider.py

@@ -0,0 +1,40 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import json
+import sys
+import traceback
+from collections import OrderedDict
+
+from msrestazure.azure_cloud import AZURE_CHINA_CLOUD, AZURE_PUBLIC_CLOUD, AZURE_GERMAN_CLOUD, AZURE_US_GOV_CLOUD
+
+AAD_ENVIRONMENTS = OrderedDict()
+AAD_ENVIRONMENTS[AZURE_PUBLIC_CLOUD.name] = AZURE_PUBLIC_CLOUD
+AAD_ENVIRONMENTS[AZURE_CHINA_CLOUD.name] = AZURE_CHINA_CLOUD
+AAD_ENVIRONMENTS[AZURE_GERMAN_CLOUD.name] = AZURE_GERMAN_CLOUD 
+AAD_ENVIRONMENTS[AZURE_US_GOV_CLOUD.name] = AZURE_US_GOV_CLOUD 
+
+class AADEnvironmentProvider(object):
+
+    def __init__(self, aad_environments = AAD_ENVIRONMENTS):
+        self.aadEnvironments = aad_environments
+
+    def  getAADEnvironments(self):
+        return self.aadEnvironments
+
+    def getEnvironmentForId(self, id):
+        return AAD_ENVIRONMENTS[id]
+
+    def getAadAuthorityHostUrl(self, id):
+        return AAD_ENVIRONMENTS[id].endpoints.active_directory
+
+    def getAadManagementUrl(self, id):
+        return AAD_ENVIRONMENTS[id].endpoints.active_directory_resource_id
+
+    def getBatchResourceUrl(self, id):
+        return AAD_ENVIRONMENTS[id].endpoints.batch_resource_id
+
+    def getResourceManager(self, id):
+        return AAD_ENVIRONMENTS[id].endpoints.resource_manager