Merge branch 'dciborow/master-cicd' into mabou/parametrize

microsoft · Dec 10, 2019 · 04ad73b · 04ad73b
2 parents 193d090 + 8098844
commit 04ad73b
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 134 deletions.
diff --git a/.ci/azure-pipelines-v2.yml b/.ci/azure-pipelines-v2.yml
@@ -1,134 +1,30 @@
-# MLHyperparameterTuning Pipeline 
-
-trigger: none
-
-variables:
-  BuildConfiguration: Release
-  BuildBinariesDirectory: $(Build.BinariesDirectory)
-  BuildPlatform: any cpu
-  DotNetCoreBuildVersion: 2.2.108
-  DotNetRuntimeTarget: ubuntu.18.04-x64
-  AgentToolsDirectory: $(Agent.ToolsDirectory)
-  CloudPlatform: AzureCloud
-  ProductName: Trident
-  TridentWorkloadType: $(WorkloadType)
-  TridentWorkloadTypeShort: $(WorkloadTypeShort)
-  DeployLocation: eastus
-  Agent: agce-ai
-  azureSubscription: AG-AzureCAT-AIDevOps-Test-COGSNonProd-IO1685734(0ca618d2-22a8-413a-96d0-0f1b531129c3)
-  azure_subscription: 0ca618d2-22a8-413a-96d0-0f1b531129c3
-
-jobs:
-- job: MLHyperparameterTuningJob
-  timeoutInMinutes: 300
-  cancelTimeoutInMinutes: 2
-  pool:
-    vmImage: 'Ubuntu-16.04'
-
-  steps:
-  - bash: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      which conda
-      conda env create -f environment.yml
-      conda env list
-      conda activate MLHyperparameterTuning
-      conda env list  
-      echo Login Azure Account
-      az login -t $(sptenent) --service-principal -u $(spidentity) --password $(spsecret)
-      echo Try and figure out what account set takes
-      az account set -h
-      echo Try and set it.
-      az account set --subscription $(subscriptionid)
-#      papermill 01_Data_Prep.ipynb 01_Data_Prep_Output.ipynb --log-output --no-progress-bar -k python3 
-    displayName: 'Configuration'
-
-  - bash: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate MLHyperparameterTuning
-      echo Executing 00_Data_Prep.ipynb
-      papermill 00_Data_Prep.ipynb 00_Data_Prep_Output.ipynb --log-output --no-progress-bar -k python3 
-    displayName: '00_Data_Prep.ipynb'
-
-  - bash: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate MLHyperparameterTuning
-      echo Executing 01_Training_Script.ipynb
-      papermill 01_Training_Script.ipynb 01_Training_Script_Output.ipynb --log-output --no-progress-bar -k python3 
-    displayName: '01_Training_Script.ipynb'
- 
-  - bash: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate MLHyperparameterTuning
-      echo Executing 02_Testing_Script.ipynb
-      papermill 02_Testing_Script.ipynb 02_Testing_Script_Output.ipynb --log-output --no-progress-bar -k python3 
-    displayName: '02_Testing_Script.ipynb'
- 
-  - bash: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate MLHyperparameterTuning
-      echo Executing 03_Run_Locally.ipynb
-      papermill 03_Run_Locally.ipynb 03_Run_Locally_Output.ipynb --log-output --no-progress-bar -k python3 -p selected_subscription $(subscriptionid) -p resource_group $(azurergname) 
-    displayName: '03_Run_Locally.ipynb'
-
-  - bash: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate MLHyperparameterTuning
-      echo Executing 04_Hyperparameter_Random_Search.ipynb
-      papermill 04_Hyperparameter_Random_Search.ipynb 04_Hyperparameter_Random_Search_Output.ipynb --log-output --no-progress-bar -k python3 -p max_total_runs $(dsmaxruns)
-    displayName: '04_Hyperparameter_Random_Search.ipynb'
-
-  - bash: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate MLHyperparameterTuning
-      echo Executing 05_Train_Best_Model.ipynb
-      papermill 05_Train_Best_Model.ipynb 05_Train_Best_Model_Output.ipynb --log-output --no-progress-bar -k python3 
-    displayName: '05_Train_Best_Model.ipynb'
-
-  - bash: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate MLHyperparameterTuning
-      echo Executing 06_Test_Best_Model.ipynb
-      papermill 06_Test_Best_Model.ipynb 06_Test_Best_Model_Output.ipynb --log-output --no-progress-bar -k python3 
-    displayName: '06_Test_Best_Model.ipynb'
-
-  - bash: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate MLHyperparameterTuning
-      echo Executing 07_Train_With_AML_Pipeline.ipynb
-      papermill 07_Train_With_AML_Pipeline.ipynb 07_Train_With_AML_Pipeline_Output.ipynb --log-output --no-progress-bar -k python3 -p max_total_runs $(dsmaxruns)
-    displayName: '07_Train_With_AML_Pipeline.ipynb'
-
-  - bash: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate MLHyperparameterTuning
-      echo Executing 08_Tear_Down.ipynb
-      papermill 08_Tear_Down.ipynb 08_Tear_Down_Output.ipynb --log-output --no-progress-bar -k python3 
-    displayName: '08_Tear_Down.ipynb'
-
-  - bash: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate MLHyperparameterTuning
-      echo Execute Resource Group Delete
-      existResponse=$(az group exists -n $(azurergname))
-      if [ "$existResponse" == "true" ]; then
-        echo Deleting project resource group  
-        az group delete --name $(azurergname) --yes
-      else
-        echo Project resource group did not exist
-      fi
-      echo Done Cleanup
-    displayName: 'Backup Cleanup'
-    condition: or(canceled(),failed())
-
-  - task: CreateWorkItem@1
-    inputs:
-      workItemType: 'Issue'
-      title: $(System.TeamProject) - Build $(Build.BuildNumber) Failed
-      assignedTo: 'Mario Bourgoin <[email protected]>'
-      associate: true
-      teamProject: $(System.TeamProject)
-
-      fieldMappings: |
-        Description=Branch: Branch $(Build.SourceBranch) failed to build. Go to Boards>WorkItems and tag the failure type.
-    displayName: 'Create work item on failure'
-    condition: failed()
+# MLHyperparameterTuning  Pipeline
+#
+# A Github Service Connection must also be created with the name "AIArchitecturesAndPractices-GitHub"
+
+resources:
+  repositories:
+    - repository: aitemplates
+      type: github
+      name: microsoft/AI
+      endpoint: AIArchitecturesAndPractices-GitHub
+
+trigger:
+  branches:
+    include:
+    - master
+    - mabou/instrument
+
+pr:
+  autoCancel: true
+  branches:
+    include:
+    - master
+    - mabou/instrument
+
+stages:
+- template: .ci/stages/deploy_notebooks_stages_v2.yml@aitemplates
+  parameters:
+    jobDisplayName: MLScoreDeployJob
+    DefaultWorkingDirectory: $(System.DefaultWorkingDirectory)
+    workload_vars: ../vars/mlhyperparametertuning_vars.yml
diff --git a/.ci/vars/agce_devops_sub_vars.yml b/.ci/vars/agce_devops_sub_vars.yml
@@ -0,0 +1,3 @@
+variables:
+  azure_subscription: 0ca618d2-22a8-413a-96d0-0f1b531129c3
+  azureSubscription: AG-AzureCAT-AIDevOps-Test-COGSNonProd-IO1685734(0ca618d2-22a8-413a-96d0-0f1b531129c3)
diff --git a/.ci/vars/mlhyperparametertuning_vars.yml b/.ci/vars/mlhyperparametertuning_vars.yml
@@ -0,0 +1,7 @@
+variables:
+  DeploymentName: MLScoreDeployJob
+  TridentWorkloadTypeShort: aimlscore
+  DeployLocation: eastus  
+  ProjectLocation: "."
+  PythonPath: "."
+  Template: MLTrainDeployAMLJob.yml