From 6408c5a31d85b9b79c5cd76b59a2dcc0e05a9dda Mon Sep 17 00:00:00 2001 From: Mario Bourgoin Date: Fri, 6 Dec 2019 17:56:58 -0500 Subject: [PATCH 01/10] Added an 'estimators' parameter --- 01_Training_Script.ipynb | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/01_Training_Script.ipynb b/01_Training_Script.ipynb index 2e35491..d539ab5 100644 --- a/01_Training_Script.ipynb +++ b/01_Training_Script.ipynb @@ -440,7 +440,27 @@ "metadata": {}, "source": [ "## Run the script to see that it works \n", - "This should take around ten minutes." + "Set the effort expended to train the classifier." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "estimators = 1000" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Run the classifier script. This should take about 10 minutes." ] }, { @@ -451,7 +471,7 @@ }, "outputs": [], "source": [ - "%run -t scripts/TrainClassifier.py --estimators 1000 --match 5 --ngrams 2 --min_child_samples 10 --save model" + "%run -t scripts/TrainClassifier.py --estimators $estimators --match 5 --ngrams 2 --min_child_samples 10 --save model" ] }, { From fa6c14f9d6fab8b62bf481d21d8a43d02ff2c482 Mon Sep 17 00:00:00 2001 From: Mario Bourgoin Date: Fri, 6 Dec 2019 17:57:21 -0500 Subject: [PATCH 02/10] Added an 'estimators' parameter --- 03_Run_Locally.ipynb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/03_Run_Locally.ipynb b/03_Run_Locally.ipynb index 22879c4..fa4e09f 100644 --- a/03_Run_Locally.ipynb +++ b/03_Run_Locally.ipynb @@ -39,7 +39,7 @@ "metadata": {}, "source": [ "## Azure subscription \n", - "If you have multiple subscriptions select the subscription you want to use. You may supply either the subscription's name or the subscription's ID. If you want to run this in a different location that supports HyperDrive, you may enter the one you want to use. You can also set the name of the resource group in which this tutorial will add resources. *IMPORTANT NOTE:* The last notebook in this example will delete this resource group and all associated resources." + "If you have multiple subscriptions select the subscription you want to use. You may supply either the subscription's name or the subscription's ID. If you want to run this in a different location that supports HyperDrive, you may enter the one you want to use. You can also set the name of the resource group in which this tutorial will add resources. *IMPORTANT NOTE:* The last notebook in this example will delete this resource group and all associated resources. We also define the number of estimators to use for the local run." ] }, { @@ -55,7 +55,8 @@ "subscription_name=\"YOUR_SUBSCRIPTION_NAME\"\n", "subscription_id=\"YOUR_SUBSCRIPTION_ID\"\n", "location=\"eastus\"\n", - "resource_group=\"hypetuning\"" + "resource_group=\"hypetuning\"\n", + "estimators = 1000" ] }, { @@ -179,10 +180,10 @@ "est = Estimator(source_directory=os.path.join('.', 'scripts'), \n", " entry_script='TrainClassifier.py',\n", " script_params={'--data-folder': os.path.abspath('.'),\n", - " '--estimators': '1000',\n", - " '--match': '5',\n", - " '--ngrams': '2',\n", - " '--min_child_samples': '10',\n", + " '--estimators': estimators,\n", + " '--match': 5,\n", + " '--ngrams': 2,\n", + " '--min_child_samples': 10,\n", " \"--save\": \"local_model\"},\n", " compute_target='local',\n", " conda_packages=['pandas==0.23.4',\n", From dc23467a7eb88a2df1a88a62c6599c89d435cd92 Mon Sep 17 00:00:00 2001 From: Mario Bourgoin Date: Fri, 6 Dec 2019 17:57:51 -0500 Subject: [PATCH 03/10] Added an 'estimators' parameter --- 04_Hyperparameter_Random_Search.ipynb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/04_Hyperparameter_Random_Search.ipynb b/04_Hyperparameter_Random_Search.ipynb index 1255112..5533e0c 100644 --- a/04_Hyperparameter_Random_Search.ipynb +++ b/04_Hyperparameter_Random_Search.ipynb @@ -202,7 +202,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This hyperparameter space specifies a grid of 9,360 unique configuration points (4 `ngrams` X 39 `match` X 30 `min_child_samples` X 2 `unweighted`). We control the resources used by the search through specifying a maximum number of configuration points to sample as `max_total_runs`." + "This hyperparameter space specifies a grid of 9,360 unique configuration points (4 `ngrams` X 39 `match` X 30 `min_child_samples` X 2 `unweighted`). We control the resources used by the search through specifying a maximum number of configuration points to sample as `max_total_runs`. We also define the number of estimators to use for each run." ] }, { @@ -215,7 +215,8 @@ }, "outputs": [], "source": [ - "max_total_runs = 96" + "max_total_runs = 96\n", + "estimators = 1000" ] }, { @@ -270,7 +271,7 @@ "estimator = Estimator(source_directory=os.path.join('.', 'scripts'),\n", " entry_script='TrainClassifier.py',\n", " script_params={'--data-folder': ds.as_mount(),\n", - " '--estimators': 1000},\n", + " '--estimators': estimators},\n", " compute_target=compute_target,\n", " conda_packages=['pandas==0.23.4',\n", " 'scikit-learn==0.21.3',\n", From 8edfaa2653f647d8ea04b9ee5a449e06f91ef69f Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 6 Dec 2019 18:34:42 -0500 Subject: [PATCH 04/10] Update azure-pipelines-v2.yml --- .ci/azure-pipelines-v2.yml | 277 ++++++++++++++++++++----------------- 1 file changed, 153 insertions(+), 124 deletions(-) diff --git a/.ci/azure-pipelines-v2.yml b/.ci/azure-pipelines-v2.yml index abfb6df..1ddd0f1 100644 --- a/.ci/azure-pipelines-v2.yml +++ b/.ci/azure-pipelines-v2.yml @@ -3,132 +3,161 @@ trigger: none variables: - BuildConfiguration: Release - BuildBinariesDirectory: $(Build.BinariesDirectory) - BuildPlatform: any cpu - DotNetCoreBuildVersion: 2.2.108 - DotNetRuntimeTarget: ubuntu.18.04-x64 - AgentToolsDirectory: $(Agent.ToolsDirectory) CloudPlatform: AzureCloud - ProductName: Trident - TridentWorkloadType: $(WorkloadType) - TridentWorkloadTypeShort: $(WorkloadTypeShort) + TridentWorkloadType: ai-ml-score-int + TridentWorkloadTypeShort: aimlscore DeployLocation: eastus - Agent: agce-ai azureSubscription: AG-AzureCAT-AIDevOps-Test-COGSNonProd-IO1685734(0ca618d2-22a8-413a-96d0-0f1b531129c3) azure_subscription: 0ca618d2-22a8-413a-96d0-0f1b531129c3 + TestPostfix: "" + DeploymentName: MLScoreDeployJob + ProjectLocation: "." + PythonPath: "." + Agent: "Hosted Ubuntu 1604" -jobs: -- job: MLHyperparameterTuningJob - timeoutInMinutes: 300 - cancelTimeoutInMinutes: 2 - pool: - vmImage: 'Ubuntu-16.04' - - steps: - - bash: | - source /usr/share/miniconda/etc/profile.d/conda.sh - which conda - conda env create -f environment.yml - conda env list - conda activate MLHyperparameterTuning - conda env list - echo Login Azure Account - az login -t $(sptenent) --service-principal -u $(spidentity) --password $(spsecret) - echo Try and figure out what account set takes - az account set -h - echo Try and set it. - az account set --subscription $(subscriptionid) -# papermill 01_Data_Prep.ipynb 01_Data_Prep_Output.ipynb --log-output --no-progress-bar -k python3 - displayName: 'Configuration' - - - bash: | - source /usr/share/miniconda/etc/profile.d/conda.sh - conda activate MLHyperparameterTuning - echo Executing 00_Data_Prep.ipynb - papermill 00_Data_Prep.ipynb 00_Data_Prep_Output.ipynb --log-output --no-progress-bar -k python3 - displayName: '00_Data_Prep.ipynb' - - - bash: | - source /usr/share/miniconda/etc/profile.d/conda.sh - conda activate MLHyperparameterTuning - echo Executing 01_Training_Script.ipynb - papermill 01_Training_Script.ipynb 01_Training_Script_Output.ipynb --log-output --no-progress-bar -k python3 - displayName: '01_Training_Script.ipynb' - - - bash: | - source /usr/share/miniconda/etc/profile.d/conda.sh - conda activate MLHyperparameterTuning - echo Executing 02_Testing_Script.ipynb - papermill 02_Testing_Script.ipynb 02_Testing_Script_Output.ipynb --log-output --no-progress-bar -k python3 - displayName: '02_Testing_Script.ipynb' - - - bash: | - source /usr/share/miniconda/etc/profile.d/conda.sh - conda activate MLHyperparameterTuning - echo Executing 03_Run_Locally.ipynb - papermill 03_Run_Locally.ipynb 03_Run_Locally_Output.ipynb --log-output --no-progress-bar -k python3 -p selected_subscription $(subscriptionid) -p resource_group $(azurergname) - displayName: '03_Run_Locally.ipynb' - - - bash: | - source /usr/share/miniconda/etc/profile.d/conda.sh - conda activate MLHyperparameterTuning - echo Executing 04_Hyperparameter_Random_Search.ipynb - papermill 04_Hyperparameter_Random_Search.ipynb 04_Hyperparameter_Random_Search_Output.ipynb --log-output --no-progress-bar -k python3 -p max_total_runs $(dsmaxruns) - displayName: '04_Hyperparameter_Random_Search.ipynb' - - - bash: | - source /usr/share/miniconda/etc/profile.d/conda.sh - conda activate MLHyperparameterTuning - echo Executing 05_Train_Best_Model.ipynb - papermill 05_Train_Best_Model.ipynb 05_Train_Best_Model_Output.ipynb --log-output --no-progress-bar -k python3 - displayName: '05_Train_Best_Model.ipynb' - - - bash: | - source /usr/share/miniconda/etc/profile.d/conda.sh - conda activate MLHyperparameterTuning - echo Executing 06_Test_Best_Model.ipynb - papermill 06_Test_Best_Model.ipynb 06_Test_Best_Model_Output.ipynb --log-output --no-progress-bar -k python3 - displayName: '06_Test_Best_Model.ipynb' - - - bash: | - source /usr/share/miniconda/etc/profile.d/conda.sh - conda activate MLHyperparameterTuning - echo Executing 07_Train_With_AML_Pipeline.ipynb - papermill 07_Train_With_AML_Pipeline.ipynb 07_Train_With_AML_Pipeline_Output.ipynb --log-output --no-progress-bar -k python3 -p max_total_runs $(dsmaxruns) - displayName: '07_Train_With_AML_Pipeline.ipynb' - - - bash: | - source /usr/share/miniconda/etc/profile.d/conda.sh - conda activate MLHyperparameterTuning - echo Executing 08_Tear_Down.ipynb - papermill 08_Tear_Down.ipynb 08_Tear_Down_Output.ipynb --log-output --no-progress-bar -k python3 - displayName: '08_Tear_Down.ipynb' - - - bash: | - source /usr/share/miniconda/etc/profile.d/conda.sh - conda activate MLHyperparameterTuning - echo Execute Resource Group Delete - existResponse=$(az group exists -n $(azurergname)) - if [ "$existResponse" == "true" ]; then - echo Deleting project resource group - az group delete --name $(azurergname) --yes - else - echo Project resource group did not exist - fi - echo Done Cleanup - displayName: 'Backup Cleanup' - condition: or(canceled(),failed()) - - - task: CreateWorkItem@1 - inputs: - workItemType: 'Issue' - title: $(System.TeamProject) - Build $(Build.BuildNumber) Failed - assignedTo: 'Mario Bourgoin ' - associate: true - teamProject: $(System.TeamProject) - - fieldMappings: | - Description=Branch: Branch $(Build.SourceBranch) failed to build. Go to Boards>WorkItems and tag the failure type. - displayName: 'Create work item on failure' - condition: failed() +# In additional to the above variables, the "template" parameter of the last step must also be hard coded for each workload + +trigger: + branches: + include: + - mabou/instrument + +stages: +- stage: stable + dependsOn: [] + jobs: + - job: build_deploy_ai + displayName: 'Build deploy AI' + + timeoutInMinutes: 180 + + workspace: + clean: all + + variables: + DeploymentGuidTag: $[ dependencies.build_relayer_sources.outputs['GenDeployGuidTag.DeploymentGuid'] ] + DeploymentTimeStamp: $[ dependencies.build_relayer_sources.outputs['GenDeployTimeStamp.timeStamp'] ] + EnvironmentPrefix: $(TridentWorkloadType)-$(CloudPlatform)-$(DeployLocation)$(TestPostfix) + ResourcePrefix: $(TridentWorkloadTypeShort)-$(DeployLocation)$(TestPostfix) + EnvironmentContext: $(EnvironmentPrefix)-$(DeploymentGuidTag) + AIResourceGroupName: $(TridentWorkloadTypeShort)-$(DeployLocation)$(TestPostfix) + + steps: + + - template: .ci/steps/docker_clean.yml@aitemplates + + - template: .ci/steps/deploy_notebook_steps.yml@aitemplates + parameters: + deployment_name: $(DeploymentName) + template: MLTrainDeployAMLJob.yml + azureSubscription: $(azureSubscription) + azure_subscription: $(azure_subscription) + azureresourcegroup: $(AIResourceGroupName) + workspacename: $(TridentWorkloadTypeShort)-$(DeployLocation) + azureregion: $(DeployLocation) + aksimagename: myimage + environment: $(EnvironmentContext) + doCleanup: False + alias: $(Build.QueuedBy) + project: $(TridentWorkloadTypeShort) + agent: $(Agent) + ENVIRONMENT_PREFIX: $(EnvironmentPrefix) + deploymentguidtag: $(DeploymentGuidTag) + aks_name: $(TridentWorkloadTypeShort)$(Deploy_Location_Short) + python_path: $(System.DefaultWorkingDirectory)$(PythonPath) + location: $(ProjectLocation) + python_secret_root: "./" + +- stage: flight_release + dependsOn: [] + jobs: + - job: build_deploy_ai + displayName: 'Build deploy AI' + + timeoutInMinutes: 180 + + workspace: + clean: all + + variables: + DeploymentGuidTag: $[ dependencies.build_relayer_sources.outputs['GenDeployGuidTag.DeploymentGuid'] ] + DeploymentTimeStamp: $[ dependencies.build_relayer_sources.outputs['GenDeployTimeStamp.timeStamp'] ] + TestPostfix: "-release" + EnvironmentPrefix: $(TridentWorkloadType)-$(CloudPlatform)-$(DeployLocation)$(TestPostfix) + ResourcePrefix: $(TridentWorkloadTypeShort)-$(DeployLocation)$(TestPostfix) + EnvironmentContext: $(EnvironmentPrefix)-$(DeploymentGuidTag) + AIResourceGroupName: $(TridentWorkloadTypeShort)-$(DeployLocation)$(TestPostfix) + + steps: + + - template: .ci/steps/docker_clean.yml@aitemplates + + - template: .ci/steps/deploy_notebook_steps.yml@aitemplates + parameters: + deployment_name: $(DeploymentName) + template: MLTrainDeployAMLJob.yml + azureSubscription: $(azureSubscription) + azure_subscription: $(azure_subscription) + azureresourcegroup: $(AIResourceGroupName) + workspacename: $(TridentWorkloadTypeShort)-$(DeployLocation) + azureregion: $(DeployLocation) + aksimagename: myimage + environment: $(EnvironmentContext) + doCleanup: False + alias: $(Build.QueuedBy) + project: $(TridentWorkloadTypeShort) + agent: $(Agent) + ENVIRONMENT_PREFIX: $(EnvironmentPrefix) + deploymentguidtag: $(DeploymentGuidTag) + aks_name: $(TridentWorkloadTypeShort)$(Deploy_Location_Short) + python_path: $(System.DefaultWorkingDirectory)$(PythonPath) + location: $(ProjectLocation) + python_secret_root: "./" + flighting_release: true + +- stage: flight_preview + dependsOn: [] + jobs: + - job: build_deploy_ai + displayName: 'Build deploy AI' + + timeoutInMinutes: 180 + + workspace: + clean: all + + variables: + DeploymentGuidTag: $[ dependencies.build_relayer_sources.outputs['GenDeployGuidTag.DeploymentGuid'] ] + DeploymentTimeStamp: $[ dependencies.build_relayer_sources.outputs['GenDeployTimeStamp.timeStamp'] ] + TestPostfix: "-preview" + EnvironmentPrefix: $(TridentWorkloadType)-$(CloudPlatform)-$(DeployLocation)$(TestPostfix) + ResourcePrefix: $(TridentWorkloadTypeShort)-$(DeployLocation)$(TestPostfix) + EnvironmentContext: $(EnvironmentPrefix)-$(DeploymentGuidTag) + AIResourceGroupName: $(TridentWorkloadTypeShort)-$(DeployLocation)$(TestPostfix) + + steps: + + - template: .ci/steps/docker_clean.yml@aitemplates + + - template: .ci/steps/deploy_notebook_steps.yml@aitemplates + parameters: + deployment_name: $(DeploymentName) + template: MLTrainDeployAMLJob.yml + azureSubscription: $(azureSubscription) + azure_subscription: $(azure_subscription) + azureresourcegroup: $(AIResourceGroupName) + workspacename: $(TridentWorkloadTypeShort)-$(DeployLocation) + azureregion: $(DeployLocation) + aksimagename: myimage + environment: $(EnvironmentContext) + doCleanup: False + alias: $(Build.QueuedBy) + project: $(TridentWorkloadTypeShort) + agent: $(Agent) + ENVIRONMENT_PREFIX: $(EnvironmentPrefix) + deploymentguidtag: $(DeploymentGuidTag) + aks_name: $(TridentWorkloadTypeShort)$(Deploy_Location_Short) + python_path: $(System.DefaultWorkingDirectory)$(PythonPath) + location: $(ProjectLocation) + python_secret_root: "./" + flighting_preview: true From e0d7be0520fe0fe5482a8b5416b9d5ab4660ffad Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 6 Dec 2019 18:35:16 -0500 Subject: [PATCH 05/10] Create agce_devops_sub_vars.yml --- .ci/vars/agce_devops_sub_vars.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .ci/vars/agce_devops_sub_vars.yml diff --git a/.ci/vars/agce_devops_sub_vars.yml b/.ci/vars/agce_devops_sub_vars.yml new file mode 100644 index 0000000..02070e3 --- /dev/null +++ b/.ci/vars/agce_devops_sub_vars.yml @@ -0,0 +1,3 @@ +variables: + azure_subscription: 0ca618d2-22a8-413a-96d0-0f1b531129c3 + azureSubscription: AG-AzureCAT-AIDevOps-Test-COGSNonProd-IO1685734(0ca618d2-22a8-413a-96d0-0f1b531129c3) From 8a36e026a3e8830d4e5ab814cbc17ec330b162b7 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 6 Dec 2019 18:35:34 -0500 Subject: [PATCH 06/10] Create mlhyperparametertuning_vars.yml --- .ci/vars/mlhyperparametertuning_vars.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .ci/vars/mlhyperparametertuning_vars.yml diff --git a/.ci/vars/mlhyperparametertuning_vars.yml b/.ci/vars/mlhyperparametertuning_vars.yml new file mode 100644 index 0000000..7d0d820 --- /dev/null +++ b/.ci/vars/mlhyperparametertuning_vars.yml @@ -0,0 +1,7 @@ +variables: + DeploymentName: MLScoreDeployJob + TridentWorkloadTypeShort: aimlscore + DeployLocation: eastus + ProjectLocation: "." + PythonPath: "." + Template: MLTrainDeployAMLJob.yml From 8098844a993eecb6e9b32ee92f1729b41611e665 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 6 Dec 2019 18:51:10 -0500 Subject: [PATCH 07/10] Update azure-pipelines-v2.yml --- .ci/azure-pipelines-v2.yml | 177 +++++-------------------------------- 1 file changed, 22 insertions(+), 155 deletions(-) diff --git a/.ci/azure-pipelines-v2.yml b/.ci/azure-pipelines-v2.yml index 1ddd0f1..e9500a4 100644 --- a/.ci/azure-pipelines-v2.yml +++ b/.ci/azure-pipelines-v2.yml @@ -1,163 +1,30 @@ -# MLHyperparameterTuning Pipeline +# MLHyperparameterTuning Pipeline +# +# A Github Service Connection must also be created with the name "AIArchitecturesAndPractices-GitHub" -trigger: none - -variables: - CloudPlatform: AzureCloud - TridentWorkloadType: ai-ml-score-int - TridentWorkloadTypeShort: aimlscore - DeployLocation: eastus - azureSubscription: AG-AzureCAT-AIDevOps-Test-COGSNonProd-IO1685734(0ca618d2-22a8-413a-96d0-0f1b531129c3) - azure_subscription: 0ca618d2-22a8-413a-96d0-0f1b531129c3 - TestPostfix: "" - DeploymentName: MLScoreDeployJob - ProjectLocation: "." - PythonPath: "." - Agent: "Hosted Ubuntu 1604" - -# In additional to the above variables, the "template" parameter of the last step must also be hard coded for each workload +resources: + repositories: + - repository: aitemplates + type: github + name: microsoft/AI + endpoint: AIArchitecturesAndPractices-GitHub trigger: branches: include: + - master - mabou/instrument -stages: -- stage: stable - dependsOn: [] - jobs: - - job: build_deploy_ai - displayName: 'Build deploy AI' - - timeoutInMinutes: 180 - - workspace: - clean: all - - variables: - DeploymentGuidTag: $[ dependencies.build_relayer_sources.outputs['GenDeployGuidTag.DeploymentGuid'] ] - DeploymentTimeStamp: $[ dependencies.build_relayer_sources.outputs['GenDeployTimeStamp.timeStamp'] ] - EnvironmentPrefix: $(TridentWorkloadType)-$(CloudPlatform)-$(DeployLocation)$(TestPostfix) - ResourcePrefix: $(TridentWorkloadTypeShort)-$(DeployLocation)$(TestPostfix) - EnvironmentContext: $(EnvironmentPrefix)-$(DeploymentGuidTag) - AIResourceGroupName: $(TridentWorkloadTypeShort)-$(DeployLocation)$(TestPostfix) - - steps: - - - template: .ci/steps/docker_clean.yml@aitemplates - - - template: .ci/steps/deploy_notebook_steps.yml@aitemplates - parameters: - deployment_name: $(DeploymentName) - template: MLTrainDeployAMLJob.yml - azureSubscription: $(azureSubscription) - azure_subscription: $(azure_subscription) - azureresourcegroup: $(AIResourceGroupName) - workspacename: $(TridentWorkloadTypeShort)-$(DeployLocation) - azureregion: $(DeployLocation) - aksimagename: myimage - environment: $(EnvironmentContext) - doCleanup: False - alias: $(Build.QueuedBy) - project: $(TridentWorkloadTypeShort) - agent: $(Agent) - ENVIRONMENT_PREFIX: $(EnvironmentPrefix) - deploymentguidtag: $(DeploymentGuidTag) - aks_name: $(TridentWorkloadTypeShort)$(Deploy_Location_Short) - python_path: $(System.DefaultWorkingDirectory)$(PythonPath) - location: $(ProjectLocation) - python_secret_root: "./" - -- stage: flight_release - dependsOn: [] - jobs: - - job: build_deploy_ai - displayName: 'Build deploy AI' - - timeoutInMinutes: 180 - - workspace: - clean: all - - variables: - DeploymentGuidTag: $[ dependencies.build_relayer_sources.outputs['GenDeployGuidTag.DeploymentGuid'] ] - DeploymentTimeStamp: $[ dependencies.build_relayer_sources.outputs['GenDeployTimeStamp.timeStamp'] ] - TestPostfix: "-release" - EnvironmentPrefix: $(TridentWorkloadType)-$(CloudPlatform)-$(DeployLocation)$(TestPostfix) - ResourcePrefix: $(TridentWorkloadTypeShort)-$(DeployLocation)$(TestPostfix) - EnvironmentContext: $(EnvironmentPrefix)-$(DeploymentGuidTag) - AIResourceGroupName: $(TridentWorkloadTypeShort)-$(DeployLocation)$(TestPostfix) - - steps: - - - template: .ci/steps/docker_clean.yml@aitemplates - - - template: .ci/steps/deploy_notebook_steps.yml@aitemplates - parameters: - deployment_name: $(DeploymentName) - template: MLTrainDeployAMLJob.yml - azureSubscription: $(azureSubscription) - azure_subscription: $(azure_subscription) - azureresourcegroup: $(AIResourceGroupName) - workspacename: $(TridentWorkloadTypeShort)-$(DeployLocation) - azureregion: $(DeployLocation) - aksimagename: myimage - environment: $(EnvironmentContext) - doCleanup: False - alias: $(Build.QueuedBy) - project: $(TridentWorkloadTypeShort) - agent: $(Agent) - ENVIRONMENT_PREFIX: $(EnvironmentPrefix) - deploymentguidtag: $(DeploymentGuidTag) - aks_name: $(TridentWorkloadTypeShort)$(Deploy_Location_Short) - python_path: $(System.DefaultWorkingDirectory)$(PythonPath) - location: $(ProjectLocation) - python_secret_root: "./" - flighting_release: true - -- stage: flight_preview - dependsOn: [] - jobs: - - job: build_deploy_ai - displayName: 'Build deploy AI' - - timeoutInMinutes: 180 - - workspace: - clean: all - - variables: - DeploymentGuidTag: $[ dependencies.build_relayer_sources.outputs['GenDeployGuidTag.DeploymentGuid'] ] - DeploymentTimeStamp: $[ dependencies.build_relayer_sources.outputs['GenDeployTimeStamp.timeStamp'] ] - TestPostfix: "-preview" - EnvironmentPrefix: $(TridentWorkloadType)-$(CloudPlatform)-$(DeployLocation)$(TestPostfix) - ResourcePrefix: $(TridentWorkloadTypeShort)-$(DeployLocation)$(TestPostfix) - EnvironmentContext: $(EnvironmentPrefix)-$(DeploymentGuidTag) - AIResourceGroupName: $(TridentWorkloadTypeShort)-$(DeployLocation)$(TestPostfix) - - steps: - - - template: .ci/steps/docker_clean.yml@aitemplates +pr: + autoCancel: true + branches: + include: + - master + - mabou/instrument - - template: .ci/steps/deploy_notebook_steps.yml@aitemplates - parameters: - deployment_name: $(DeploymentName) - template: MLTrainDeployAMLJob.yml - azureSubscription: $(azureSubscription) - azure_subscription: $(azure_subscription) - azureresourcegroup: $(AIResourceGroupName) - workspacename: $(TridentWorkloadTypeShort)-$(DeployLocation) - azureregion: $(DeployLocation) - aksimagename: myimage - environment: $(EnvironmentContext) - doCleanup: False - alias: $(Build.QueuedBy) - project: $(TridentWorkloadTypeShort) - agent: $(Agent) - ENVIRONMENT_PREFIX: $(EnvironmentPrefix) - deploymentguidtag: $(DeploymentGuidTag) - aks_name: $(TridentWorkloadTypeShort)$(Deploy_Location_Short) - python_path: $(System.DefaultWorkingDirectory)$(PythonPath) - location: $(ProjectLocation) - python_secret_root: "./" - flighting_preview: true +stages: +- template: .ci/stages/deploy_notebooks_stages_v2.yml@aitemplates + parameters: + jobDisplayName: MLScoreDeployJob + DefaultWorkingDirectory: $(System.DefaultWorkingDirectory) + workload_vars: ../vars/mlhyperparametertuning_vars.yml From 9e124381bb3d766d1c643ec9036d0a4860e2086d Mon Sep 17 00:00:00 2001 From: Mario Bourgoin Date: Mon, 9 Dec 2019 09:50:16 -0500 Subject: [PATCH 08/10] Rename model_estimators to estimators --- 05_Train_Best_Model.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/05_Train_Best_Model.ipynb b/05_Train_Best_Model.ipynb index 9fc101c..95015c2 100644 --- a/05_Train_Best_Model.ipynb +++ b/05_Train_Best_Model.ipynb @@ -166,8 +166,8 @@ }, "outputs": [], "source": [ - "model_estimators = 8 * int(best_parameters['--estimators'])\n", - "model_estimators" + "estimators = 8 * int(best_parameters['--estimators'])\n", + "estimators" ] }, { @@ -186,7 +186,7 @@ "ds = ws.get_default_datastore()\n", "model_parameters = best_parameters.copy()\n", "model_parameters['--data-folder'] = ds.as_mount()\n", - "model_parameters['--estimators'] = model_estimators\n", + "model_parameters['--estimators'] = estimators\n", "model_parameters['--save'] = 'FAQ_ranker'\n", "pd.Series(model_parameters, name='Value').to_frame()" ] From 0a770f753663e3a47978e2ce7c39847cb4126814 Mon Sep 17 00:00:00 2001 From: Mario Bourgoin Date: Mon, 9 Dec 2019 09:52:43 -0500 Subject: [PATCH 09/10] Add estimators variable --- azure-pipelines.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ee44441..61835f7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,6 +8,7 @@ trigger: variables: - group: AzureKeyVault + estimators: 1 jobs: - job: MLHyperparameterTuningJob @@ -44,7 +45,7 @@ jobs: source /usr/share/miniconda/etc/profile.d/conda.sh conda activate MLHyperparameterTuning echo Executing 01_Training_Script.ipynb - papermill 01_Training_Script.ipynb 01_Training_Script_Output.ipynb --log-output --no-progress-bar -k python3 + papermill 01_Training_Script.ipynb 01_Training_Script_Output.ipynb --log-output --no-progress-bar -k python3 -p estimators $(estimators) displayName: '01_Training_Script.ipynb' - bash: | @@ -58,21 +59,21 @@ jobs: source /usr/share/miniconda/etc/profile.d/conda.sh conda activate MLHyperparameterTuning echo Executing 03_Run_Locally.ipynb - papermill 03_Run_Locally.ipynb 03_Run_Locally_Output.ipynb --log-output --no-progress-bar -k python3 -p subscription_id $(subscriptionid) -p resource_group $(azurergname) + papermill 03_Run_Locally.ipynb 03_Run_Locally_Output.ipynb --log-output --no-progress-bar -k python3 -p subscription_id $(subscriptionid) -p resource_group $(azurergname) -p estimators $(estimators) displayName: '03_Run_Locally.ipynb' - bash: | source /usr/share/miniconda/etc/profile.d/conda.sh conda activate MLHyperparameterTuning echo Executing 04_Hyperparameter_Random_Search.ipynb - papermill 04_Hyperparameter_Random_Search.ipynb 04_Hyperparameter_Random_Search_Output.ipynb --log-output --no-progress-bar -k python3 -p max_total_runs $(dsmaxruns) + papermill 04_Hyperparameter_Random_Search.ipynb 04_Hyperparameter_Random_Search_Output.ipynb --log-output --no-progress-bar -k python3 -p max_total_runs $(dsmaxruns) -p estimators $(estimators) displayName: '04_Hyperparameter_Random_Search.ipynb' - bash: | source /usr/share/miniconda/etc/profile.d/conda.sh conda activate MLHyperparameterTuning echo Executing 05_Train_Best_Model.ipynb - papermill 05_Train_Best_Model.ipynb 05_Train_Best_Model_Output.ipynb --log-output --no-progress-bar -k python3 + papermill 05_Train_Best_Model.ipynb 05_Train_Best_Model_Output.ipynb --log-output --no-progress-bar -k python3 -p estimators $(estimators) displayName: '05_Train_Best_Model.ipynb' - bash: | From 193d090f0eeee81f32442bde307df6dfbcefd2c9 Mon Sep 17 00:00:00 2001 From: Mario Bourgoin Date: Mon, 9 Dec 2019 10:10:35 -0500 Subject: [PATCH 10/10] Replace estimators variable with a 1 for now --- azure-pipelines.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 61835f7..028d2eb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,7 +8,7 @@ trigger: variables: - group: AzureKeyVault - estimators: 1 + # estimators: 1 jobs: - job: MLHyperparameterTuningJob @@ -45,7 +45,7 @@ jobs: source /usr/share/miniconda/etc/profile.d/conda.sh conda activate MLHyperparameterTuning echo Executing 01_Training_Script.ipynb - papermill 01_Training_Script.ipynb 01_Training_Script_Output.ipynb --log-output --no-progress-bar -k python3 -p estimators $(estimators) + papermill 01_Training_Script.ipynb 01_Training_Script_Output.ipynb --log-output --no-progress-bar -k python3 -p estimators 1 displayName: '01_Training_Script.ipynb' - bash: | @@ -59,21 +59,21 @@ jobs: source /usr/share/miniconda/etc/profile.d/conda.sh conda activate MLHyperparameterTuning echo Executing 03_Run_Locally.ipynb - papermill 03_Run_Locally.ipynb 03_Run_Locally_Output.ipynb --log-output --no-progress-bar -k python3 -p subscription_id $(subscriptionid) -p resource_group $(azurergname) -p estimators $(estimators) + papermill 03_Run_Locally.ipynb 03_Run_Locally_Output.ipynb --log-output --no-progress-bar -k python3 -p subscription_id $(subscriptionid) -p resource_group $(azurergname) -p estimators 1 displayName: '03_Run_Locally.ipynb' - bash: | source /usr/share/miniconda/etc/profile.d/conda.sh conda activate MLHyperparameterTuning echo Executing 04_Hyperparameter_Random_Search.ipynb - papermill 04_Hyperparameter_Random_Search.ipynb 04_Hyperparameter_Random_Search_Output.ipynb --log-output --no-progress-bar -k python3 -p max_total_runs $(dsmaxruns) -p estimators $(estimators) + papermill 04_Hyperparameter_Random_Search.ipynb 04_Hyperparameter_Random_Search_Output.ipynb --log-output --no-progress-bar -k python3 -p max_total_runs $(dsmaxruns) -p estimators 1 displayName: '04_Hyperparameter_Random_Search.ipynb' - bash: | source /usr/share/miniconda/etc/profile.d/conda.sh conda activate MLHyperparameterTuning echo Executing 05_Train_Best_Model.ipynb - papermill 05_Train_Best_Model.ipynb 05_Train_Best_Model_Output.ipynb --log-output --no-progress-bar -k python3 -p estimators $(estimators) + papermill 05_Train_Best_Model.ipynb 05_Train_Best_Model_Output.ipynb --log-output --no-progress-bar -k python3 -p estimators 1 displayName: '05_Train_Best_Model.ipynb' - bash: |