diff --git a/.jenkins/perf_test b/.jenkins/perf_test new file mode 100644 index 0000000000..187444f6ba --- /dev/null +++ b/.jenkins/perf_test @@ -0,0 +1,798 @@ +//def shared_library_branch = scm.branches[0].name +//if (shared_library_branch .contains("*/")) { +// shared_library_branch = shared_library_branch.split("\\*/")[1] +// } +//def util_lib="jenkins-shared@${shared_library_branch}" +//echo "${util_lib}" + +//library "${util_lib}" + +pipeline { + agent { node { label 'austin' } } + environment { + docker_args="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -v=/var/jenkins/:/var/jenkins" + extra_log_env = " MIOPEN_LOG_LEVEL=5 " + Fp16_flags = " -DMIOPEN_TEST_HALF=Off" + Bf16_flags = " -DMIOPEN_TEST_BFLOAT16=Off" + Int8_flags = " -DMIOPEN_TEST_INT8=Off" + Full_test = " -DMIOPEN_TEST_ALL=Off" + Smoke_targets = " check MIOpenDriver" + NOCOMGR_flags = " -DMIOPEN_USE_COMGR=Off" + NOMLIR_flags = " -DMIOPEN_USE_MLIR=Off" + } + parameters { + booleanParam(name: 'COMPARE_TO_BASE', defaultValue : true, description: 'Compare test results to base results') + booleanParam(name: 'PERF_TEST_FP32', defaultValue : true, description: 'Run FP16 tests') + booleanParam(name: 'PERF_TEST_FP16', defaultValue : true, description: 'Run FP16 tests') + booleanParam(name: 'PERF_TEST_ARCHIVE', defaultValue : true, description: 'Archive results from this run') + string(name: 'PERF_TEST_OVERRIDE', defaultValue: '', description: 'Add extra env vars for the MIOpenDriver cmd, comma separated') + } + stages { + stage('Build Docker'){ + steps{ + script { + echo "Build Docker Stage" + //def utils = load "vars/utils.groovy" + //utils.getDockerImage() + def utils = load "vars/utils.groovy" + docker_image = utils.buildHipClangJob(needs_gpu:false, needs_reboot:false, needs_cleanup:true, build_install: true, make_targets: "install") + withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) { + docker_image.push() + } + } + } + } + /*stage("Packages") { + parallel { + stage("HIP Package") { + agent{ node { label 'nogpu'} } + steps{ + script { + echo "Packages Stage" + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot( package_build: "true", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false) + } + } + } + } + }*/ + //stage("Performance Tests - gfx90a") { + //parallel{ + stage('Fp32 BS128 Hip Performance Resnet50_v1.5 gfx90A'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP32_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install", docker: docker_image) + } + } + }/* + stage('Fp32 BS256 Hip Performance Resnet50_v1.5 gfx90A'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP32_BS256.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install") + } + } + } + stage('Fp32 BS512 Hip Performance Resnet50_v1.5 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP32_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS128 Hip Performance Resnet50_v1.5 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP16_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS256 Hip Performance Resnet50_v1.5 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP16_BS256.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS512 Hip Performance Resnet50_v1.5 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP16_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS128 Hip Performance Alexnet_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP16_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS512 Hip Performance Alexnet_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP16_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS4 Hip Performance Alexnet_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP32_BS4.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS64 Hip Performance Alexnet_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP32_BS64.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS128 Hip Performance Alexnet_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP32_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS512 Hip Performance Alexnet_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP32_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS256 Hip Performance Densenet201_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Densenet201_v1_FP16_BS256.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS256 Hip Performance Densenet201_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Densenet201_v1_FP32_BS256.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS256 Hip Performance Densenet_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Densenet_v1_FP16_BS256.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS256 Hip Performance Densenet_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Densenet_v1_FP32_BS256.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS128 Hip Performance Googlenet_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Googlenet_v1_FP16_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS512 Hip Performance Googlenet_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Googlenet_v1_FP16_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS128 Hip Performance Googlenet_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Googlenet_v1_FP32_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS512 Hip Performance Googlenet_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Googlenet_v1_FP32_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS128 Hip Performance Inception3_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Inception3_v1_FP16_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS128 Hip Performance Inception3_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Inception3_v1_FP32_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS512 Hip Performance Inception3_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Inception3_v1_FP32_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS128 Hip Performance Inception4_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Inception4_v1_FP16_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS512 Hip Performance Inception4_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Inception4_v1_FP16_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS128 Hip Performance Inception4_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Inception4_v1_FP32_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS512 Hip Performance Inception4_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Inception4_v1_FP32_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS4 Hip Performance Mobilenet_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Mobilenet_v1_FP32_BS4.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS64 Hip Performance Mobilenet_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Mobilenet_v1_FP32_BS64.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS32 Hip Performance Resnet101_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP16_BS32.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS128 Hip Performance Resnet101_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP16_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS256 Hip Performance Resnet101_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP16_BS256.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS512 Hip Performance Resnet101_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP16_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS128 Hip Performance Resnet101_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP32_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS256 Hip Performance Resnet101_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP32_BS256.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS512 Hip Performance Resnet101_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP32_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS256 Hip Performance Resnet152_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v1_FP16_BS256.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS256 Hip Performance Resnet152_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v1_FP32_BS256.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS128 Hip Performance Resnet152_v2 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v2_FP16_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS512 Hip Performance Resnet152_v2 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v2_FP16_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS128 Hip Performance Resnet152_v2 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v2_FP32_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS512 Hip Performance Resnet152_v2 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v2_FP32_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS32 Hip Performance Resnet50_v1 gfx90a'){ + + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS32.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS64 Hip Performance Resnet50_v1 gfx90a'){ + + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS64.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS128 Hip Performance Resnet50_v1 gfx90a'){ + + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS256 Hip Performance Resnet50_v1 gfx90a'){ + + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS256.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 B512 Hip Performance Resnet50_v1 gfx90a'){ + + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS128 Hip Performance Resnet50_v1 gfx90a'){ + + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP32_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS256 Hip Performance Resnet50_v1 gfx90a'){ + + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP32_BS256.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS512 Hip Performance Resnet50_v1 gfx90a'){ + + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP32_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS128 Hip Performance Shufflenet_v2 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "Shufflenet_v2_FP16_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS128 Hip Performance SSD_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "SSD_v1_FP16_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS128 Hip Performance SSD_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "SSD_v1_FP32_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS128 Hip Performance VGG11_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "VGG11_v1_FP16_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS256 Hip Performance VGG11_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "VGG11_v1_FP16_BS256.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS512 Hip Performance VGG11_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "VGG11_v1_FP16_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS512 Hip Performance VGG11_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "VGG11_v1_FP32_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS128 Hip Performance VGG16_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP16_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS4 Hip Performance VGG16_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP32_BS4.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS64 Hip Performance VGG16_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP32_BS64.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS128 Hip Performance VGG16_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP32_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS512 Hip Performance VGG16_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP32_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS128 Hip Performance VGG19_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "VGG19_v1_FP16_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp16 BS512 Hip Performance VGG19_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP16} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "VGG19_v1_FP16_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS128 Hip Performance VGG19_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "VGG19_v1_FP32_BS128.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + } + stage('Fp32 BS512 Hip Performance VGG19_v1 gfx90a'){ + when { + expression {params.PERF_TEST_FP32} + } + steps{ + script { + def utils = load "vars/utils.groovy" + utils.RunPerfTest(gpu_arch: "gfx90a", filename: "VGG19_v1_FP32_BS512.txt", needs_gpu:true, needs_reboot:false, build_install: "true", needs_cleanup:false, make_targets: "install" ) + } + } + }*/ + //} + //} + } +} diff --git a/Jenkinsfile b/Jenkinsfile index 5551844867..2b60db4879 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,391 +2,17 @@ def rocmnode(name) { return '(rocmtest || miopen) && (' + name + ')' } -def miopenCheckout() -{ - checkout([ - $class: 'GitSCM', - branches: scm.branches, - doGenerateSubmoduleConfigurations: true, - extensions: scm.extensions + [[$class: 'SubmoduleOption', parentCredentials: true]], - userRemoteConfigs: scm.userRemoteConfigs - ]) -} - -def show_node_info() { - sh """ - echo "NODE_NAME = \$NODE_NAME" - lsb_release -sd - uname -r - cat /sys/module/amdgpu/version - ls /opt/ -la - """ -} - -//default -// CXX=/opt/rocm/llvm/bin/clang++ CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=Off -DCMAKE_PREFIX_PATH=/usr/local -DBUILD_DEV=On -DCMAKE_BUILD_TYPE=release .. -// -def cmake_build(Map conf=[:]){ - - def compiler = conf.get("compiler","/opt/rocm/llvm/bin/clang++") - def make_targets = conf.get("make_targets","check") - def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined -Wno-option-ignored " + conf.get("extradebugflags", "") - def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","") - def prefixpath = conf.get("prefixpath","/opt/rocm") - def build_type_debug = (conf.get("build_type",'release') == 'debug') - - def mlir_args = " -DMIOPEN_USE_MLIR=" + conf.get("mlir_build", "ON") - // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors. - if (build_type_debug) - { - mlir_args = " -DMIOPEN_USE_MLIR=OFF" - } - - def setup_args = mlir_args + " -DMIOPEN_GPU_SYNC=Off " + conf.get("setup_flags","") - def build_fin = conf.get("build_fin", "OFF") - - setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} " - - //cmake_env can overwrite default CXX variables. - def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","") - - def package_build = (conf.get("package_build","") == "true") - - if (package_build == true) { - make_targets = "miopen_gtest package miopen_gtest_check" - setup_args = " -DMIOPEN_TEST_DISCRETE=OFF " + setup_args - } - - def miopen_install_path = "${env.WORKSPACE}/install" - if(conf.get("build_install","") == "true") - { - make_targets = 'install ' + make_targets - setup_args = " -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=${miopen_install_path}" + setup_args - } else if(package_build == true) { - setup_args = ' -DBUILD_DEV=Off' + setup_args - } else { - setup_args = ' -DBUILD_DEV=On' + setup_args - } - - // test_flags = ctest -> MIopen flags - def test_flags = conf.get("test_flags","") +//def shared_library_branch = scm.branches[0].name +//if (shared_library_branch .contains("*/")) { +// shared_library_branch = shared_library_branch.split("\\*/")[1] +// } +//def util_lib="jenkins-shared@${shared_library_branch}" +//echo "${util_lib}" - if (conf.get("vcache_enable","") == "true"){ - def vcache = conf.get(vcache_path,"/var/jenkins/.cache/miopen/vcache") - build_envs = " MIOPEN_VERIFY_CACHE_PATH='${vcache}' " + build_envs - } else{ - test_flags = " --disable-verification-cache " + test_flags - } - - if(build_type_debug){ - setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args - }else{ - setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args - } - - if(test_flags != ""){ - setup_args = "-DMIOPEN_TEST_FLAGS='${test_flags}'" + setup_args - } - - if(conf.containsKey("find_mode")) - { - def fmode = conf.get("find_mode", "") - setup_args = " -DMIOPEN_DEFAULT_FIND_MODE=${fmode} " + setup_args - } - if(env.CCACHE_HOST) - { - setup_args = " -DCMAKE_CXX_COMPILER_LAUNCHER='ccache' -DCMAKE_C_COMPILER_LAUNCHER='ccache' " + setup_args - } +//library "${util_lib}" - if ( build_fin == "ON" ) - { - setup_args = " -DMIOPEN_INSTALL_CXX_HEADERS=On " + setup_args - } - - def pre_setup_cmd = """ - echo \$HSA_ENABLE_SDMA - ulimit -c unlimited - rm -rf build - mkdir build - rm -rf install - mkdir install - rm -f src/kernels/*.ufdb.txt - rm -f src/kernels/miopen*.udb - cd build - """ - def setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake ${setup_args} .. ") - // WORKAROUND_SWDEV_290754 - // It seems like this W/A is not required since 4.5. - def build_cmd = conf.get("build_cmd", "LLVM_PATH=/opt/rocm/llvm ${build_envs} dumb-init make -j\$(nproc) ${make_targets}") - def execute_cmd = conf.get("execute_cmd", "") - - def cmd = conf.get("cmd", """ - ${pre_setup_cmd} - ${setup_cmd} - ${build_cmd} - """) - - if ( build_fin == "ON" ) - { - def fin_build_cmd = cmake_fin_build_cmd(miopen_install_path) - cmd += """ - export RETDIR=\$PWD - cd ${env.WORKSPACE}/fin - ${fin_build_cmd} - cd \$RETDIR - """ - } - - cmd += """ - ${execute_cmd} - """ - - echo cmd - sh cmd - - // Only archive from master or develop - if (package_build == true && (env.BRANCH_NAME == "develop" || env.BRANCH_NAME == "master" || - env.BRANCH_NAME == env.MIOPEN_GOLDEN_PERF_BRANCH || params.PERF_TEST_BRANCH_OVERRIDE)) { - archiveArtifacts artifacts: "build/*.deb", allowEmptyArchive: true, fingerprint: true - archiveArtifacts artifacts: "build/*.rpm", allowEmptyArchive: true, fingerprint: true - stash includes: "build/*tar.gz", name: 'miopen_tar' - } -} - -def cmake_fin_build_cmd(prefixpath){ - def flags = "-DCMAKE_INSTALL_PREFIX=${prefixpath} -DCMAKE_BUILD_TYPE=release" - def compiler = 'clang++' - def make_targets = "install" - def compilerpath = "/opt/rocm/llvm/bin/" + compiler - def configargs = "" - if (prefixpath != "") - { - configargs = "-DCMAKE_PREFIX_PATH=${prefixpath}" - } - - def fin_cmd = """ - echo \$HSA_ENABLE_SDMA - ulimit -c unlimited - rm -rf build - mkdir build - cd build - CXX=${compilerpath} cmake ${configargs} ${flags} .. - dumb-init make -j\$(nproc) ${make_targets} - """ - return fin_cmd -} - -def getDockerImageName(dockerArgs) -{ - sh "echo ${dockerArgs} > factors.txt" - def image = "${env.MIOPEN_DOCKER_IMAGE_URL}" - sh "md5sum Dockerfile requirements.txt dev-requirements.txt >> factors.txt" - def docker_hash = sh(script: "md5sum factors.txt | awk '{print \$1}' | head -c 6", returnStdout: true) - sh "rm factors.txt" - echo "Docker tag hash: ${docker_hash}" - image = "${image}:ci_${docker_hash}" - if(params.DOCKER_IMAGE_OVERRIDE != '') - { - echo "Overriding the base docker image with ${params.DOCKER_IMAGE_OVERRIDE}" - image = "${params.DOCKER_IMAGE_OVERRIDE}" - } - return image - -} - -def getDockerImage(Map conf=[:]) -{ - env.DOCKER_BUILDKIT=1 - def prefixpath = conf.get("prefixpath", "/opt/rocm") // one image for each prefix 1: /usr/local 2:/opt/rocm - def gpu_arch = "gfx908;gfx90a;gfx942;gfx1100" // prebuilt dockers should have all the architectures enabled so one image can be used for all stages - def mlir_build = conf.get("mlir_build", "ON") // always ON - def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg GPU_ARCHS='\"${gpu_arch}\"' --build-arg USE_MLIR='${mlir_build}' " - if(env.CCACHE_HOST) - { - def check_host = sh(script:"""(printf "PING\r\n";) | nc -N ${env.CCACHE_HOST} 6379 """, returnStdout: true).trim() - if(check_host == "+PONG") - { - echo "FOUND CCACHE SERVER: ${CCACHE_HOST}" - } - else - { - echo "CCACHE SERVER: ${CCACHE_HOST} NOT FOUND, got ${check_host} response" - } - dockerArgs = dockerArgs + " --build-arg CCACHE_SECONDARY_STORAGE='redis://${env.CCACHE_HOST}' --build-arg COMPILER_LAUNCHER='ccache' " - env.CCACHE_DIR = """/tmp/ccache_store""" - env.CCACHE_SECONDARY_STORAGE="""redis://${env.CCACHE_HOST}""" - } - echo "Docker Args: ${dockerArgs}" - - def image = getDockerImageName(dockerArgs) - - def dockerImage - try{ - echo "Pulling down image: ${image}" - dockerImage = docker.image("${image}") - dockerImage.pull() - } - catch(org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){ - echo "The job was cancelled or aborted" - throw e - } - catch(Exception ex) - { - dockerImage = docker.build("${image}", "${dockerArgs} .") - withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) { - dockerImage.push() - } - } - return [dockerImage, image] -} - -def buildHipClangJob(Map conf=[:]){ - show_node_info() - miopenCheckout() - env.HSA_ENABLE_SDMA=0 - env.DOCKER_BUILDKIT=1 - def image - def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined" - if (conf.get("enforce_xnack_on", false)) { - dockerOpts = dockerOpts + " --env HSA_XNACK=1" - } - - def variant = env.STAGE_NAME - - def needs_gpu = conf.get("needs_gpu", true) - def lfs_pull = conf.get("lfs_pull", false) - - def retimage - gitStatusWrapper(credentialsId: "${env.miopen_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'MIOpen') { - try { - (retimage, image) = getDockerImage(conf) - if (needs_gpu) { - withDockerContainer(image: image, args: dockerOpts) { - timeout(time: 5, unit: 'MINUTES') - { - sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo' - } - } - } - } - catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){ - echo "The job was cancelled or aborted" - throw e - } - catch(Exception ex) { - (retimage, image) = getDockerImage(conf) - if (needs_gpu) { - withDockerContainer(image: image, args: dockerOpts) { - timeout(time: 5, unit: 'MINUTES') - { - sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo' - } - } - } - } - - withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') { - timeout(time: 420, unit:'MINUTES') - { - if (lfs_pull) { - sh "git lfs pull --exclude=" - } - - cmake_build(conf) - } - } - } - return retimage -} - -def reboot(){ - build job: 'reboot-slaves', propagate: false , parameters: [string(name: 'server', value: "${env.NODE_NAME}"),] -} - -def buildHipClangJobAndReboot(Map conf=[:]){ - try{ - buildHipClangJob(conf) - cleanWs() - } - catch(e){ - echo "throwing error exception for the stage" - echo 'Exception occurred: ' + e.toString() - throw e - } - finally{ - if (conf.get("needs_reboot", true)) { - reboot() - } - } -} - -def RunPerfTest(Map conf=[:]){ - def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined" - try { - (retimage, image) = getDockerImage(conf) - withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') { - timeout(time: 600, unit: 'MINUTES') - { - unstash 'miopen_tar' - sh "tar -zxvf build/miopen-hip-*-Linux-runtime.tar.gz" - ld_lib="${env.WORKSPACE}/opt/rocm/lib" - def filename = conf.get("filename", "") - if (env.BRANCH_NAME == env.MIOPEN_GOLDEN_PERF_BRANCH || params.PERF_TEST_BRANCH_OVERRIDE){ - if(params.PERF_TEST_OVERRIDE != '') - { - echo "Appending MIOpenDriver cmd env vars: ${params.PERF_TEST_OVERRIDE}" - sh "export LD_LIBRARY_PATH=${ld_lib} && ${env.WORKSPACE}/opt/rocm/bin/test_perf.py --filename ${filename} --install_path ${env.WORKSPACE}/opt/rocm --override ${params.PERF_TEST_OVERRRIDE}" - }else - { - sh "export LD_LIBRARY_PATH=${ld_lib} && ${env.WORKSPACE}/opt/rocm/bin/test_perf.py --filename ${filename} --install_path ${env.WORKSPACE}/opt/rocm" - } - sh "export LD_LIBRARY_PATH=${ld_lib} && ${env.WORKSPACE}/opt/rocm/bin/test_perf.py --filename ${filename} --install_path ${env.WORKSPACE}/opt/rocm" - jenkins_url = "${env.artifact_path}/${env.MIOPEN_GOLDEN_PERF_BRANCH}/lastSuccessfulBuild/artifact" - try { - sh "rm -rf ${env.WORKSPACE}/opt/rocm/bin/old_results/" - sh "wget -P ${env.WORKSPACE}/opt/rocm/bin/old_results/ ${jenkins_url}/opt/rocm/bin/perf_results/${filename}" - } - catch (Exception err){ - currentBuild.result = 'SUCCESS' - } - } - - archiveArtifacts artifacts: "opt/rocm/bin/perf_results/${filename}", allowEmptyArchive: true, fingerprint: true - try{ - if (env.BRANCH_NAME != env.MIOPEN_GOLDEN_PERF_BRANCH){ - sh "${env.WORKSPACE}/opt/rocm/bin/test_perf.py --compare_results --old_results_path ${env.WORKSPACE}/opt/rocm/bin/old_results --filename ${filename}" - } - } - catch (Exception err){ - currentBuild.result = 'SUCCESS' - } - cleanWs() - } - } - } - catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){ - echo "The job was cancelled or aborted" - throw e - } -} - - -def CheckPerfDbValid(Map conf=[:]){ - def pdb_image = buildHipClangJob(conf) - pdb_image.inside(){ - dir(path: "$WORKSPACE"){ - sh "ls install/bin/" - sh "MIOPEN_LOG_LEVEL=4 LD_LIBRARY_PATH='install/lib:/opt/rocm/lib/' install/bin/fin -i fin/tests/pdb_check_all.json -o pdb_valid_err.json" - archiveArtifacts "pdb_valid_err.json" - sh "grep clear pdb_valid_err.json" - def has_error = sh ( - script: "echo \$?", - returnStdout: true - ).trim() - assert has_error.toInteger() == 0 - } - } -} +//library "jenkins-shared@$branch_name" +//def utils = load "vars/utils.groovy" /// Stage name format: /// [DataType] Backend[/Compiler] BuildType [TestSet] [Target] @@ -497,29 +123,10 @@ pipeline { name: "DATATYPE_INT8", defaultValue: true, description: "") - booleanParam( - name: "PERF_TEST", - defaultValue: false, - description: "Enable performance testing stages") - booleanParam( - name: "PERF_TEST_FP16", - defaultValue: false, - description: "Enable performance testing stages") - booleanParam( - name: "PERF_TEST_FP32", - defaultValue: false, - description: "Enable performance testing stages") - booleanParam( - name: "PERF_TEST_BRANCH_OVERRIDE", - defaultValue: false, - description: "Enable performance testing stages") booleanParam( name: "DBSYNC_TEST", defaultValue: true, description: "Enable database synchronization testing stages") - string(name: "PERF_TEST_OVERRIDE", - defaultValue: '', - description: "Add extra env vars for the MIOpenDriver cmd, comma separated") string(name: "DOCKER_IMAGE_OVERRIDE", defaultValue: '', description: "") @@ -550,7 +157,10 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - getDockerImage() + script { + def utils = load "vars/utils.groovy" + utils.getDockerImage() + } } } stage("Packages") { @@ -561,7 +171,10 @@ pipeline { stage("HIP Package") { agent{ label rocmnode("nogpu") } steps{ - buildHipClangJobAndReboot( package_build: "true", needs_gpu:false, needs_reboot:false) + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot( package_build: "true", needs_gpu:false, needs_reboot:false, needs_cleanup:true) + } } } } @@ -578,7 +191,10 @@ pipeline { build_cmd = "make -j\$(nproc) -k analyze" } steps{ - buildHipClangJobAndReboot(setup_cmd: setup_cmd, build_cmd: build_cmd, needs_gpu:false, needs_reboot:false) + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot(setup_cmd: setup_cmd, build_cmd: build_cmd, needs_gpu:false, needs_reboot:false, needs_cleanup:true) + } } } stage('Clang Format') { @@ -595,7 +211,10 @@ pipeline { | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-12 -style=file {} | diff - {}\'" } steps{ - buildHipClangJobAndReboot(setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, needs_gpu:false, needs_reboot:false) + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot(setup_cmd: "", build_cmd: "", execute_cmd: execute_cmd, needs_gpu:false, needs_reboot:false, needs_cleanup:true) + } } } stage('HipNoGPU Debug Build Test') { @@ -609,7 +228,10 @@ pipeline { build_cmd = "make -j\$(nproc)" } steps{ - buildHipClangJob( build_type: 'debug', setup_flags: HipNoGPU_flags, build_cmd: build_cmd, needs_gpu:false, needs_reboot:false) + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJob( build_type: 'debug', setup_flags: HipNoGPU_flags, build_cmd: build_cmd, needs_gpu:false, needs_reboot:false, needs_cleanup:true) + } } } stage('Tuna Fin Build Test') { @@ -618,7 +240,10 @@ pipeline { fin_flags = "-DMIOPEN_BACKEND=HIPNOGPU" } steps{ - buildHipClangJobAndReboot(setup_flags: fin_flags, make_targets: "all", build_fin: "ON", needs_gpu:false, needs_reboot:false, build_install: "true") + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot(setup_flags: fin_flags, make_targets: "all", build_fin: "ON", needs_gpu:false, needs_reboot:false, build_install: "true", needs_cleanup:true) + } } } } @@ -638,7 +263,10 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(make_targets: Smoke_targets, build_install: "true") + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot(make_targets: Smoke_targets, build_install: "true", needs_cleanup:true) + } } } stage('Fp32 Hip Debug gfx90a') { @@ -651,7 +279,10 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, build_install: "true") + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, build_install: "true", needs_cleanup:true) + } } } stage('Fp32 Hip Debug gfx908') { @@ -664,7 +295,10 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, build_install: "true") + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, build_install: "true", needs_cleanup:true) + } } } stage('Fp32 Hip Debug gfx94X') { @@ -677,7 +311,10 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, needs_reboot:false, build_install: "true") + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, needs_reboot:false, build_install: "true", needs_cleanup:true) + } } } } @@ -701,7 +338,10 @@ pipeline { NOCOMGR_build_cmd = "CTEST_PARALLEL_LEVEL=4 MIOPEN_LOG_LEVEL=5 make -j\$(nproc) check" } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOCOMGR_flags, build_cmd: NOCOMGR_build_cmd, test_flags: ' --verbose ', build_install: "true") + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOCOMGR_flags, build_cmd: NOCOMGR_build_cmd, test_flags: ' --verbose ', build_install: "true", needs_cleanup:true) + } } } stage('Fp32 Hip Debug NOMLIR gfx90a') { @@ -718,7 +358,10 @@ pipeline { NOMLIR_build_cmd = "CTEST_PARALLEL_LEVEL=4 MIOPEN_LOG_LEVEL=5 make -j\$(nproc) check" } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOMLIR_flags, build_cmd: NOMLIR_build_cmd, test_flags: ' --verbose ', build_install: "true") + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOMLIR_flags, build_cmd: NOMLIR_build_cmd, test_flags: ' --verbose ', build_install: "true", needs_cleanup:true) + } } } stage('Fp32 Hip Debug NOCK gfx90a Build-Only') { @@ -731,7 +374,10 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: "-DMIOPEN_USE_COMPOSABLEKERNEL=Off", make_targets: "", build_install: "true") + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot( build_type: 'debug', setup_flags: "-DMIOPEN_USE_COMPOSABLEKERNEL=Off", make_targets: "", build_install: "true", needs_cleanup:true) + } } } stage('Fp32 Hip Debug Embedded Vega20') { @@ -747,7 +393,10 @@ pipeline { Embedded_flags = "-DMIOPEN_EMBED_DB='gfx906_60'" } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: Embedded_flags, build_env: extra_log_env, test_flags: ' --verbose ', build_install: "true") + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot( build_type: 'debug', setup_flags: Embedded_flags, build_env: extra_log_env, test_flags: ' --verbose ', build_install: "true", needs_cleanup:true) + } } } stage('Fp32 Hip Static gfx90a') { @@ -760,7 +409,10 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot( setup_flags: "-DBUILD_SHARED_LIBS=Off", mlir_build: 'OFF', build_install: "true") + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot( setup_flags: "-DBUILD_SHARED_LIBS=Off", mlir_build: 'OFF', build_install: "true", needs_cleanup:true) + } } } stage('Fp32 Hip Normal-Find gfx90a') { @@ -777,7 +429,10 @@ pipeline { execute_cmd = "bin/test_conv2d --disable-verification-cache" } steps{ - buildHipClangJobAndReboot(make_targets: make_targets, execute_cmd: execute_cmd, find_mode: "Normal", build_install: "true") + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot(make_targets: make_targets, execute_cmd: execute_cmd, find_mode: "Normal", build_install: "true", needs_cleanup:true) + } } } stage('Fp32 Hip Fast-Find gfx90a') { @@ -794,7 +449,10 @@ pipeline { execute_cmd = "MIOPEN_FIND_MODE=2 CTEST_PARALLEL_LEVEL=4 bin/test_conv2d --disable-verification-cache" } steps{ - buildHipClangJobAndReboot( make_targets: make_targets, execute_cmd: execute_cmd, build_install: "true") + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot( make_targets: make_targets, execute_cmd: execute_cmd, build_install: "true", needs_cleanup:true) + } } } stage('Fp32 Hip gfx90a') { @@ -807,7 +465,10 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot() + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot(needs_cleanup:true) + } } } stage('Fp32 Hip SqlitePerfdb gfx90a') { @@ -820,7 +481,10 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(make_targets: Smoke_targets, setup_flags: "-DMIOPEN_USE_SQLITE_PERF_DB=On", build_install: "true") + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot(make_targets: Smoke_targets, setup_flags: "-DMIOPEN_USE_SQLITE_PERF_DB=On", build_install: "true", needs_cleanup:true) + } } } } @@ -840,7 +504,10 @@ pipeline { } agent{ label rocmnode("vega20") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true") + script { + def utils = load "vars/utils.groovy" + utils.buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true", needs_cleanup:true) + } } } stage('Bf16 Hip Vega20') { @@ -853,7 +520,9 @@ pipeline { } agent{ label rocmnode("vega20") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true") + script { + utils.buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true", needs_cleanup:true) + } } } stage('Fp16 Hip gfx908') { @@ -866,7 +535,9 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true") + script { + utils.buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true", needs_cleanup:true) + } } } stage('Bf16 Hip gfx908') { @@ -879,7 +550,9 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true") + script { + utils.buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true", needs_cleanup:true) + } } } stage('Fp16 Hip gfx90a') { @@ -892,7 +565,9 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true") + script { + utils.buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true", needs_cleanup:true) + } } } stage('Bf16 Hip gfx90a') { @@ -905,7 +580,9 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true") + script { + utils.buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true", needs_cleanup:true) + } } } stage('Fp16 Hip gfx94X') { @@ -918,7 +595,9 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, needs_reboot:false, build_install: "true") + script { + utils.buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, needs_reboot:false, build_install: "true", needs_cleanup:true) + } } } stage('Bf16 Hip gfx94X') { @@ -931,7 +610,9 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, needs_reboot:false, build_install: "true") + script { + utils.buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, needs_reboot:false, build_install: "true", needs_cleanup:true) + } } } } @@ -956,13 +637,16 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot(lfs_pull: true, + script { + utils.buildHipClangJobAndReboot(lfs_pull: true, setup_flags: "-DMIOPEN_TEST_DBSYNC=1", make_targets: 'test_db_sync', execute_cmd: './bin/test_db_sync', needs_gpu:false, needs_reboot:false, - build_install: "true") + build_install: "true", + needs_cleanup:true) + } } } stage('Dbsync gfx90a') { @@ -975,13 +659,16 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(lfs_pull: true, + script { + utils.buildHipClangJobAndReboot(lfs_pull: true, setup_flags: "-DMIOPEN_TEST_DBSYNC=1", make_targets: 'test_db_sync', execute_cmd: './bin/test_db_sync', needs_gpu:false, needs_reboot:false, - build_install: "true") + build_install: "true", + needs_cleanup:true) + } } } stage('Dbsync gfx942') { @@ -994,13 +681,16 @@ pipeline { } agent{ label rocmnode("gfx942") } steps{ - buildHipClangJobAndReboot(lfs_pull: true, + script { + utils.buildHipClangJobAndReboot(lfs_pull: true, setup_flags: "-DMIOPEN_TEST_DBSYNC=1", make_targets: 'test_db_sync', execute_cmd: './bin/test_db_sync', needs_gpu:false, needs_reboot:false, - build_install: "true") + build_install: "true", + needs_cleanup:true) + } } } stage('Int8 HIP All Vega20') { @@ -1013,7 +703,9 @@ pipeline { } agent{ label rocmnode("vega20") } steps{ - buildHipClangJobAndReboot( setup_flags: Int8_flags + Full_test) + script { + utils.buildHipClangJobAndReboot( setup_flags: Int8_flags + Full_test, needs_cleanup:true) + } } } stage('Bf16 Hip Install All gfx908') { @@ -1026,7 +718,9 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags + Full_test, build_install: "true") + script { + utils.buildHipClangJobAndReboot(setup_flags: Bf16_flags + Full_test, build_install: "true", needs_cleanup:true) + } } } stage('Bf16 Hip Install All gfx90a') { @@ -1039,7 +733,9 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags + Full_test, build_install: "true") + script { + utils.buildHipClangJobAndReboot(setup_flags: Bf16_flags + Full_test, build_install: "true", needs_cleanup:true) + } } } stage('Bf16 Hip Install All gfx94X') { @@ -1052,7 +748,9 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags + Full_test, build_install: "true", needs_reboot:false) + script { + utils.buildHipClangJobAndReboot(setup_flags: Bf16_flags + Full_test, build_install: "true", needs_reboot:false, needs_cleanup:true) + } } } stage('Fp16 Hip All gfx1030') { @@ -1065,7 +763,9 @@ pipeline { } agent{ label rocmnode("navi21") } steps{ - buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags, build_cmd: Navi21_build_cmd) + script { + utils.buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags, build_cmd: Navi21_build_cmd, needs_cleanup:true) + } } } stage('Fp16 Hip All gfx1101') { @@ -1078,7 +778,9 @@ pipeline { } agent{ label rocmnode("navi32") } steps{ - buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags) + script { + utils.buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags, needs_cleanup:true) + } } } stage('Fp32 Hip All gfx908') { @@ -1091,7 +793,9 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot(setup_flags: Full_test) + script { + utils.buildHipClangJobAndReboot(setup_flags: Full_test, needs_cleanup:true) + } } } stage('Fp32 Hip All gfx90a') { @@ -1104,7 +808,9 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(setup_flags: Full_test) + script { + utils.buildHipClangJobAndReboot(setup_flags: Full_test, needs_cleanup:true) + } } } // stage('Fp32 Hip All gfx90a Xnack+') { @@ -1114,7 +820,9 @@ pipeline { // } // agent{ label rocmnode("gfx90a") } // steps{ - // buildHipClangJobAndReboot(setup_flags: Full_test, enforce_xnack_on: true) + // script { + // utils.buildHipClangJobAndReboot(setup_flags: Full_test, enforce_xnack_on: true, needs_cleanup:true) + // } // } // } stage('Fp32 Hip All gfx94X') { @@ -1127,7 +835,9 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot(setup_flags: Full_test, needs_reboot:false) + script { + utils.buildHipClangJobAndReboot(setup_flags: Full_test, needs_reboot:false, needs_cleanup:true) + } } } stage('Fp16 Hip Install All Vega20') { @@ -1140,7 +850,9 @@ pipeline { } agent{ label rocmnode("vega20") } steps{ - buildHipClangJobAndReboot( setup_flags: Full_test + Fp16_flags, build_install: "true") + script { + utils.buildHipClangJobAndReboot( setup_flags: Full_test + Fp16_flags, build_install: "true", needs_cleanup:true) + } } } stage('Fp32 Hip All Vega20') { @@ -1153,7 +865,9 @@ pipeline { } agent{ label rocmnode("vega20") } steps{ - buildHipClangJobAndReboot( setup_flags: Full_test) + script { + utils.buildHipClangJobAndReboot( setup_flags: Full_test) + } } } stage('Fp32 Hip All Install gfx1030') { @@ -1166,7 +880,9 @@ pipeline { } agent{ label rocmnode("navi21") } steps{ - buildHipClangJobAndReboot(setup_flags: Full_test, build_cmd: Navi21_build_cmd, build_install: "true") + script { + utils.buildHipClangJobAndReboot(setup_flags: Full_test, build_cmd: Navi21_build_cmd, build_install: "true", needs_cleanup:true) + } } } stage('Fp32 Hip All Install gfx1101') { @@ -1179,7 +895,9 @@ pipeline { } agent{ label rocmnode("navi32") } steps{ - buildHipClangJobAndReboot(setup_flags: Full_test, build_install: "true") + script { + utils.buildHipClangJobAndReboot(setup_flags: Full_test, build_install: "true", needs_cleanup:true) + } } } stage('Fp16 Hip All Install gfx908') { @@ -1192,7 +910,9 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags, build_install: "true") + script { + utils.buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags, build_install: "true", needs_cleanup:true) + } } } stage('Fp16 Hip All Install gfx90a') { @@ -1205,7 +925,9 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags, build_install: "true") + script { + utils.buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags, build_install: "true", needs_cleanup:true) + } } } stage('Fp16 Hip All Install gfx94X') { @@ -1218,616 +940,9 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags, build_install: "true", needs_reboot:false) - } - } - } - } - stage("Performance Tests - gfx90a") { - when { - expression {params.PERF_TEST && params.TARGET_GFX90A} - } - parallel{ - stage('Fp32 BS128 Hip Performance Resnet50_v1.5 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP32_BS128.txt" ) - } - } - stage('Fp32 BS256 Hip Performance Resnet50_v1.5 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP32_BS256.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Resnet50_v1.5 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP32_BS512.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Resnet50_v1.5 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP16_BS128.txt" ) - } - } - stage('Fp16 BS256 Hip Performance Resnet50_v1.5 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP16_BS256.txt" ) - } - } - stage('Fp16 BS512 Hip Performance Resnet50_v1.5 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1.5_FP16_BS512.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Alexnet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP16_BS128.txt" ) - } - } - stage('Fp16 BS512 Hip Performance Alexnet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP16_BS512.txt" ) - } - } - stage('Fp32 BS4 Hip Performance Alexnet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP32_BS4.txt" ) - } - } - stage('Fp32 BS64 Hip Performance Alexnet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP32_BS64.txt" ) - } - } - stage('Fp32 BS128 Hip Performance Alexnet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Alexnet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Alexnet_v1_FP32_BS512.txt" ) - } - } - stage('Fp16 BS256 Hip Performance Densenet201_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Densenet201_v1_FP16_BS256.txt" ) - } - } - stage('Fp32 BS256 Hip Performance Densenet201_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Densenet201_v1_FP32_BS256.txt" ) - } - } - stage('Fp16 BS256 Hip Performance Densenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Densenet_v1_FP16_BS256.txt" ) - } - } - stage('Fp32 BS256 Hip Performance Densenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Densenet_v1_FP32_BS256.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Googlenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Googlenet_v1_FP16_BS128.txt" ) - } - } - stage('Fp16 BS512 Hip Performance Googlenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Googlenet_v1_FP16_BS512.txt" ) - } - } - stage('Fp32 BS128 Hip Performance Googlenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Googlenet_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Googlenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Googlenet_v1_FP32_BS512.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Inception3_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Inception3_v1_FP16_BS128.txt" ) - } - } - stage('Fp32 BS128 Hip Performance Inception3_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Inception3_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Inception3_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Inception3_v1_FP32_BS512.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Inception4_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Inception4_v1_FP16_BS128.txt" ) - } - } - stage('Fp16 BS512 Hip Performance Inception4_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Inception4_v1_FP16_BS512.txt" ) - } - } - stage('Fp32 BS128 Hip Performance Inception4_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Inception4_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Inception4_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Inception4_v1_FP32_BS512.txt" ) - } - } - stage('Fp32 BS4 Hip Performance Mobilenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Mobilenet_v1_FP32_BS4.txt" ) - } - } - stage('Fp32 BS64 Hip Performance Mobilenet_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Mobilenet_v1_FP32_BS64.txt" ) - } - } - stage('Fp16 BS32 Hip Performance Resnet101_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP16_BS32.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Resnet101_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP16_BS128.txt" ) - } - } - stage('Fp16 BS256 Hip Performance Resnet101_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP16_BS256.txt" ) - } - } - stage('Fp16 BS512 Hip Performance Resnet101_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP16_BS512.txt" ) - } - } - stage('Fp32 BS128 Hip Performance Resnet101_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS256 Hip Performance Resnet101_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP32_BS256.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Resnet101_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet101_v1_FP32_BS512.txt" ) - } - } - stage('Fp16 BS256 Hip Performance Resnet152_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v1_FP16_BS256.txt" ) - } - } - stage('Fp32 BS256 Hip Performance Resnet152_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v1_FP32_BS256.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Resnet152_v2 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v2_FP16_BS128.txt" ) - } - } - stage('Fp16 BS512 Hip Performance Resnet152_v2 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v2_FP16_BS512.txt" ) - } - } - stage('Fp32 BS128 Hip Performance Resnet152_v2 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v2_FP32_BS128.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Resnet152_v2 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet152_v2_FP32_BS512.txt" ) - } - } - stage('Fp16 BS32 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS32.txt" ) - } - } - stage('Fp16 BS64 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS64.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS128.txt" ) - } - } - stage('Fp16 BS256 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS256.txt" ) - } - } - stage('Fp16 B512 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP16_BS512.txt" ) - } - } - stage('Fp32 BS128 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS256 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP32_BS256.txt" ) - } - } - stage('Fp32 BS512 Hip Performance Resnet50_v1 gfx90a'){ - - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Resnet50_v1_FP32_BS512.txt" ) - } - } - stage('Fp16 BS128 Hip Performance Shufflenet_v2 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "Shufflenet_v2_FP16_BS128.txt" ) - } - } - stage('Fp16 BS128 Hip Performance SSD_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "SSD_v1_FP16_BS128.txt" ) - } - } - stage('Fp32 BS128 Hip Performance SSD_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "SSD_v1_FP32_BS128.txt" ) - } - } - stage('Fp16 BS128 Hip Performance VGG11_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG11_v1_FP16_BS128.txt" ) - } - } - stage('Fp16 BS256 Hip Performance VGG11_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG11_v1_FP16_BS256.txt" ) - } - } - stage('Fp16 BS512 Hip Performance VGG11_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG11_v1_FP16_BS512.txt" ) - } - } - stage('Fp32 BS512 Hip Performance VGG11_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG11_v1_FP32_BS512.txt" ) - } - } - stage('Fp16 BS128 Hip Performance VGG16_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP16_BS128.txt" ) - } - } - stage('Fp32 BS4 Hip Performance VGG16_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP32_BS4.txt" ) - } - } - stage('Fp32 BS64 Hip Performance VGG16_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP32_BS64.txt" ) - } - } - stage('Fp32 BS128 Hip Performance VGG16_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS512 Hip Performance VGG16_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG16_v1_FP32_BS512.txt" ) - } - } - stage('Fp16 BS128 Hip Performance VGG19_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG19_v1_FP16_BS128.txt" ) - } - } - stage('Fp16 BS512 Hip Performance VGG19_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP16} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG19_v1_FP16_BS512.txt" ) - } - } - stage('Fp32 BS128 Hip Performance VGG19_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG19_v1_FP32_BS128.txt" ) - } - } - stage('Fp32 BS512 Hip Performance VGG19_v1 gfx90a'){ - when { - expression {params.PERF_TEST_FP32} - } - agent{ label rocmnode("austin")} - steps{ - RunPerfTest(gpu_arch: "gfx90a", filename: "VGG19_v1_FP32_BS512.txt" ) + script { + utils.buildHipClangJobAndReboot(setup_flags: Full_test + Fp16_flags, build_install: "true", needs_reboot:false, needs_cleanup:true) + } } } } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 796078b2b0..a4f15becf5 100755 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -28,11 +28,9 @@ cmake_policy(SET CMP0057 NEW) set(test_perf.py ${CMAKE_INSTALL_BINDIR}/test_perf.py) -if( NOT ENABLE_ASAN_PACKAGING ) - install(FILES test_perf.py - PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE - DESTINATION ${CMAKE_INSTALL_BINDIR}) -endif() +install(FILES test_perf.py + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + DESTINATION ${CMAKE_INSTALL_BINDIR}) set(MODEL_FILES) file (GLOB model_src CONFIGURE_DEPENDS perf_models/*.txt) @@ -40,15 +38,13 @@ foreach (src ${model_src}) list (APPEND MODEL_FILES ${src}) endforeach() -if( NOT ENABLE_ASAN_PACKAGING ) - install(FILES - ${MODEL_FILES} - DESTINATION ${DATA_INSTALL_DIR}/perf_models) -endif() - foreach(MODEL_FILE ${MODEL_FILES}) - get_filename_component(MODEL_FILE_FILENAME "${MODEL_FILE}" NAME) - configure_file("${MODEL_FILE}" "${PROJECT_BINARY_DIR}/${DATA_INSTALL_DIR}/perf_models/${MODEL_FILE_FILENAME}" COPYONLY) - endforeach() +install(FILES + ${MODEL_FILES} + DESTINATION ${DATA_INSTALL_DIR}/perf_models) +foreach(MODEL_FILE ${MODEL_FILES}) + get_filename_component(MODEL_FILE_FILENAME "${MODEL_FILE}" NAME) + configure_file("${MODEL_FILE}" "${PROJECT_BINARY_DIR}/${DATA_INSTALL_DIR}/perf_models/${MODEL_FILE_FILENAME}" COPYONLY) +endforeach() include(CheckCXXCompilerFlag) diff --git a/test/test_perf.py b/test/test_perf.py index 94038faeea..e266c38d5a 100755 --- a/test/test_perf.py +++ b/test/test_perf.py @@ -31,9 +31,12 @@ import subprocess import argparse import csv +import time from decimal import Decimal +import multiprocessing as mp -results_path = f"{os.path.dirname(__file__)}/perf_results" +curr_path = os.path.abspath(os.path.dirname(__file__)) +results_path = curr_path + "/perf_results" TOLERANCE = -5 #tolerance 5% re_Elapsed = re.compile(r"(\d*\.*\d+)") @@ -41,8 +44,186 @@ re_GPU = re.compile(r"^GPU Kernel Time .* Elapsed: (\d+\.\d+) ms") re_Key = re.compile(r"^.*Key match: ([\w\-]*)") +queue = mp.Queue() #gpu idx queue, used in HIP_VISIBLE_DEVICES +results_queue = mp.Queue() + + +class Manager(mp.Process): + """Queue manager""" + + def __init__(self, **kwargs): + allowed_keys = set(['filename', 'install_path', 'overrride']) + self.filename = None + self.install_path = None + self.override = False + self.results_path = f"{os.path.dirname(__file__)}/perf_results" + self.__dict__.update( + (key, value) for key, value in kwargs.items() if key in allowed_keys) + + self.in_qs = {} + self.num_gpus = int(self.get_num_gpus()) + self.resfile = f"{self.results_path}/{self.filename}" + print(self.resfile) + print('install_path: %s', self.install_path) + self.model_path = f"{self.install_path}/share/miopen/perf_models/{self.filename}" + self.driver_cmds = [] + self.set_driver_cmds() + self.writer = None + + def get_num_gpus(self): + """Get num_gpus""" + cmd = "/opt/rocm/bin/rocminfo | grep ${arch}:sramecc+:xnack | wc -l" + proc = subprocess.Popen(cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = proc.communicate()[0] + return output.decode('utf-8').strip() + + def set_driver_cmds(self): + if self.override: + var_list = self.override.split(',') + var_str = "" + for var in var_list: + var_str += var + " &&" + + with open(os.path.expanduser(self.model_path), "r", + encoding='utf-8') as infile: + for line in infile: + try: + if line.find('MIOpenDriver') == -1: + print(f"Skipping line '{line}'") + continue + idx = line.index('MIOpenDriver') + driver_cmd = line[idx:-1] + if self.override: + cmd = f"export LD_LIBRARY_PATH={self.install_path}/lib && export MIOPEN_LOG_LEVEL=6 && "\ + f"export MIOPEN_SYSTEM_DB_PATH={self.install_path}/share/miopen/db && "\ + f"export HIP_VISIBLE_DEVICES=GPU_ID && "\ + f"{var_str} "\ + f"{self.install_path}/bin/{driver_cmd} -V 0 -i 10 -w 1 -t 1 -G 1" + else: + cmd = f"export LD_LIBRARY_PATH={self.install_path}/lib && export MIOPEN_LOG_LEVEL=6 && "\ + f"export MIOPEN_SYSTEM_DB_PATH={self.install_path}/share/miopen/db && "\ + f"export HIP_VISIBLE_DEVICES=GPU_ID && "\ + f"{self.install_path}/bin/{driver_cmd} -V 0 -i 10 -w 1 -t 1 -G 1" + print(f'Appending cm: {cmd}') + self.driver_cmds.append(cmd) + + except Exception as err: + print(f"Could not get driver commands: {err}") + + print('#driver commands: %s', len(self.driver_cmds)) + + def run(self): + """Main function to launch worker pool""" + for gpu_id in range(self.num_gpus): + queue.put(gpu_id) + self.launch_pool() + + def write_to_file(self, results): + """Write results to csv file""" + field_names = [ + 'Driver', 'k_time', 'wall_time', 'solver_id', 'solver_name', 'fdb_key' + ] + with open(os.path.expanduser(self.resfile), 'w+', encoding='utf-8') as outfile: + self.writer = csv.DictWriter(outfile, fieldnames=field_names) + self.writer.writeheader() + try: + self.writer.writerows(results) + print(f"Perf results written to: {self.resfile}") + except Exception as exp: + print(exp) + + def launch_pool(self): + """Launch pool of driver cmds""" + pool = mp.Pool(processes=self.num_gpus) + for result in pool.imap_unordered(self.run_driver_cmd, self.driver_cmds): + self.parse_result(result) + pool.close() + pool.join() + print(f"Size of results Q: {results_queue.qsize()}") + results = [] + while not results_queue.empty(): + results.append(results_queue.get()) + self.write_to_file(results) + + def parse_result(self, result): + """Potential to use result as it becomes available""" + print(result) + + def run_driver_cmd(self, driver_cmd): + """Launch each driver cmd in subproc""" + while queue.empty(): + time.sleep(2) + print('GPUs busy, sleeping') + gpu_id = queue.get() + cmd = driver_cmd.replace('GPU_ID', str(gpu_id)) + try: + print(f"Starting process on GPU {gpu_id}") + print(cmd) + + proc = subprocess.Popen(cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + p_out = proc.stdout.readlines() + res = None + e = Entry() + for line in p_out: + line = line.decode("utf-8") + line = line.strip() + print(line) + if (line.find('MIOpenDriver') != -1) and (line.find('MIOpen(HIP)') + == -1): #fragile solution + e.cmd = line + print(e.cmd) + continue + if line.find('Wall-clock Time') != -1: + res = re_Elapsed.findall(line) + print("") + print(res) + e.wall_elapsed = res[0] + e.wall_aux = res[1] + e.wall_gwss = res[2] + continue + if re_Solver.match(line): + res = re_Solver.findall(line)[0] + print(res) + e.algo = res[0] + e.sol_id = res[1] + e.sol_name = res[2] + continue + if re_Key.match(line): + e.fdb_key = re_Key.findall(line)[0] + if re_GPU.match(line): + res = re_GPU.findall(line) + print(res) + e.sol_time = res[0] + print() + print('k_time: %s', e.sol_time) + if line.find('error') != -1: + raise ValueError(p_out) + + res_dict = { + 'Driver': e.cmd, + 'k_time': e.sol_time, + 'wall_time': e.wall_elapsed, + 'solver_id': e.sol_id, + 'solver_name': e.sol_name, + 'fdb_key': e.fdb_key + } + results_queue.put(res_dict) + ret = res_dict + print(f"k_time: {e.sol_time}") + print(f"driver: {e.cmd}") + finally: + queue.put(gpu_id) + return ret + class Entry: + """Module to hold runtime values""" def __init__(self): self.cmd = '' @@ -97,100 +278,6 @@ def parse_args(): return args -def run_driver_cmds(filename, install_path, override=None): - """Parse model file and launch Driver cmds""" - resfile = f"{results_path}/{filename}" - model_path = f"{install_path}/share/miopen/perf_models/{filename}" - if override: - var_list = override.split(',') - var_str = "" - for var in var_list: - var_str += var + " &&" - - try: - outfile = open(os.path.expanduser(resfile), 'w+', encoding='utf-8') - results = [] - field_names = [ - 'Driver', 'k_time', 'wall_time', 'solver_id', 'solver_name', 'fdb_key' - ] - writer = csv.DictWriter(outfile, fieldnames=field_names) - writer.writeheader() - with open(os.path.expanduser(model_path), "r", encoding='utf-8') as infile: - for line in infile: - try: - if (line.find('MIOpenDriver') == -1): - print(f"Skipping line '{line}'") - continue - idx = line.index('MIOpenDriver') - driver_cmd = line[idx:-1] - if override: - cmd = f"export LD_LIBRARY_PATH={install_path}/lib && export MIOPEN_LOG_LEVEL=6 && "\ - f"export MIOPEN_SYSTEM_DB_PATH={install_path}/share/miopen/db && "\ - f"{var_str} "\ - f"{install_path}/bin/{driver_cmd} -V 0 -i 10 -w 1 -t 1 -G 1" - else: - cmd = f"export LD_LIBRARY_PATH={install_path}/lib && export MIOPEN_LOG_LEVEL=6 && "\ - f"export MIOPEN_SYSTEM_DB_PATH={install_path}/share/miopen/db && "\ - f"{install_path}/bin/{driver_cmd} -V 0 -i 10 -w 1 -t 1 -G 1" - print(f'Running cm: {cmd}') - proc = subprocess.Popen(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - p_out = proc.stdout.readlines() - k_time = -1 - res = None - e = None - for line in p_out: - line = line.decode("utf-8") - line = line.strip() - print(line) - if (line.find('MIOpenDriver') != -1): - e = Entry() - e.cmd = line - continue - if (line.find('Wall-clock Time') != -1): - res = re_Elapsed.findall(line) - e.wall_elapsed = res[0] - e.wall_aux = res[1] - e.wall_gwss = res[2] - continue - if (re_Solver.match(line)): - res = re_Solver.findall(line)[0] - e.algo = res[0] - e.sol_id = res[1] - e.sol_name = res[2] - continue - if (re_Key.match(line)): - e.fdb_key = re_Key.findall(line)[0] - if (re_GPU.match(line)): - res = re_GPU.findall(line) - e.sol_time = res[0] - print(e) - if line.find('error') != -1: - raise ValueError(p_out) - results.append({ - 'Driver': e.cmd, - 'k_time': e.sol_time, - 'wall_time': e.wall_elapsed, - 'solver_id': e.sol_id, - 'solver_name': e.sol_name, - 'fdb_key': e.fdb_key - }) - print(f'k_time: {e.sol_time}') - - except Exception as ex: - raise ValueError(f"Could not get kernel time: {ex}") - - writer.writerows(results) - print(f"Perf results written to: {resfile}") - outfile.close() - - except Exception as err: - outfile.close() - raise ValueError(f"Could not perform performance measurement: {err}") - - def compare_results(args): """Compare current results with previous results""" if not os.path.exists(results_path): @@ -249,7 +336,11 @@ def main(): os.makedirs(results_path) try: - run_driver_cmds(f"{args.filename}", args.install_path, args.override) + #run_driver_cmds(f"{args.filename}", args.install_path, args.override) + manager = Manager(filename=args.filename, + install_path=args.install_path, + override=args.override) + manager.run() except Exception as ex: print(f'ERR: {ex}') sys.exit(1) diff --git a/vars/utils.groovy b/vars/utils.groovy new file mode 100644 index 0000000000..eef1f28ac4 --- /dev/null +++ b/vars/utils.groovy @@ -0,0 +1,438 @@ +def miopenCheckout() +{ + checkout([ + $class: 'GitSCM', + branches: scm.branches, + doGenerateSubmoduleConfigurations: true, + extensions: scm.extensions + [[$class: 'SubmoduleOption', parentCredentials: true]], + userRemoteConfigs: scm.userRemoteConfigs + ]) +} + +def show_node_info() { + sh """ + echo "NODE_NAME = \$NODE_NAME" + lsb_release -sd + uname -r + cat /sys/module/amdgpu/version + ls /opt/ -la + """ +} + +//default +// CXX=/opt/rocm/llvm/bin/clang++ CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=Off -DCMAKE_PREFIX_PATH=/usr/local -DBUILD_DEV=On -DCMAKE_BUILD_TYPE=release .. +// +def cmake_build(Map conf=[:]){ + + def compiler = conf.get("compiler","/opt/rocm/llvm/bin/clang++") + def make_targets = conf.get("make_targets","check") + def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined -Wno-option-ignored " + conf.get("extradebugflags", "") + def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","") + def prefixpath = conf.get("prefixpath","/opt/rocm") + def build_type_debug = (conf.get("build_type",'release') == 'debug') + + def mlir_args = " -DMIOPEN_USE_MLIR=" + conf.get("mlir_build", "ON") + // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors. + if (build_type_debug) + { + mlir_args = " -DMIOPEN_USE_MLIR=OFF" + } + + def setup_args = mlir_args + " -DMIOPEN_GPU_SYNC=Off " + conf.get("setup_flags","") + def build_fin = conf.get("build_fin", "OFF") + + setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} " + + //cmake_env can overwrite default CXX variables. + def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","") + + def package_build = (conf.get("package_build","") == "true") + + if (package_build == true) { + make_targets = "miopen_gtest package miopen_gtest_check" + setup_args = " -DMIOPEN_TEST_DISCRETE=OFF " + setup_args + } + + def miopen_install_path = "${env.WORKSPACE}/install" + if(conf.get("build_install","") == "true") + { + make_targets = 'install ' + make_targets + setup_args = " -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=${miopen_install_path}" + setup_args + } else if(package_build == true) { + setup_args = ' -DBUILD_DEV=Off' + setup_args + } else { + setup_args = ' -DBUILD_DEV=On' + setup_args + } + + // test_flags = ctest -> MIopen flags + def test_flags = conf.get("test_flags","") + + if (conf.get("vcache_enable","") == "true"){ + def vcache = conf.get(vcache_path,"/var/jenkins/.cache/miopen/vcache") + build_envs = " MIOPEN_VERIFY_CACHE_PATH='${vcache}' " + build_envs + } else{ + test_flags = " --disable-verification-cache " + test_flags + } + + if(build_type_debug){ + setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args + }else{ + setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args + } + + if(test_flags != ""){ + setup_args = "-DMIOPEN_TEST_FLAGS='${test_flags}'" + setup_args + } + + if(conf.containsKey("find_mode")) + { + def fmode = conf.get("find_mode", "") + setup_args = " -DMIOPEN_DEFAULT_FIND_MODE=${fmode} " + setup_args + } + if(env.CCACHE_HOST) + { + setup_args = " -DCMAKE_CXX_COMPILER_LAUNCHER='ccache' -DCMAKE_C_COMPILER_LAUNCHER='ccache' " + setup_args + } + + if ( build_fin == "ON" ) + { + setup_args = " -DMIOPEN_INSTALL_CXX_HEADERS=On " + setup_args + } + + def pre_setup_cmd = """ + echo \$HSA_ENABLE_SDMA + ulimit -c unlimited + rm -rf build + mkdir build + rm -rf install + mkdir install + rm -f src/kernels/*.ufdb.txt + rm -f src/kernels/miopen*.udb + cd build + """ + def setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake ${setup_args} .. ") + // WORKAROUND_SWDEV_290754 + // It seems like this W/A is not required since 4.5. + def build_cmd = conf.get("build_cmd", "LLVM_PATH=/opt/rocm/llvm ${build_envs} dumb-init make -j\$(nproc) ${make_targets}") + def execute_cmd = conf.get("execute_cmd", "") + + def cmd = conf.get("cmd", """ + ${pre_setup_cmd} + ${setup_cmd} + ${build_cmd} + """) + + if ( build_fin == "ON" ) + { + def fin_build_cmd = cmake_fin_build_cmd(miopen_install_path) + cmd += """ + export RETDIR=\$PWD + cd ${env.WORKSPACE}/fin + ${fin_build_cmd} + cd \$RETDIR + """ + } + + cmd += """ + ${execute_cmd} + """ + + echo cmd + sh cmd + + // Only archive from master or develop + if (package_build == true && (env.BRANCH_NAME == "develop" || env.BRANCH_NAME == "master" || + params.PERF_TEST_ARCHIVE == true)) { + archiveArtifacts artifacts: "build/*.deb", allowEmptyArchive: true, fingerprint: true + archiveArtifacts artifacts: "build/*.rpm", allowEmptyArchive: true, fingerprint: true + stash includes: "build/*tar.gz", name: 'miopen_tar' + } +} + +def cmake_fin_build_cmd(prefixpath){ + def flags = "-DCMAKE_INSTALL_PREFIX=${prefixpath} -DCMAKE_BUILD_TYPE=release" + def compiler = 'clang++' + def make_targets = "install" + def compilerpath = "/opt/rocm/llvm/bin/" + compiler + def configargs = "" + if (prefixpath != "") + { + configargs = "-DCMAKE_PREFIX_PATH=${prefixpath}" + } + + def fin_cmd = """ + echo \$HSA_ENABLE_SDMA + ulimit -c unlimited + rm -rf build + mkdir build + cd build + CXX=${compilerpath} cmake ${configargs} ${flags} .. + dumb-init make -j\$(nproc) ${make_targets} + """ + return fin_cmd +} + +def getDockerImageName(dockerArgs) +{ + checkout scm + sh "echo ${dockerArgs} > factors.txt" + def image = "${env.MIOPEN_DOCKER_IMAGE_URL}" + sh "md5sum Dockerfile requirements.txt dev-requirements.txt >> factors.txt" + def docker_hash = sh(script: "md5sum factors.txt | awk '{print \$1}' | head -c 6", returnStdout: true) + sh "rm factors.txt" + echo "Docker tag hash: ${docker_hash}" + image = "${image}:ci_${docker_hash}" + if(params.DOCKER_IMAGE_OVERRIDE && !params.DOCKER_IMAGE_OVERRIDE.empty) + { + echo "Overriding the base docker image with ${params.DOCKER_IMAGE_OVERRIDE}" + image = "${params.DOCKER_IMAGE_OVERRIDE}" + } + return image + +} + +def getDockerImage(Map conf=[:]) +{ + checkout scm + env.DOCKER_BUILDKIT=1 + def prefixpath = conf.get("prefixpath", "/opt/rocm") // one image for each prefix 1: /usr/local 2:/opt/rocm + def gpu_arch = "gfx908;gfx90a;gfx942;gfx1100;1201" // prebuilt dockers should have all the architectures enabled so one image can be used for all stages + def mlir_build = conf.get("mlir_build", "ON") // always ON + def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg GPU_ARCHS='\"${gpu_arch}\"' --build-arg USE_MLIR='${mlir_build}' " + //def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg GPU_ARCHS='\"${gpu_arch}\"' --build-arg USE_MLIR='${mlir_build}' " + if(env.CCACHE_HOST) + { + def check_host = sh(script:"""(printf "PING\r\n";) | nc -N ${env.CCACHE_HOST} 6379 """, returnStdout: true).trim() + if(check_host == "+PONG") + { + echo "FOUND CCACHE SERVER: ${CCACHE_HOST}" + } + else + { + echo "CCACHE SERVER: ${CCACHE_HOST} NOT FOUND, got ${check_host} response" + } + dockerArgs = dockerArgs + " --build-arg CCACHE_SECONDARY_STORAGE='redis://${env.CCACHE_HOST}' --build-arg COMPILER_LAUNCHER='ccache' " + env.CCACHE_DIR = """/tmp/ccache_store""" + env.CCACHE_SECONDARY_STORAGE="""redis://${env.CCACHE_HOST}""" + } + echo "Docker Args: ${dockerArgs}" + + def image = getDockerImageName(dockerArgs) + + def dockerImage + try{ + echo "Pulling down image: ${image}" + dockerImage = docker.image("${image}") + dockerImage.pull() + } + catch(org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){ + echo "The job was cancelled or aborted" + throw e + } + catch(Exception ex) + { + dockerImage = docker.build("${image}", "${dockerArgs} .") + withDockerRegistry([ credentialsId: "docker_test_cred", url: "" ]) { + dockerImage.push() + } + } + return [dockerImage, image] +} + +def buildHipClangJob(Map conf=[:]){ + show_node_info() + miopenCheckout() + checkout scm + env.HSA_ENABLE_SDMA=0 + env.DOCKER_BUILDKIT=1 + def image + def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined" + if (conf.get("enforce_xnack_on", false)) { + dockerOpts = dockerOpts + " --env HSA_XNACK=1" + } + + def variant = env.STAGE_NAME + + def needs_gpu = conf.get("needs_gpu", true) + def lfs_pull = conf.get("lfs_pull", false) + + def retimage + gitStatusWrapper(credentialsId: "${env.miopen_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'MIOpen') { + try { + (retimage, image) = getDockerImage(conf) + if (needs_gpu) { + withDockerContainer(image: image, args: dockerOpts) { + timeout(time: 5, unit: 'MINUTES') + { + sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo' + } + } + } + } + catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){ + echo "The job was cancelled or aborted" + throw e + } + catch(Exception ex) { + (retimage, image) = getDockerImage(conf) + if (needs_gpu) { + withDockerContainer(image: image, args: dockerOpts) { + timeout(time: 5, unit: 'MINUTES') + { + sh 'PATH="/opt/rocm/opencl/bin:/opt/rocm/opencl/bin/x86_64:$PATH" clinfo' + } + } + } + } + + withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') { + timeout(time: 420, unit:'MINUTES') + { + if (lfs_pull) { + sh "git lfs pull --exclude=" + } + + cmake_build(conf) + } + } + } + return retimage +} + +def reboot(){ + build job: 'reboot-slaves', propagate: false , parameters: [string(name: 'server', value: "${env.NODE_NAME}"),] +} + +def buildHipClangJobAndReboot(Map conf=[:]){ + try{ + buildHipClangJob(conf) + if (conf.get("needs_cleanup", true)) { + cleanWs() + } + } + catch(e){ + echo "throwing error exception for the stage" + echo 'Exception occurred: ' + e.toString() + throw e + } + finally{ + if (conf.get("needs_reboot", true)) { + reboot() + } + } +} + + +def CheckPerfDbValid(Map conf=[:]){ + def pdb_image = buildHipClangJob(conf) + pdb_image.inside(){ + dir(path: "$WORKSPACE"){ + sh "ls install/bin/" + sh "MIOPEN_LOG_LEVEL=4 LD_LIBRARY_PATH='install/lib:/opt/rocm/lib/' install/bin/fin -i fin/tests/pdb_check_all.json -o pdb_valid_err.json" + archiveArtifacts "pdb_valid_err.json" + sh "grep clear pdb_valid_err.json" + def has_error = sh ( + script: "echo \$?", + returnStdout: true + ).trim() + assert has_error.toInteger() == 0 + } + } +} +def evaluate(params) +{ + (build_args, partition) = getBuildArgs() + def tuna_docker_name = getDockerName("${backend}") + + docker.withRegistry('', "$DOCKER_CRED"){ + def tuna_docker + tuna_docker = docker.build("${tuna_docker_name}", "${build_args} ." ) + tuna_docker.push() + } + + env_list = params.env.split(' ') + for(item in env_list) + { + if(item.replaceAll("\\s","") != "") + { + if(item.contains("=")) + { + docker_args += " -e ${item}" + } + else + { + error "Not added to env: ${item}" + } + } + } + def eval_cmd = '' + if(params.stage == 'fin_find') + { + eval_cmd = '--fin_steps miopen_find_eval' + } + else + { + eval_cmd = '--fin_steps miopen_perf_eval' + } + if(params.dynamic_solvers_only) + { + eval_cmd += ' --dynamic_solvers_only' + } + + sh "docker run ${docker_args} ${tuna_docker_name} python3 /tuna/tuna/go_fish.py miopen ${eval_cmd} --session_id ${params.session_id} --enqueue_only &" + sh "srun --no-kill -p ${partition} -N 1-10 -l bash -c 'echo ${env.CREDS_PSW} | HOME=/home/slurm docker login -u ${env.CREDS_USR} --password-stdin && HOME=/home/slurm docker run ${docker_args} ${tuna_docker_name} python3 /tuna/tuna/go_fish.py miopen ${eval_cmd} --session_id ${params.session_id}'" +} + +def RunPerfTest(Map conf=[:]){ + checkout scm + def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined" + try { + //(retimage, image) = getDockerImage(conf) + image = conf.get("docker_image").pull() + withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') { + timeout(time: 100, unit: 'MINUTES') + { + //cmake_build(conf) + //unstash 'miopen_tar' + //sh "tar -zxvf build/miopen-hip-*-Linux-runtime.tar.gz" + ld_lib="${env.WORKSPACE}/install/lib" + def filename = conf.get("filename", "") + if(params.PERF_TEST_OVERRIDE != '') + { + echo "Appending MIOpenDriver cmd env vars: ${params.PERF_TEST_OVERRIDE}" + sh "export LD_LIBRARY_PATH=${ld_lib} && ${env.WORKSPACE}/install/bin/test_perf.py --filename ${filename} --install_path ${env.WORKSPACE}/install/ --override ${params.PERF_TEST_OVERRRIDE}" + }else + { + sh "export LD_LIBRARY_PATH=${ld_lib} && ${env.WORKSPACE}/install/bin/test_perf.py --filename ${filename} --install_path ${env.WORKSPACE}/install/" + } + archiveArtifacts artifacts: "install/bin/perf_results/${filename}", allowEmptyArchive: true, fingerprint: true + //sh "export LD_LIBRARY_PATH=${ld_lib} && ${env.WORKSPACE}/install/bin/test_perf.py --filename ${filename} --install_path ${env.WORKSPACE}/install/" + jenkins_url = "${env.artifact_path}/MIOpenPerf/lastSuccessfulBuild/artifact" + if(params.COMPARE_TO_BASE) + { + try { + sh "rm -rf ${env.WORKSPACE}/install/bin/old_results/" + sh "wget -P ${env.WORKSPACE}/install/bin/old_results/ ${jenkins_url}/install/bin/perf_results/${filename}" + } + catch (Exception err){ + currentBuild.result = 'SUCCESS' + } + + try{ + sh "${env.WORKSPACE}/install/bin/test_perf.py --compare_results --old_results_path ${env.WORKSPACE}/install/bin/old_results --filename ${filename}" + } + catch (Exception err){ + currentBuild.result = 'SUCCESS' + } + //cleanWs() + } + } + } + } + catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e){ + echo "The job was cancelled or aborted" + throw e + } +} + +return this