diff --git a/.github/workflows/benchmarks-reusable.yml b/.github/workflows/benchmarks-reusable.yml index bfd1e971f9..e823de389c 100644 --- a/.github/workflows/benchmarks-reusable.yml +++ b/.github/workflows/benchmarks-reusable.yml @@ -205,6 +205,7 @@ jobs: --umf ${{ github.workspace }}/umf_build --adapter ${{ matrix.adapter.str_name }} --compute-runtime ${{ inputs.compute_runtime_commit }} + --build-igc ${{ inputs.upload_report && '--output-html' || '' }} ${{ inputs.bench_script_params }} diff --git a/scripts/benchmarks/README.md b/scripts/benchmarks/README.md index bd6de60a0a..9cef0e52a3 100644 --- a/scripts/benchmarks/README.md +++ b/scripts/benchmarks/README.md @@ -9,11 +9,11 @@ Scripts for running performance tests on SYCL and Unified Runtime. ## Running -`$ ./main.py ~/benchmarks_workdir/ ~/llvm/build/ ~/ur adapter_name` +`$ ./main.py ~/benchmarks_workdir/ --sycl ~/llvm/build/ --ur ~/ur --adapter adapter_name` This will download and build everything in `~/benchmarks_workdir/` using the compiler in `~/llvm/build/`, UR source from `~/ur` and then run the benchmarks for `adapter_name` adapter. The results will be stored in `benchmark_results.md`. -The scripts will try to reuse the files stored in `~/benchmarks_workdir/`, but the benchmarks will be rebuilt every time. To avoid that, use `-no-rebuild` option. +The scripts will try to reuse the files stored in `~/benchmarks_workdir/`, but the benchmarks will be rebuilt every time. To avoid that, use `--no-rebuild` option. ## Running in CI @@ -47,7 +47,27 @@ are stored [here](https://oneapi-src.github.io/unified-runtime/benchmark_results ### Python dataclasses-json==0.6.7 +PyYAML==6.0.2 +Mako==1.3.0 ### System -libopencv-dev +Sobel Filter benchmark: + +`$ sudo apt-get install libopencv-dev` + +### Compute-runtime and IGC + +The scripts have an option to build compute-runtime and all related components from source: + +`$ ./main.py ~/benchmarks_workdir/ --compute-runtime [tag] --build-igc` + +For this to work, the system needs to have the appropriate dependencies installed. + +compute-runtime (Ubuntu): + +`$ sudo apt-get install cmake g++ git pkg-config` + +IGC (Ubuntu): + +`$ sudo apt-get install flex bison libz-dev cmake libc6 libstdc++6 python3-pip` diff --git a/scripts/benchmarks/main.py b/scripts/benchmarks/main.py index e5f55b32fe..1d076932f6 100755 --- a/scripts/benchmarks/main.py +++ b/scripts/benchmarks/main.py @@ -244,7 +244,7 @@ def validate_and_parse_env_args(env_args): parser.add_argument('--ur', type=str, help='UR install prefix path', default=None) parser.add_argument('--umf', type=str, help='UMF install prefix path', default=None) parser.add_argument('--adapter', type=str, help='Options to build the Unified Runtime as part of the benchmark', default="level_zero") - parser.add_argument("--no-rebuild", help='Rebuild the benchmarks from scratch.', action="store_true") + parser.add_argument("--no-rebuild", help='Do not rebuild the benchmarks from scratch.', action="store_true") parser.add_argument("--env", type=str, help='Use env variable for a benchmark run.', action="append", default=[]) parser.add_argument("--save", type=str, help='Save the results for comparison under a specified name.') parser.add_argument("--compare", type=str, help='Compare results against previously saved data.', action="append", default=["baseline"]) @@ -262,6 +262,7 @@ def validate_and_parse_env_args(env_args): parser.add_argument("--dry-run", help='Do not run any actual benchmarks', action="store_true", default=False) parser.add_argument("--compute-runtime", nargs='?', const=options.compute_runtime_tag, help="Fetch and build compute runtime") parser.add_argument("--iterations-stddev", type=int, help="Max number of iterations of the loop calculating stddev after completed benchmark runs", default=options.iterations_stddev) + parser.add_argument("--build-igc", help="Build IGC from source instead of using the OS-installed version", action="store_true", default=options.build_igc) args = parser.parse_args() additional_env_vars = validate_and_parse_env_args(args.env) @@ -283,6 +284,10 @@ def validate_and_parse_env_args(env_args): options.dry_run = args.dry_run options.umf = args.umf options.iterations_stddev = args.iterations_stddev + options.build_igc = args.build_igc + + if args.build_igc and args.compute_runtime is None: + parser.error("--build-igc requires --compute-runtime to be set") if args.compute_runtime is not None: options.build_compute_runtime = True options.compute_runtime_tag = args.compute_runtime diff --git a/scripts/benchmarks/options.py b/scripts/benchmarks/options.py index 39f34a73e6..3af3adb868 100644 --- a/scripts/benchmarks/options.py +++ b/scripts/benchmarks/options.py @@ -30,7 +30,8 @@ class Options: build_compute_runtime: bool = False extra_ld_libraries: list[str] = field(default_factory=list) extra_env_vars: dict = field(default_factory=dict) - compute_runtime_tag: str = 'c1ed0334d65f6ce86d7273fe4137d1d4a5b5fa7c' + compute_runtime_tag: str = '24.52.32224.8' + build_igc: bool = False options = Options() diff --git a/scripts/benchmarks/utils/compute_runtime.py b/scripts/benchmarks/utils/compute_runtime.py index b9fb39be4c..671f99192a 100644 --- a/scripts/benchmarks/utils/compute_runtime.py +++ b/scripts/benchmarks/utils/compute_runtime.py @@ -5,6 +5,7 @@ import os import re +import yaml from pathlib import Path from .utils import * @@ -21,9 +22,7 @@ def replace_in_file(file_path, search_pattern, replacement): class ComputeRuntime: def __init__(self): - self.gmmlib = self.build_gmmlib() - self.level_zero = self.build_level_zero() - self.compute_runtime = self.build_compute_runtime(self.gmmlib, self.level_zero) + self.compute_runtime = self.build_compute_runtime() return @@ -32,14 +31,15 @@ def ld_libraries(self) -> list[str]: os.path.join(self.gmmlib, "lib64"), os.path.join(self.level_zero, "lib64"), os.path.join(self.compute_runtime, "bin"), + os.path.join(self.igc, "lib"), ] def env_vars(self) -> dict: return {"ZE_ENABLE_ALT_DRIVERS" : os.path.join(self.compute_runtime, "bin", "libze_intel_gpu.so"), "OCL_ICD_FILENAMES" : os.path.join(self.compute_runtime, "bin", "libigdrcl.so")} - def build_gmmlib(self): - self.gmmlib_repo = git_clone(options.workdir, "gmmlib-repo", "https://github.com/intel/gmmlib.git", "9104c2090158b35d440afdf8ec940d89cc7b3c6a") + def build_gmmlib(self, repo, commit): + self.gmmlib_repo = git_clone(options.workdir, "gmmlib-repo", repo, commit) self.gmmlib_build = os.path.join(options.workdir, "gmmlib-build") self.gmmlib_install = os.path.join(options.workdir, "gmmlib-install") configure_command = [ @@ -54,8 +54,8 @@ def build_gmmlib(self): run(f"cmake --install {self.gmmlib_build}") return self.gmmlib_install - def build_level_zero(self): - self.level_zero_repo = git_clone(options.workdir, "level-zero-repo", "https://github.com/oneapi-src/level-zero.git", "3969f34c16a843b943b948f8fe7081ef87deb369") + def build_level_zero(self, repo, commit): + self.level_zero_repo = git_clone(options.workdir, "level-zero-repo", repo, commit) self.level_zero_build = os.path.join(options.workdir, "level-zero-build") self.level_zero_install = os.path.join(options.workdir, "level-zero-install") @@ -75,10 +75,64 @@ def build_level_zero(self): run(f"cmake --install {self.level_zero_build}") return self.level_zero_install - def build_compute_runtime(self, gmmlib, level_zero): + + def build_igc(self, repo, commit): + self.igc_repo = git_clone(options.workdir, "igc", repo, commit) + self.vc_intr = git_clone(options.workdir, "vc-intrinsics", "https://github.com/intel/vc-intrinsics", "facb2076a2ce6cd6527c1e16570ba0fbaa2f1dba") + self.llvm_project = git_clone(options.workdir, "llvm-project", "https://github.com/llvm/llvm-project", "llvmorg-14.0.5") + llvm_projects = os.path.join(self.llvm_project, "llvm", "projects") + self.ocl = git_clone(llvm_projects, "opencl-clang", "https://github.com/intel/opencl-clang", "ocl-open-140") + self.translator = git_clone(llvm_projects, "llvm-spirv", "https://github.com/KhronosGroup/SPIRV-LLVM-Translator", "llvm_release_140") + self.spirv_tools = git_clone(options.workdir, "SPIRV-Tools", "https://github.com/KhronosGroup/SPIRV-Tools.git", "173fe3c60a8d9c7d35d7842ae267bb9df267a127") + self.spirv_headers = git_clone(options.workdir, "SPIRV-Headers", "https://github.com/KhronosGroup/SPIRV-Headers.git", "2b2e05e088841c63c0b6fd4c9fb380d8688738d3") + + self.igc_build = os.path.join(options.workdir, "igc-build") + self.igc_install = os.path.join(options.workdir, "igc-install") + configure_command = [ + "cmake", + f"-B {self.igc_build}", + f"-S {self.igc_repo}", + f"-DCMAKE_INSTALL_PREFIX={self.igc_install}", + f"-DCMAKE_BUILD_TYPE=Release", + ] + run(configure_command) + + # set timeout to 30min. IGC takes A LONG time to build if building from scratch. + run(f"cmake --build {self.igc_build} -j", timeout=600 * 3) + # cmake --install doesn't work... + run("make install", cwd=self.igc_build) + return self.igc_install + + def read_manifest(self, manifest_path): + with open(manifest_path, 'r') as file: + manifest = yaml.safe_load(file) + return manifest + + def get_repo_info(self, manifest, component_name): + component = manifest['components'].get(component_name) + if component: + repo = component.get('repository') + revision = component.get('revision') + return repo, revision + return None, None + + def build_compute_runtime(self): self.compute_runtime_repo = git_clone(options.workdir, "compute-runtime-repo", "https://github.com/intel/compute-runtime.git", options.compute_runtime_tag) self.compute_runtime_build = os.path.join(options.workdir, "compute-runtime-build") + manifest_path = os.path.join(self.compute_runtime_repo, "manifests", "manifest.yml") + manifest = self.read_manifest(manifest_path) + + level_zero_repo, level_zero_commit = self.get_repo_info(manifest, 'level_zero') + self.level_zero = self.build_level_zero(level_zero_repo, level_zero_commit) + + gmmlib_repo, gmmlib_commit = self.get_repo_info(manifest, 'gmmlib') + self.gmmlib = self.build_gmmlib(gmmlib_repo, gmmlib_commit) + + if options.build_igc: + igc_repo, igc_commit = self.get_repo_info(manifest, 'igc') + self.igc = self.build_igc(igc_repo, igc_commit) + cmakelists_path = os.path.join(self.compute_runtime_repo, "level_zero", "cmake", "FindLevelZero.cmake") # specifying custom L0 is problematic... replace_in_file(cmakelists_path, r'(\$\{LEVEL_ZERO_ROOT\}\s*)', r'\1NO_DEFAULT_PATH\n') @@ -95,9 +149,12 @@ def build_compute_runtime(self, gmmlib, level_zero): "-DNEO_ENABLE_i915_PRELIM_DETECTION=1", "-DNEO_ENABLE_I915_PRELIM_DETECTION=1", "-DNEO_SKIP_UNIT_TESTS=1", - f"-DGMM_DIR={gmmlib}", - f"-DLEVEL_ZERO_ROOT={level_zero}" + f"-DGMM_DIR={self.gmmlib}", + f"-DLEVEL_ZERO_ROOT={self.level_zero}" ] + if options.build_igc: + configure_command.append(f"-DIGC_DIR={self.igc}") + run(configure_command) run(f"cmake --build {self.compute_runtime_build} -j") return self.compute_runtime_build diff --git a/scripts/benchmarks/utils/utils.py b/scripts/benchmarks/utils/utils.py index 08d26bd708..9353df0762 100644 --- a/scripts/benchmarks/utils/utils.py +++ b/scripts/benchmarks/utils/utils.py @@ -13,7 +13,7 @@ from options import options from pathlib import Path -def run(command, env_vars={}, cwd=None, add_sycl=False, ld_library=[]): +def run(command, env_vars={}, cwd=None, add_sycl=False, ld_library=[], timeout=options.timeout): try: if isinstance(command, str): command = command.split() @@ -32,7 +32,7 @@ def run(command, env_vars={}, cwd=None, add_sycl=False, ld_library=[]): env.update(env_vars) - result = subprocess.run(command, cwd=cwd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=options.timeout) # nosec B603 + result = subprocess.run(command, cwd=cwd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=timeout) # nosec B603 if options.verbose: print(result.stdout.decode()) diff --git a/third_party/benchmark_requirements.txt b/third_party/benchmark_requirements.txt index c01a2215c5..0df355c4be 100644 --- a/third_party/benchmark_requirements.txt +++ b/third_party/benchmark_requirements.txt @@ -41,3 +41,4 @@ sphinxcontrib-websupport==1.2.4 sphinx-rtd-theme==1.0.0 urllib3==2.2.2 dataclasses-json==0.6.7 +PyYAML==6.0.1