Merge pull request #2621 from pbalcer/compute-runtime-manifest

[benchmarks] use manifest to build compute-runtime dependencies

.github/workflows/benchmarks-reusable.yml

@@ -205,6 +205,7 @@ jobs:
         --umf ${{ github.workspace }}/umf_build
         --adapter ${{ matrix.adapter.str_name }}
         --compute-runtime ${{ inputs.compute_runtime_commit }}
+        --build-igc
         ${{ inputs.upload_report && '--output-html' || '' }}
         ${{ inputs.bench_script_params }}
 

scripts/benchmarks/README.md

@@ -9,11 +9,11 @@ Scripts for running performance tests on SYCL and Unified Runtime.
 
 ## Running
 
-`$ ./main.py ~/benchmarks_workdir/ ~/llvm/build/ ~/ur adapter_name`
+`$ ./main.py ~/benchmarks_workdir/ --sycl ~/llvm/build/ --ur ~/ur --adapter adapter_name`
 
 This will download and build everything in `~/benchmarks_workdir/` using the compiler in `~/llvm/build/`, UR source from `~/ur` and then run the benchmarks for `adapter_name` adapter. The results will be stored in `benchmark_results.md`.
 
-The scripts will try to reuse the files stored in `~/benchmarks_workdir/`, but the benchmarks will be rebuilt every time. To avoid that, use `-no-rebuild` option.
+The scripts will try to reuse the files stored in `~/benchmarks_workdir/`, but the benchmarks will be rebuilt every time. To avoid that, use `--no-rebuild` option.
 
 ## Running in CI
 
@@ -47,7 +47,27 @@ are stored [here](https://oneapi-src.github.io/unified-runtime/benchmark_results
 ### Python
 
 dataclasses-json==0.6.7
+PyYAML==6.0.2
+Mako==1.3.0
 
 ### System
 
-libopencv-dev
+Sobel Filter benchmark:
+
+`$ sudo apt-get install libopencv-dev`
+
+### Compute-runtime and IGC
+
+The scripts have an option to build compute-runtime and all related components from source:
+
+`$ ./main.py ~/benchmarks_workdir/ --compute-runtime [tag] --build-igc`
+
+For this to work, the system needs to have the appropriate dependencies installed.
+
+compute-runtime (Ubuntu):
+
+`$ sudo apt-get install cmake g++ git pkg-config`
+
+IGC (Ubuntu):
+
+`$ sudo apt-get install flex bison libz-dev cmake libc6 libstdc++6 python3-pip`

scripts/benchmarks/main.py

@@ -244,7 +244,7 @@ def validate_and_parse_env_args(env_args):
     parser.add_argument('--ur', type=str, help='UR install prefix path', default=None)
     parser.add_argument('--umf', type=str, help='UMF install prefix path', default=None)
     parser.add_argument('--adapter', type=str, help='Options to build the Unified Runtime as part of the benchmark', default="level_zero")
-    parser.add_argument("--no-rebuild", help='Rebuild the benchmarks from scratch.', action="store_true")
+    parser.add_argument("--no-rebuild", help='Do not rebuild the benchmarks from scratch.', action="store_true")
     parser.add_argument("--env", type=str, help='Use env variable for a benchmark run.', action="append", default=[])
     parser.add_argument("--save", type=str, help='Save the results for comparison under a specified name.')
     parser.add_argument("--compare", type=str, help='Compare results against previously saved data.', action="append", default=["baseline"])
@@ -262,6 +262,7 @@ def validate_and_parse_env_args(env_args):
     parser.add_argument("--dry-run", help='Do not run any actual benchmarks', action="store_true", default=False)
     parser.add_argument("--compute-runtime", nargs='?', const=options.compute_runtime_tag, help="Fetch and build compute runtime")
     parser.add_argument("--iterations-stddev", type=int, help="Max number of iterations of the loop calculating stddev after completed benchmark runs", default=options.iterations_stddev)
+    parser.add_argument("--build-igc", help="Build IGC from source instead of using the OS-installed version", action="store_true", default=options.build_igc)
 
     args = parser.parse_args()
     additional_env_vars = validate_and_parse_env_args(args.env)
@@ -283,6 +284,10 @@ def validate_and_parse_env_args(env_args):
     options.dry_run = args.dry_run
     options.umf = args.umf
     options.iterations_stddev = args.iterations_stddev
+    options.build_igc = args.build_igc
+
+    if args.build_igc and args.compute_runtime is None:
+        parser.error("--build-igc requires --compute-runtime to be set")
     if args.compute_runtime is not None:
         options.build_compute_runtime = True
         options.compute_runtime_tag = args.compute_runtime

scripts/benchmarks/options.py

@@ -30,7 +30,8 @@ class Options:
     build_compute_runtime: bool = False
     extra_ld_libraries: list[str] = field(default_factory=list)
     extra_env_vars: dict = field(default_factory=dict)
-    compute_runtime_tag: str = 'c1ed0334d65f6ce86d7273fe4137d1d4a5b5fa7c'
+    compute_runtime_tag: str = '24.52.32224.8'
+    build_igc: bool = False
 
 options = Options()
 

scripts/benchmarks/utils/compute_runtime.py

@@ -5,6 +5,7 @@
 
 import os
 import re
+import yaml
 
 from pathlib import Path
 from .utils import *
@@ -21,9 +22,7 @@ def replace_in_file(file_path, search_pattern, replacement):
 
 class ComputeRuntime:
     def __init__(self):
-        self.gmmlib = self.build_gmmlib()
-        self.level_zero = self.build_level_zero()
-        self.compute_runtime = self.build_compute_runtime(self.gmmlib, self.level_zero)
+        self.compute_runtime = self.build_compute_runtime()
 
         return
 
@@ -32,14 +31,15 @@ def ld_libraries(self) -> list[str]:
             os.path.join(self.gmmlib, "lib64"),
             os.path.join(self.level_zero, "lib64"),
             os.path.join(self.compute_runtime, "bin"),
+            os.path.join(self.igc, "lib"),
         ]
 
     def env_vars(self) -> dict:
         return {"ZE_ENABLE_ALT_DRIVERS" : os.path.join(self.compute_runtime, "bin", "libze_intel_gpu.so"),
                 "OCL_ICD_FILENAMES" : os.path.join(self.compute_runtime, "bin", "libigdrcl.so")}
 
-    def build_gmmlib(self):
-        self.gmmlib_repo = git_clone(options.workdir, "gmmlib-repo", "https://github.com/intel/gmmlib.git", "9104c2090158b35d440afdf8ec940d89cc7b3c6a")
+    def build_gmmlib(self, repo, commit):
+        self.gmmlib_repo = git_clone(options.workdir, "gmmlib-repo", repo, commit)
         self.gmmlib_build = os.path.join(options.workdir, "gmmlib-build")
         self.gmmlib_install = os.path.join(options.workdir, "gmmlib-install")
         configure_command = [
@@ -54,8 +54,8 @@ def build_gmmlib(self):
         run(f"cmake --install {self.gmmlib_build}")
         return self.gmmlib_install
 
-    def build_level_zero(self):
-        self.level_zero_repo = git_clone(options.workdir, "level-zero-repo", "https://github.com/oneapi-src/level-zero.git", "3969f34c16a843b943b948f8fe7081ef87deb369")
+    def build_level_zero(self, repo, commit):
+        self.level_zero_repo = git_clone(options.workdir, "level-zero-repo", repo, commit)
         self.level_zero_build = os.path.join(options.workdir, "level-zero-build")
         self.level_zero_install = os.path.join(options.workdir, "level-zero-install")
 
@@ -75,10 +75,64 @@ def build_level_zero(self):
         run(f"cmake --install {self.level_zero_build}")
         return self.level_zero_install
 
-    def build_compute_runtime(self, gmmlib, level_zero):
+
+    def build_igc(self, repo, commit):
+        self.igc_repo = git_clone(options.workdir, "igc", repo, commit)
+        self.vc_intr = git_clone(options.workdir, "vc-intrinsics", "https://github.com/intel/vc-intrinsics", "facb2076a2ce6cd6527c1e16570ba0fbaa2f1dba")
+        self.llvm_project = git_clone(options.workdir, "llvm-project", "https://github.com/llvm/llvm-project", "llvmorg-14.0.5")
+        llvm_projects = os.path.join(self.llvm_project, "llvm", "projects")
+        self.ocl = git_clone(llvm_projects, "opencl-clang", "https://github.com/intel/opencl-clang", "ocl-open-140")
+        self.translator = git_clone(llvm_projects, "llvm-spirv", "https://github.com/KhronosGroup/SPIRV-LLVM-Translator", "llvm_release_140")
+        self.spirv_tools = git_clone(options.workdir, "SPIRV-Tools", "https://github.com/KhronosGroup/SPIRV-Tools.git", "173fe3c60a8d9c7d35d7842ae267bb9df267a127")
+        self.spirv_headers = git_clone(options.workdir, "SPIRV-Headers", "https://github.com/KhronosGroup/SPIRV-Headers.git", "2b2e05e088841c63c0b6fd4c9fb380d8688738d3")
+
+        self.igc_build = os.path.join(options.workdir, "igc-build")
+        self.igc_install = os.path.join(options.workdir, "igc-install")
+        configure_command = [
+            "cmake",
+            f"-B {self.igc_build}",
+            f"-S {self.igc_repo}",
+            f"-DCMAKE_INSTALL_PREFIX={self.igc_install}",
+            f"-DCMAKE_BUILD_TYPE=Release",
+        ]
+        run(configure_command)
+
+        # set timeout to 30min. IGC takes A LONG time to build if building from scratch.
+        run(f"cmake --build {self.igc_build} -j", timeout=600 * 3)
+        # cmake --install doesn't work...
+        run("make install", cwd=self.igc_build)
+        return self.igc_install
+
+    def read_manifest(self, manifest_path):
+        with open(manifest_path, 'r') as file:
+            manifest = yaml.safe_load(file)
+        return manifest
+
+    def get_repo_info(self, manifest, component_name):
+        component = manifest['components'].get(component_name)
+        if component:
+            repo = component.get('repository')
+            revision = component.get('revision')
+            return repo, revision
+        return None, None
+
+    def build_compute_runtime(self):
         self.compute_runtime_repo = git_clone(options.workdir, "compute-runtime-repo", "https://github.com/intel/compute-runtime.git", options.compute_runtime_tag)
         self.compute_runtime_build = os.path.join(options.workdir, "compute-runtime-build")
 
+        manifest_path = os.path.join(self.compute_runtime_repo, "manifests", "manifest.yml")
+        manifest = self.read_manifest(manifest_path)
+
+        level_zero_repo, level_zero_commit = self.get_repo_info(manifest, 'level_zero')
+        self.level_zero = self.build_level_zero(level_zero_repo, level_zero_commit)
+
+        gmmlib_repo, gmmlib_commit = self.get_repo_info(manifest, 'gmmlib')
+        self.gmmlib = self.build_gmmlib(gmmlib_repo, gmmlib_commit)
+
+        if options.build_igc:
+            igc_repo, igc_commit = self.get_repo_info(manifest, 'igc')
+            self.igc = self.build_igc(igc_repo, igc_commit)
+
         cmakelists_path = os.path.join(self.compute_runtime_repo, "level_zero", "cmake", "FindLevelZero.cmake")
         # specifying custom L0 is problematic...
         replace_in_file(cmakelists_path, r'(\$\{LEVEL_ZERO_ROOT\}\s*)', r'\1NO_DEFAULT_PATH\n')
@@ -95,9 +149,12 @@ def build_compute_runtime(self, gmmlib, level_zero):
             "-DNEO_ENABLE_i915_PRELIM_DETECTION=1",
             "-DNEO_ENABLE_I915_PRELIM_DETECTION=1",
             "-DNEO_SKIP_UNIT_TESTS=1",
-            f"-DGMM_DIR={gmmlib}",
-            f"-DLEVEL_ZERO_ROOT={level_zero}"
+            f"-DGMM_DIR={self.gmmlib}",
+            f"-DLEVEL_ZERO_ROOT={self.level_zero}"
         ]
+        if options.build_igc:
+            configure_command.append(f"-DIGC_DIR={self.igc}")
+
         run(configure_command)
         run(f"cmake --build {self.compute_runtime_build} -j")
         return self.compute_runtime_build

scripts/benchmarks/utils/utils.py

@@ -13,7 +13,7 @@
 from options import options
 from pathlib import Path
 
-def run(command, env_vars={}, cwd=None, add_sycl=False, ld_library=[]):
+def run(command, env_vars={}, cwd=None, add_sycl=False, ld_library=[], timeout=options.timeout):
     try:
         if isinstance(command, str):
             command = command.split()
@@ -32,7 +32,7 @@ def run(command, env_vars={}, cwd=None, add_sycl=False, ld_library=[]):
 
         env.update(env_vars)
 
-        result = subprocess.run(command, cwd=cwd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=options.timeout) # nosec B603
+        result = subprocess.run(command, cwd=cwd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=timeout) # nosec B603
 
         if options.verbose:
             print(result.stdout.decode())

third_party/benchmark_requirements.txt

@@ -41,3 +41,4 @@ sphinxcontrib-websupport==1.2.4
 sphinx-rtd-theme==1.0.0
 urllib3==2.2.2
 dataclasses-json==0.6.7
+PyYAML==6.0.1