diff --git a/scripts/benchmarks/benches/base.py b/scripts/benchmarks/benches/base.py index 3871938bfd..84e1b8287c 100644 --- a/scripts/benchmarks/benches/base.py +++ b/scripts/benchmarks/benches/base.py @@ -8,7 +8,7 @@ from pathlib import Path from .result import Result from .options import options -from utils.utils import run +from utils.utils import download, run import urllib.request import tarfile @@ -26,7 +26,7 @@ def get_adapter_full_path(): assert False, \ f"could not find adapter file {adapter_path} (and in similar lib paths)" - def run_bench(self, command, env_vars): + def run_bench(self, command, env_vars, ld_library=[]): env_vars_with_forced_adapter = env_vars.copy() if options.ur is not None: env_vars_with_forced_adapter.update( @@ -36,7 +36,8 @@ def run_bench(self, command, env_vars): command=command, env_vars=env_vars_with_forced_adapter, add_sycl=True, - cwd=options.benchmark_cwd + cwd=options.benchmark_cwd, + ld_library=ld_library ).stdout.decode() def create_data_path(self, name): @@ -49,17 +50,9 @@ def create_data_path(self, name): return data_path - def download_untar(self, name, url, file): + def download(self, name, url, file, untar = False): self.data_path = self.create_data_path(name) - data_file = os.path.join(self.data_path, file) - if not Path(data_file).exists(): - print(f"{data_file} does not exist, downloading") - urllib.request.urlretrieve(url, data_file) - file = tarfile.open(data_file) - file.extractall(self.data_path) - file.close() - else: - print(f"{data_file} exists, skipping...") + return download(self.data_path, url, file, True) def name(self): raise NotImplementedError() @@ -79,6 +72,9 @@ def run(self, env_vars) -> list[Result]: def teardown(self): raise NotImplementedError() + def ignore_iterations(self): + return False + class Suite: def benchmarks(self) -> list[Benchmark]: raise NotImplementedError() diff --git a/scripts/benchmarks/benches/llamacpp.py b/scripts/benchmarks/benches/llamacpp.py new file mode 100644 index 0000000000..3ff7963bd1 --- /dev/null +++ b/scripts/benchmarks/benches/llamacpp.py @@ -0,0 +1,196 @@ +# Copyright (C) 2024 Intel Corporation +# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +# See LICENSE.TXT +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import csv +import io +from pathlib import Path +import re +import shutil +from utils.utils import download, git_clone +from .base import Benchmark, Suite +from .result import Result +from utils.utils import run, create_build_path +from .options import options +import os + +class OneAPI: + # random unique number for benchmark oneAPI installation + ONEAPI_BENCHMARK_INSTANCE_ID = 98765 + def __init__(self, directory): + self.oneapi_dir = os.path.join(directory, 'oneapi') + Path(self.oneapi_dir).mkdir(parents=True, exist_ok=True) + # delete if some option is set? + + # can we just hardcode these links? + self.install_package('dnnl', 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/87e117ab-039b-437d-9c80-dcd5c9e675d5/intel-onednn-2025.0.0.862_offline.sh') + self.install_package('mkl', 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940_offline.sh') + return + + def install_package(self, name, url): + package_path = os.path.join(self.oneapi_dir, name) + if Path(package_path).exists(): + print(f"{package_path} exists, skipping installing oneAPI package {name}...") + return + + package = download(self.oneapi_dir, url, f'package_{name}.sh') + try: + print(f"installing f{name}") + run(f"sh {package} -a -s --eula accept --install-dir {self.oneapi_dir} --instance f{self.ONEAPI_BENCHMARK_INSTANCE_ID}") + except: + print("oneAPI installation likely exists already") + return + print(f"f{name} installation complete") + + def package_dir(self, package, dir): + return os.path.join(self.oneapi_dir, package, 'latest', dir) + + def package_cmake(self, package): + package_lib = self.package_dir(package, 'lib') + return os.path.join(package_lib, 'cmake', package) + + def mkl_lib(self): + return self.package_dir('mkl', 'lib') + + def mkl_include(self): + return self.package_dir('mkl', 'include') + + def mkl_cmake(self): + return self.package_cmake('mkl') + + def dnn_lib(self): + return self.package_dir('dnnl', 'lib') + + def dnn_include(self): + return self.package_dir('dnnl', 'include') + + def dnn_cmake(self): + return self.package_cmake('dnnl') + + def tbb_lib(self): + return self.package_dir('tbb', 'lib') + + def tbb_cmake(self): + return self.package_cmake('tbb') + + def compiler_lib(self): + return self.package_dir('compiler', 'lib') + + def ld_libraries(self): + return [ + self.compiler_lib(), + self.mkl_lib(), + self.tbb_lib(), + self.dnn_lib() + ] + +class LlamaCppBench(Suite): + def __init__(self, directory): + if options.sycl is None: + return + + self.directory = directory + + def setup(self): + if options.sycl is None: + return + + repo_path = git_clone(self.directory, "llamacpp-repo", "https://github.com/ggerganov/llama.cpp", "1ee9eea094fe5846c7d8d770aa7caa749d246b23") + + self.models_dir = os.path.join(self.directory, 'models') + Path(self.models_dir).mkdir(parents=True, exist_ok=True) + + self.model = download(self.models_dir, "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi-3-mini-4k-instruct-q4.gguf", "Phi-3-mini-4k-instruct-q4.gguf") + + self.oneapi = OneAPI(self.directory) + + self.build_path = create_build_path(self.directory, 'llamacpp-build') + + configure_command = [ + "cmake", + f"-B {self.build_path}", + f"-S {repo_path}", + f"-DCMAKE_BUILD_TYPE=Release", + f"-DGGML_SYCL=ON", + f"-DCMAKE_C_COMPILER=clang", + f"-DCMAKE_CXX_COMPILER=clang++", + f"-DDNNL_DIR={self.oneapi.dnn_cmake()}", + f"-DTBB_DIR={self.oneapi.tbb_cmake()}", + f'-DCMAKE_CXX_FLAGS=-I"{self.oneapi.mkl_include()}"', + f'-DCMAKE_SHARED_LINKER_FLAGS=-L{self.oneapi.compiler_lib()} -L{self.oneapi.mkl_lib()}' + ] + print(f"{self.__class__.__name__}: Run {configure_command}") + run(configure_command, add_sycl=True) + print(f"{self.__class__.__name__}: Run cmake --build {self.build_path} -j") + run(f"cmake --build {self.build_path} -j", add_sycl=True, ld_library=self.oneapi.ld_libraries()) + + def benchmarks(self) -> list[Benchmark]: + if options.sycl is None: + return [] + + return [ + LlamaBench(self) + ] + +class LlamaBench(Benchmark): + def __init__(self, bench): + self.bench = bench + super().__init__(bench.directory) + + def unit(self): + return "token/s" + + def setup(self): + self.benchmark_bin = os.path.join(self.bench.build_path, 'bin', 'llama-bench') + + def name(self): + return f"llama.cpp" + + def lower_is_better(self): + return False + + def ignore_iterations(self): + return True + + def run(self, env_vars) -> list[Result]: + command = [ + f"{self.benchmark_bin}", + "--output", "csv", + "-n", "128", + "-p", "512", + "-b", "128,256,512", + "--numa", "isolate", + "-t", "56", # TODO: use only as many threads as numa node 0 has cpus + "--model", f"{self.bench.model}", + ] + + result = self.run_bench(command, env_vars, ld_library=self.bench.oneapi.ld_libraries()) + parsed = self.parse_output(result) + results = [] + for r in parsed: + (extra_label, mean) = r + label = f"{self.name()} {extra_label}" + results.append(Result(label=label, value=mean, command=command, env=env_vars, stdout=result)) + return results + + def parse_output(self, output): + csv_file = io.StringIO(output) + reader = csv.DictReader(csv_file) + + results = [] + for row in reader: + try: + n_batch = row["n_batch"] + avg_ts = float(row["avg_ts"]) + n_prompt = int(row["n_prompt"]) + label = "Prompt Processing" if n_prompt != 0 else "Text Generation" + label += f" Batched {n_batch}" + results.append((label, avg_ts)) + except KeyError as e: + raise ValueError(f"Error parsing output: {e}") + + return results + + def teardown(self): + return diff --git a/scripts/benchmarks/benches/velocity.py b/scripts/benchmarks/benches/velocity.py index 38efa42f56..856fd993db 100644 --- a/scripts/benchmarks/benches/velocity.py +++ b/scripts/benchmarks/benches/velocity.py @@ -140,7 +140,7 @@ def __init__(self, vb: VelocityBench): super().__init__("sobel_filter", "sobel_filter", vb) def download_deps(self): - self.download_untar("sobel_filter", "https://github.com/oneapi-src/Velocity-Bench/raw/main/sobel_filter/res/sobel_filter_data.tgz?download=", "sobel_filter_data.tgz") + self.download("sobel_filter", "https://github.com/oneapi-src/Velocity-Bench/raw/main/sobel_filter/res/sobel_filter_data.tgz?download=", "sobel_filter_data.tgz", untar=True) return def name(self): @@ -203,7 +203,7 @@ def __init__(self, vb: VelocityBench): super().__init__("easywave", "easyWave_sycl", vb) def download_deps(self): - self.download_untar("easywave", "https://git.gfz-potsdam.de/id2/geoperil/easyWave/-/raw/master/data/examples.tar.gz", "examples.tar.gz") + self.download("easywave", "https://git.gfz-potsdam.de/id2/geoperil/easyWave/-/raw/master/data/examples.tar.gz", "examples.tar.gz", untar=True) def name(self): return "Velocity-Bench Easywave" diff --git a/scripts/benchmarks/main.py b/scripts/benchmarks/main.py index a31268a240..9dd77f14b2 100755 --- a/scripts/benchmarks/main.py +++ b/scripts/benchmarks/main.py @@ -8,6 +8,7 @@ from benches.compute import * from benches.velocity import VelocityBench from benches.syclbench import * +from benches.llamacpp import * from benches.test import TestSuite from benches.options import Compare, options from output_markdown import generate_markdown @@ -27,7 +28,8 @@ def main(directory, additional_env_vars, save_name, compare_names, filter): suites = [ ComputeBench(directory), VelocityBench(directory), - SyclBench(directory) + SyclBench(directory), + LlamaCppBench(directory), #TestSuite() ] if not options.dry_run else [] @@ -64,7 +66,8 @@ def main(directory, additional_env_vars, save_name, compare_names, filter): try: merged_env_vars = {**additional_env_vars} iteration_results = [] - for iter in range(options.iterations): + iterations = options.iterations if not benchmark.ignore_iterations() else 1 + for iter in range(iterations): print(f"running {benchmark.name()}, iteration {iter}... ", end='', flush=True) bench_results = benchmark.run(merged_env_vars) if bench_results is not None: diff --git a/scripts/benchmarks/utils/utils.py b/scripts/benchmarks/utils/utils.py index 0cd10b9513..d077184e5c 100644 --- a/scripts/benchmarks/utils/utils.py +++ b/scripts/benchmarks/utils/utils.py @@ -5,17 +5,24 @@ import os import shutil -import subprocess # nosec B404 +import subprocess + +import tarfile +import urllib # nosec B404 from benches.options import options from pathlib import Path -def run(command, env_vars={}, cwd=None, add_sycl=False): +def run(command, env_vars={}, cwd=None, add_sycl=False, ld_library=[]): try: if isinstance(command, str): command = command.split() env = os.environ.copy() + for ldlib in ld_library: + env['LD_LIBRARY_PATH'] = ldlib + os.pathsep + env.get('LD_LIBRARY_PATH', '') + + # order is important, we want provided sycl rt libraries to be first if add_sycl: sycl_bin_path = os.path.join(options.sycl, 'bin') env['PATH'] = sycl_bin_path + os.pathsep + env.get('PATH', '') @@ -23,6 +30,7 @@ def run(command, env_vars={}, cwd=None, add_sycl=False): env['LD_LIBRARY_PATH'] = sycl_lib_path + os.pathsep + env.get('LD_LIBRARY_PATH', '') env.update(env_vars) + result = subprocess.run(command, cwd=cwd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=options.timeout) # nosec B603 if options.verbose: @@ -88,3 +96,16 @@ def create_build_path(directory, name): Path(build_path).mkdir(parents=True, exist_ok=True) return build_path + +def download(dir, url, file, untar = False): + data_file = os.path.join(dir, file) + if not Path(data_file).exists(): + print(f"{data_file} does not exist, downloading") + urllib.request.urlretrieve(url, data_file) + if untar: + file = tarfile.open(data_file) + file.extractall(dir) + file.close() + else: + print(f"{data_file} exists, skipping...") + return data_file