Introduce BENCHMARKING_METHOD for Triton benchmarks (#2376)

Required for #2343 because it will add a new benchmarking method: `UPSTREAM_PYTORCH_PROFILER` The option is added more with an eye on CI, so for now I left `USE_IPEX` option so as not to change the interaction interface (default behavior is also not changed). --------- Signed-off-by: Anatoly Myachev <[email protected]> Co-authored-by: Pavel Chekin <[email protected]>
intel · Sep 27, 2024 · edc8bdb · edc8bdb
1 parent 59e2621
commit edc8bdb
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 10 deletions.
diff --git a/.github/workflows/triton-benchmarks.yml b/.github/workflows/triton-benchmarks.yml
@@ -11,18 +11,22 @@ on:
         description: Tag for benchmark results
         type: string
         default: "test"
-      install_ipex:
-        description: Install Intel PyTorch Extension
-        type: boolean
-        default: true
+      benchmarking_method:
+        description: The method used to obtain performance numbers
+        type: choice
+        options:
+          - PYTORCH_LEGACY_PROFILER_USING_IPEX
+          - ELAPSED_TIME
+        default: PYTORCH_LEGACY_PROFILER_USING_IPEX
   schedule:
     - cron: "5 23 * * *"
 
 permissions: read-all
 
 env:
   PYTHON_VERSION: "3.10"
-  USE_IPEX: ${{ github.event_name == 'schedule' && '1' || inputs.install_ipex && '1' || '0' }}
+  BENCHMARKING_METHOD: ${{ github.event_name == 'schedule' && 'PYTORCH_LEGACY_PROFILER_USING_IPEX' || inputs.benchmarking_method }}
+  USE_IPEX: ${{ github.event_name == 'schedule' && '1' || inputs.benchmarking_method  == 'PYTORCH_LEGACY_PROFILER_USING_IPEX' && '1' || '0' }}
 
 jobs:
   build:

diff --git a/benchmarks/triton_kernels_benchmark/benchmark_testing.py b/benchmarks/triton_kernels_benchmark/benchmark_testing.py
@@ -4,6 +4,10 @@
 from typing import Any, Dict, List
 
 USE_IPEX_OPTION = os.getenv("USE_IPEX", "1") == "1"
+if USE_IPEX_OPTION:
+    BENCHMARKING_METHOD = "PYTORCH_LEGACY_PROFILER_USING_IPEX"
+else:
+    BENCHMARKING_METHOD = os.getenv("BENCHMARKING_METHOD", "ELAPSED_TIME")
 
 
 def synchronize():
@@ -122,8 +126,8 @@ def extract_kernels(funcs):
     return _summarize_statistics(times, quantiles, return_mode)
 
 
-def do_bench_no_ipex(fn, warmup=25, rep=100, grad_to_none=None, quantiles=None, fast_flush=True, return_mode="mean",
-                     device="xpu"):
+def do_bench_elapsed_time(fn, warmup=25, rep=100, grad_to_none=None, quantiles=None, fast_flush=True,
+                          return_mode="mean", device="xpu"):
     """
     Benchmark the runtime of the provided function. By default, return the median runtime of :code:`fn` along with
     the 20-th and 80-th performance percentile.
@@ -151,9 +155,12 @@ def do_bench_no_ipex(fn, warmup=25, rep=100, grad_to_none=None, quantiles=None,
     return _summarize_statistics(times, quantiles, return_mode)
 
 
-do_bench = do_bench_no_ipex
-if USE_IPEX_OPTION:
+if BENCHMARKING_METHOD == "PYTORCH_LEGACY_PROFILER_USING_IPEX":
     do_bench = do_bench_ipex
+elif BENCHMARKING_METHOD == "ELAPSED_TIME":
+    do_bench = do_bench_elapsed_time
+else:
+    raise NotImplementedError(f"BENCHMARKING_METHOD: {BENCHMARKING_METHOD} isn't implemented")
 
 
 def assert_close(x, y, atol=None, rtol=None, err_msg=""):

diff --git a/scripts/capture-hw-details.sh b/scripts/capture-hw-details.sh
@@ -61,7 +61,7 @@ fi
 if [[ "${USE_IPEX:-}" == "1" ]]; then
     export BENCHMARKING_METHOD="PYTORCH_LEGACY_PROFILER_USING_IPEX"
 elif [[ "${USE_IPEX:-}" == "0" ]]; then
-    export BENCHMARKING_METHOD="ELAPSED_TIME"
+    export BENCHMARKING_METHOD="${BENCHMARKING_METHOD:-ELAPSED_TIME}"
 fi
 
 if [ "$QUIET" = false ]; then