From f36966482baf16b92fdbcae1204e0b33eb4f238b Mon Sep 17 00:00:00 2001 From: srividya sundaram Date: Wed, 17 Jan 2024 08:40:38 -0800 Subject: [PATCH 1/7] [Driver] Save PTX files for SYCL kernels in the user input dir. --- clang/lib/Driver/Compilation.cpp | 22 +++++++++++++++++----- clang/lib/Driver/ToolChains/SYCL.cpp | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/clang/lib/Driver/Compilation.cpp b/clang/lib/Driver/Compilation.cpp index e3c6c2d2ee34f..ebcb00f0e9e80 100644 --- a/clang/lib/Driver/Compilation.cpp +++ b/clang/lib/Driver/Compilation.cpp @@ -128,17 +128,29 @@ bool Compilation::CleanupFile(const char *File, bool IssueErrors) const { // able to remove), or non-regular files. Underlying tools may have // intentionally not overwritten them. - // Save the device code files(spv files) only if -fsycl-dump-device-code - // option is enabled. + // Save the device code files if -fsycl-dump-device-code option is enabled. if (TheDriver.isDumpDeviceCodeEnabled()) { + bool IsNVPTX = false; Arg *DumpDeviceCodeArg = getArgs().getLastArg(options::OPT_fsycl_dump_device_code_EQ); + // Check if SYCL code is being offloaded to CUDA-enabled GPUs. + if (const Arg *SYCLTargets = + getArgs().getLastArg(options::OPT_fsycl_targets_EQ)) { + for (StringRef Val : SYCLTargets->getValues()) { + if (Val.starts_with("nvptx") || Val.starts_with("nvptx64")) + IsNVPTX = true; + } + } std::string ExpectedDir = DumpDeviceCodeArg ? DumpDeviceCodeArg->getValue() : ""; std::string ActualFile(File); - if (ActualFile.find(ExpectedDir) != std::string::npos && - llvm::sys::path::extension(ActualFile).equals(".spv")) - return false; + + if (ActualFile.find(ExpectedDir) != std::string::npos) { + if (IsNVPTX) + return false; + if (llvm::sys::path::extension(ActualFile).equals(".spv")) + return false; + } } if (!llvm::sys::fs::can_write(File) || !llvm::sys::fs::is_regular_file(File)) diff --git a/clang/lib/Driver/ToolChains/SYCL.cpp b/clang/lib/Driver/ToolChains/SYCL.cpp index 5575aaebb3ed0..225a72b688df0 100644 --- a/clang/lib/Driver/ToolChains/SYCL.cpp +++ b/clang/lib/Driver/ToolChains/SYCL.cpp @@ -110,6 +110,26 @@ void SYCL::constructLLVMForeachCommand(Compilation &C, const JobAction &JA, C.getArgs().MakeArgString("--out-dir=" + OutputDirName)); } + // If fsycl-dump-device-code is passed, put the PTX files + // into the path provided in fsycl-dump-device-code. + if (T->getToolChain().getTriple().isNVPTX() && + C.getDriver().isDumpDeviceCodeEnabled() && Ext.equals("s")) { + SmallString<128> OutputDir; + + Arg *DumpDeviceCodeArg = + C.getArgs().getLastArg(options::OPT_fsycl_dump_device_code_EQ); + + OutputDir = (DumpDeviceCodeArg ? DumpDeviceCodeArg->getValue() : ""); + + // If the output directory path is empty, put the PTX files in the + // current directory. + if (OutputDir.empty()) + llvm::sys::path::native(OutputDir = "./"); + else + OutputDir.append(llvm::sys::path::get_separator()); + ForeachArgs.push_back(C.getArgs().MakeArgString("--out-dir=" + OutputDir)); + } + ForeachArgs.push_back(C.getArgs().MakeArgString("--")); ForeachArgs.push_back( C.getArgs().MakeArgString(InputCommand->getExecutable())); From b6af1dd2630869b5c07ba974e64b568f719b833f Mon Sep 17 00:00:00 2001 From: srividya sundaram Date: Mon, 5 Feb 2024 15:08:41 -0800 Subject: [PATCH 2/7] Filter PTX files by toolchain. --- clang/lib/Driver/Compilation.cpp | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/clang/lib/Driver/Compilation.cpp b/clang/lib/Driver/Compilation.cpp index ebcb00f0e9e80..1393b072aeb28 100644 --- a/clang/lib/Driver/Compilation.cpp +++ b/clang/lib/Driver/Compilation.cpp @@ -130,23 +130,16 @@ bool Compilation::CleanupFile(const char *File, bool IssueErrors) const { // Save the device code files if -fsycl-dump-device-code option is enabled. if (TheDriver.isDumpDeviceCodeEnabled()) { - bool IsNVPTX = false; Arg *DumpDeviceCodeArg = getArgs().getLastArg(options::OPT_fsycl_dump_device_code_EQ); - // Check if SYCL code is being offloaded to CUDA-enabled GPUs. - if (const Arg *SYCLTargets = - getArgs().getLastArg(options::OPT_fsycl_targets_EQ)) { - for (StringRef Val : SYCLTargets->getValues()) { - if (Val.starts_with("nvptx") || Val.starts_with("nvptx64")) - IsNVPTX = true; - } - } std::string ExpectedDir = DumpDeviceCodeArg ? DumpDeviceCodeArg->getValue() : ""; std::string ActualFile(File); if (ActualFile.find(ExpectedDir) != std::string::npos) { - if (IsNVPTX) + // Save PTX files generated by LLVM NVPTX Back-End, + // when the nvptx*-nvidia-cuda is passed to -fsycl-targets. + if (DefaultToolChain.getTriple().isNVPTX()) return false; if (llvm::sys::path::extension(ActualFile).equals(".spv")) return false; From 1bbad72c0c13384e28b41276804f29eb41aebd36 Mon Sep 17 00:00:00 2001 From: srividya sundaram Date: Mon, 12 Feb 2024 15:51:07 -0800 Subject: [PATCH 3/7] Add test to save PTX files. --- clang/test/Driver/save-ptx-files.cpp | 43 ++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 clang/test/Driver/save-ptx-files.cpp diff --git a/clang/test/Driver/save-ptx-files.cpp b/clang/test/Driver/save-ptx-files.cpp new file mode 100644 index 0000000000000..dd7253ac79ae8 --- /dev/null +++ b/clang/test/Driver/save-ptx-files.cpp @@ -0,0 +1,43 @@ +// Save PTX files during PTX target processing using -fsycl-dump-device-code option. + +// Verify that -fsycl-dump-device-code saves PTX files in the user provided directory +// while targeting CUDA enabled GPUs. + +// Linux +// RUN: %clang -### -fsycl -fsycl-targets=nvptx64-nvidia-cuda,spir64-unknown-unknown --cuda-path=%S/Inputs/CUDA/usr/local/cuda -fsycl-dump-device-code=/user/input/path %s 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=CHECK-PTX-FILES,CHECK-SPIRV-FILES + +// clang --driver-mode=g++ +// RUN: %clangxx -### -fsycl -fsycl-targets=nvptx64-nvidia-cuda --cuda-path=%S/Inputs/CUDA/usr/local/cuda -fsycl-dump-device-code=/user/input/path %s 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=CHECK-PTX-FILES + +// RUN: %clang -### -fsycl -fsycl-targets=nvptx64-nvidia-cuda,spir64-unknown-unknown --cuda-path=%S/Inputs/CUDA/usr/local/cuda -fsycl-dump-device-code= %s 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=CHECK-PTX-FILES-CWD,CHECK-SPIRV-FILES-CWD + +// CHECK-PTX-FILES: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}clang-18" {{.*}} "-fsycl-is-device" {{.*}}.s{{.*}} +// CHECK-SPIRV-FILES: llvm-foreach{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv" +// CHECK-PTX-FILES-CWD: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}clang-18" {{.*}} "-fsycl-is-device" +// CHECK-SPIRV-FILES-CWD: llvm-foreach{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv" + +// Windows - Check if PTX files are saved in the user provided path. +// RUN: %clang_cl -### -fsycl \ +// RUN: -fsycl-targets=nvptx64-nvidia-cuda --cuda-path=%S/Inputs/CUDA/usr/local/cuda \ +// RUN: -fsycl-dump-device-code=/user/input/path %s 2>&1 \ +// RUN: | FileCheck -check-prefix=CHECK-PTX-WIN %s + +// Windows - Check if PTX and SPV files are saved in user provided path. +// RUN: %clang_cl -### -fsycl \ +// RUN: -fsycl-targets=nvptx64-nvidia-cuda,spir64-unknown-unknown --cuda-path=%S/Inputs/CUDA/usr/local/cuda \ +// RUN: -fsycl-dump-device-code=/user/input/path %s 2>&1 \ +// RUN: | FileCheck -check-prefixes=CHECK-PTX-WIN,CHECK-SPV-WIN %s + +// Windows - Check PTX files saved in current working directory when -fsycl-dump-device-code +// is empty. +// RUN: %clang_cl -### -fsycl \ +// RUN: -fsycl-targets=nvptx64-nvidia-cuda --cuda-path=%S/Inputs/CUDA/usr/local/cuda \ +// RUN: -fsycl-dump-device-code= %s 2>&1 \ +// RUN: | FileCheck -check-prefix=CHECK-PTX-WIN-CWD %s + +// CHECK-PTX-WIN: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}clang-18" {{.*}} "-fsycl-is-device" {{.*}}.asm{{.*}} +// CHECK-PTX-WIN-CWD: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}clang-18" {{.*}} "-fsycl-is-device" {{.*}}.asm{{.*}} +// CHECK-SPV-WIN: llvm-foreach{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv" From 3db83d929a32903411686c33cbc690ecfb0874b3 Mon Sep 17 00:00:00 2001 From: srividya sundaram Date: Tue, 13 Feb 2024 11:45:03 -0800 Subject: [PATCH 4/7] Update test. --- clang/test/Driver/save-ptx-files.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clang/test/Driver/save-ptx-files.cpp b/clang/test/Driver/save-ptx-files.cpp index dd7253ac79ae8..153456756bc8c 100644 --- a/clang/test/Driver/save-ptx-files.cpp +++ b/clang/test/Driver/save-ptx-files.cpp @@ -4,14 +4,14 @@ // while targeting CUDA enabled GPUs. // Linux -// RUN: %clang -### -fsycl -fsycl-targets=nvptx64-nvidia-cuda,spir64-unknown-unknown --cuda-path=%S/Inputs/CUDA/usr/local/cuda -fsycl-dump-device-code=/user/input/path %s 2>&1 \ +// RUN: %clang -### -fsycl -fsycl-targets=nvptx64-nvidia-cuda,spir64-unknown-unknown -target x86_64-unknown-linux-gnu --cuda-path=%S/Inputs/CUDA/usr/local/cuda -fsycl-dump-device-code=/user/input/path %s 2>&1 \ // RUN: | FileCheck %s --check-prefixes=CHECK-PTX-FILES,CHECK-SPIRV-FILES // clang --driver-mode=g++ -// RUN: %clangxx -### -fsycl -fsycl-targets=nvptx64-nvidia-cuda --cuda-path=%S/Inputs/CUDA/usr/local/cuda -fsycl-dump-device-code=/user/input/path %s 2>&1 \ +// RUN: %clangxx -### -fsycl -fsycl-targets=nvptx64-nvidia-cuda -target x86_64-unknown-linux-gnu --cuda-path=%S/Inputs/CUDA/usr/local/cuda -fsycl-dump-device-code=/user/input/path %s 2>&1 \ // RUN: | FileCheck %s --check-prefixes=CHECK-PTX-FILES -// RUN: %clang -### -fsycl -fsycl-targets=nvptx64-nvidia-cuda,spir64-unknown-unknown --cuda-path=%S/Inputs/CUDA/usr/local/cuda -fsycl-dump-device-code= %s 2>&1 \ +// RUN: %clang -### -fsycl -fsycl-targets=nvptx64-nvidia-cuda,spir64-unknown-unknown -target x86_64-unknown-linux-gnu --cuda-path=%S/Inputs/CUDA/usr/local/cuda -fsycl-dump-device-code= %s 2>&1 \ // RUN: | FileCheck %s --check-prefixes=CHECK-PTX-FILES-CWD,CHECK-SPIRV-FILES-CWD // CHECK-PTX-FILES: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}clang-18" {{.*}} "-fsycl-is-device" {{.*}}.s{{.*}} From d3e79257be43e3b8586f00c39d2f9462afde0c69 Mon Sep 17 00:00:00 2001 From: srividya sundaram Date: Wed, 14 Feb 2024 13:32:40 -0800 Subject: [PATCH 5/7] Update test. --- clang/test/Driver/save-ptx-files.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/test/Driver/save-ptx-files.cpp b/clang/test/Driver/save-ptx-files.cpp index 153456756bc8c..ce582be9c7bc0 100644 --- a/clang/test/Driver/save-ptx-files.cpp +++ b/clang/test/Driver/save-ptx-files.cpp @@ -14,7 +14,7 @@ // RUN: %clang -### -fsycl -fsycl-targets=nvptx64-nvidia-cuda,spir64-unknown-unknown -target x86_64-unknown-linux-gnu --cuda-path=%S/Inputs/CUDA/usr/local/cuda -fsycl-dump-device-code= %s 2>&1 \ // RUN: | FileCheck %s --check-prefixes=CHECK-PTX-FILES-CWD,CHECK-SPIRV-FILES-CWD -// CHECK-PTX-FILES: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}clang-18" {{.*}} "-fsycl-is-device" {{.*}}.s{{.*}} +// CHECK-PTX-FILES: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}clang-{{[0-9]+}}" {{.*}} "-fsycl-is-device" {{.*}}.s{{.*}} // CHECK-SPIRV-FILES: llvm-foreach{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv" // CHECK-PTX-FILES-CWD: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}clang-18" {{.*}} "-fsycl-is-device" // CHECK-SPIRV-FILES-CWD: llvm-foreach{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv" From dd84fb50026bec385fd767a4c7f6834af03bd442 Mon Sep 17 00:00:00 2001 From: srividya sundaram Date: Wed, 14 Feb 2024 13:59:51 -0800 Subject: [PATCH 6/7] Update test. --- clang/test/Driver/save-ptx-files.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clang/test/Driver/save-ptx-files.cpp b/clang/test/Driver/save-ptx-files.cpp index ce582be9c7bc0..382dd579d2c17 100644 --- a/clang/test/Driver/save-ptx-files.cpp +++ b/clang/test/Driver/save-ptx-files.cpp @@ -16,7 +16,7 @@ // CHECK-PTX-FILES: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}clang-{{[0-9]+}}" {{.*}} "-fsycl-is-device" {{.*}}.s{{.*}} // CHECK-SPIRV-FILES: llvm-foreach{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv" -// CHECK-PTX-FILES-CWD: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}clang-18" {{.*}} "-fsycl-is-device" +// CHECK-PTX-FILES-CWD: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}clang-{{[0-9]+}}" {{.*}} "-fsycl-is-device" // CHECK-SPIRV-FILES-CWD: llvm-foreach{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv" // Windows - Check if PTX files are saved in the user provided path. @@ -38,6 +38,6 @@ // RUN: -fsycl-dump-device-code= %s 2>&1 \ // RUN: | FileCheck -check-prefix=CHECK-PTX-WIN-CWD %s -// CHECK-PTX-WIN: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}clang-18" {{.*}} "-fsycl-is-device" {{.*}}.asm{{.*}} -// CHECK-PTX-WIN-CWD: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}clang-18" {{.*}} "-fsycl-is-device" {{.*}}.asm{{.*}} +// CHECK-PTX-WIN: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}clang-{{[0-9]+}}" {{.*}} "-fsycl-is-device" {{.*}}.asm{{.*}} +// CHECK-PTX-WIN-CWD: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}clang-{{[0-9]+}}" {{.*}} "-fsycl-is-device" {{.*}}.asm{{.*}} // CHECK-SPV-WIN: llvm-foreach{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv" From 3de0309d2ba0faf1c5f5f964adaaef707095b3b5 Mon Sep 17 00:00:00 2001 From: srividya sundaram Date: Tue, 20 Feb 2024 13:15:37 -0800 Subject: [PATCH 7/7] Fix test failure on windows. --- clang/test/Driver/save-ptx-files.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clang/test/Driver/save-ptx-files.cpp b/clang/test/Driver/save-ptx-files.cpp index 382dd579d2c17..d9545d16ef890 100644 --- a/clang/test/Driver/save-ptx-files.cpp +++ b/clang/test/Driver/save-ptx-files.cpp @@ -14,9 +14,9 @@ // RUN: %clang -### -fsycl -fsycl-targets=nvptx64-nvidia-cuda,spir64-unknown-unknown -target x86_64-unknown-linux-gnu --cuda-path=%S/Inputs/CUDA/usr/local/cuda -fsycl-dump-device-code= %s 2>&1 \ // RUN: | FileCheck %s --check-prefixes=CHECK-PTX-FILES-CWD,CHECK-SPIRV-FILES-CWD -// CHECK-PTX-FILES: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}clang-{{[0-9]+}}" {{.*}} "-fsycl-is-device" {{.*}}.s{{.*}} +// CHECK-PTX-FILES: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}clang{{.*}}" {{.*}} "-fsycl-is-device" {{.*}}.s{{.*}} // CHECK-SPIRV-FILES: llvm-foreach{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv" -// CHECK-PTX-FILES-CWD: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}clang-{{[0-9]+}}" {{.*}} "-fsycl-is-device" +// CHECK-PTX-FILES-CWD: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}clang{{.*}}" {{.*}} "-fsycl-is-device" // CHECK-SPIRV-FILES-CWD: llvm-foreach{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv" // Windows - Check if PTX files are saved in the user provided path. @@ -38,6 +38,6 @@ // RUN: -fsycl-dump-device-code= %s 2>&1 \ // RUN: | FileCheck -check-prefix=CHECK-PTX-WIN-CWD %s -// CHECK-PTX-WIN: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}clang-{{[0-9]+}}" {{.*}} "-fsycl-is-device" {{.*}}.asm{{.*}} -// CHECK-PTX-WIN-CWD: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}clang-{{[0-9]+}}" {{.*}} "-fsycl-is-device" {{.*}}.asm{{.*}} +// CHECK-PTX-WIN: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}clang{{.*}}" {{.*}} "-fsycl-is-device" {{.*}}.asm{{.*}} +// CHECK-PTX-WIN-CWD: llvm-foreach{{.*}} "--out-ext=s"{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}clang{{.*}}" {{.*}} "-fsycl-is-device" {{.*}}.asm{{.*}} // CHECK-SPV-WIN: llvm-foreach{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv"