Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Enabled device_type AUTO:GPU.1,CPU in OVEP #389

Open
wants to merge 2 commits into
base: ovep-develop-lnl-1.1
Choose a base branch
from
Open

Enabled device_type AUTO:GPU.1,CPU in OVEP #389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2915a0a
update: Enabled OV devices update parsing on runtime
ankitm3k Jul 12, 2024
8aabe31
Merge branch 'ovep-develop-lnl-1.1' into ankit/enable_ovep_device_type
ankitm3k Jul 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion onnxruntime/core/providers/openvino/openvino_execution_provider.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv
info.device_type_.find("AUTO") != std::string::npos) {
device_found = true;
} else {
for (std::string device : available_devices) {
for (const std::string& device : available_devices) {
if (device.rfind(info.device_type_, 0) == 0) {
if (info.device_type_.find("GPU") != std::string::npos && (info.precision_ == "FP32" ||
info.precision_ == "FP16" ||
Expand Down
44 changes: 35 additions & 9 deletions onnxruntime/core/providers/openvino/openvino_execution_provider.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,23 @@

namespace onnxruntime {

struct OVDevices {
ov::Core core;
std::vector<std::string> get_ov_devices() const {
return core.get_available_devices();
}
};

static void print_build_options() {
std::cout << "[ERROR] INVALID DEVICE BUILD TYPE SPECIFIED" << std::endl;
std::cout << "Specify the keyword HETERO (or) MULTI (or) AUTO followed by the devices in the order of priority "
<< "you want to build"
<< std::endl;
std::cout << "The different hardware devices that can be added with HETERO/MULTI/AUTO build "
<< "are ['CPU','GPU','NPU']"
<< "are ['CPU','GPU','NPU','GPU.x'] where x = 0,1,2 and so on"
<< std::endl;
std::cout << "An example of how to specify the HETERO or MULTI or AUTO build type. "
<< "Ex: HETERO:GPU,CPU Ex: MULTI:GPU,CPU Ex: AUTO:GPU,CPU"
<< "Ex: HETERO:GPU,CPU Ex: MULTI:GPU,CPU Ex: AUTO:GPU,CPU Ex: AUTO:GPU.0,CPU Ex: AUTO:GPU.1,CPU"
<< std::endl;
}

Expand All @@ -40,7 +47,8 @@ static std::vector<std::string> split(const std::string& s, char delim) {
return result;
}

static std::vector<std::string> parseDevices(const std::string& device_string) {
static std::vector<std::string> parseDevices(const std::string& device_string,
const std::vector<std::string>& available_devices) {
std::string comma_separated_devices = device_string;
if (comma_separated_devices.find(":") != std::string::npos) {
comma_separated_devices = comma_separated_devices.substr(comma_separated_devices.find(":") + 1);
Expand All @@ -50,8 +58,15 @@ static std::vector<std::string> parseDevices(const std::string& device_string) {
print_build_options();
ORT_THROW("Invalid device string: " + device_string);
}
std::vector<std::string> dev_options = {"CPU", "GPU", "NPU"};
for (std::string dev : devices) {
std::set<std::string> dev_options = {"CPU", "GPU", "NPU"};

for (auto& device : available_devices) {
if (dev_options.find(device) == dev_options.end()) {
auto dev_options_update = dev_options.emplace(device);
}
}

for (const std::string& dev : devices) {
if (!std::count(dev_options.begin(), dev_options.end(), dev)) {
print_build_options();
ORT_THROW("Invalid device string: " + device_string);
Expand Down Expand Up @@ -79,8 +94,9 @@ struct OpenVINOExecutionProviderInfo {

OpenVINOExecutionProviderInfo() = delete;

explicit OpenVINOExecutionProviderInfo(std::string dev_type, std::string precision, bool enable_npu_fast_compile,
size_t num_of_threads, std::string cache_dir, std::string model_priority,
explicit OpenVINOExecutionProviderInfo(const std::string& dev_type, const std::string& precision,
bool enable_npu_fast_compile, size_t num_of_threads,
const std::string& cache_dir, const std::string& model_priority,
int num_streams, void* context, bool enable_opencl_throttling,
bool disable_dynamic_shapes, bool export_ep_ctx_blob,
bool enable_qdq_optimizer, bool disable_cpu_fallback,
Expand All @@ -100,6 +116,16 @@ struct OpenVINOExecutionProviderInfo {
so_epctx_embed_mode_{so_epctx_embed_mode} {
std::set<std::string> ov_supported_device_types = {"CPU", "GPU",
"GPU.0", "GPU.1", "NPU"};

OVDevices devices;
std::vector<std::string> available_devices = devices.get_ov_devices();

for (auto& device : available_devices) {
if (ov_supported_device_types.find(device) == ov_supported_device_types.end()) {
auto ov_supported_device_types_update = ov_supported_device_types.emplace(device);
}
}

if (dev_type == "") {
LOGS_DEFAULT(INFO) << "[OpenVINO-EP]"
<< "No runtime device selection option provided.";
Expand All @@ -119,7 +145,7 @@ struct OpenVINOExecutionProviderInfo {
dev_type = DEVICE;

if (dev_type.find("HETERO") == 0 || dev_type.find("MULTI") == 0 || dev_type.find("AUTO") == 0) {
std::vector<std::string> devices = parseDevices(dev_type);
std::vector<std::string> devices = parseDevices(dev_type, available_devices);
precision_ = "FP16";
if (devices[0] == "CPU") {
precision_ = "FP32";
Expand All @@ -130,7 +156,7 @@ struct OpenVINOExecutionProviderInfo {
} else if (ov_supported_device_types.find(dev_type) != ov_supported_device_types.end()) {
device_type_ = std::move(dev_type);
} else if (dev_type.find("HETERO") == 0 || dev_type.find("MULTI") == 0 || dev_type.find("AUTO") == 0) {
std::vector<std::string> devices = parseDevices(dev_type);
std::vector<std::string> devices = parseDevices(dev_type, available_devices);
device_type_ = std::move(dev_type);
} else {
ORT_THROW("Invalid device string: " + dev_type);
Expand Down
12 changes: 10 additions & 2 deletions onnxruntime/core/providers/openvino/openvino_provider_factory.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,14 @@ struct OpenVINO_Provider : Provider {
std::set<std::string> deprecated_device_types = {"CPU_FP32", "GPU_FP32",
"GPU.0_FP32", "GPU.1_FP32", "GPU_FP16",
"GPU.0_FP16", "GPU.1_FP16"};
OVDevices devices;
std::vector<std::string> available_devices = devices.get_ov_devices();

for (auto& device : available_devices) {
if (ov_supported_device_types.find(device) == ov_supported_device_types.end()) {
auto ov_supported_device_types_update = ov_supported_device_types.emplace(device);
}
}
if (deprecated_device_types.find(device_type) != deprecated_device_types.end()) {
std::string deprecated_device = device_type;
int delimit = device_type.find("_");
Expand All @@ -135,8 +143,8 @@ struct OpenVINO_Provider : Provider {
(device_type.find("MULTI:") == 0) ||
(device_type.find("AUTO:") == 0))) {
ORT_THROW(
"[ERROR] [OpenVINO] You have selcted wrong configuration value for the key 'device_type'. "
"Select from 'CPU', 'GPU', 'GPU.0', 'GPU.1', 'NPU' or from"
"[ERROR] [OpenVINO] You have selected wrong configuration value for the key 'device_type'. "
"Select from 'CPU', 'GPU', 'NPU', 'GPU.x' where x = 0,1,2 and so on or from"
" HETERO/MULTI/AUTO options available. \n");
}
}
Expand Down
147 changes: 147 additions & 0 deletions onnxruntime/test/perftest/ort_test_session.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,153 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
session_options.AppendExecutionProvider_CUDA(cuda_options);
#else
ORT_THROW("TensorRT is not supported in this build\n");
#endif
} else if (provider_name_ == onnxruntime::kOpenVINOExecutionProvider) {
#ifdef USE_OPENVINO
#ifdef _MSC_VER
std::string ov_string = ToUTF8String(performance_test_config.run_config.ep_runtime_config_string);
#else
std::string ov_string = performance_test_config.run_config.ep_runtime_config_string;
#endif
std::unordered_map<std::string, std::string> ov_options;
std::istringstream ss(ov_string);
std::string token;
while (ss >> token) {
if (token == "") {
continue;
}
auto pos = token.find("|");
if (pos == std::string::npos || pos == 0 || pos == token.length()) {
ORT_THROW("[ERROR] [OpenVINO] Use a '|' to separate the key and value for the run-time option"
" you are trying to use.\n");
}

auto key = token.substr(0, pos);
auto value = token.substr(pos + 1);

if (key == "device_type") {
std::set<std::string> ov_supported_device_types = {"CPU", "GPU",
"GPU.0", "GPU.1", "NPU"};
std::set<std::string> deprecated_device_types = {"CPU_FP32", "GPU_FP32",
"GPU.0_FP32", "GPU.1_FP32", "GPU_FP16",
"GPU.0_FP16", "GPU.1_FP16"};
size_t num_gpus = 10;
for (size_t i = 0; i <= num_gpus; i++) {
ov_supported_device_types.emplace("GPU." + std::to_string(i));
}
if (ov_supported_device_types.find(value) != ov_supported_device_types.end()) {
ov_options[key] = value;
} else if (deprecated_device_types.find(value) != deprecated_device_types.end()) {
ov_options[key] = value;
} else if (value.find("HETERO:") == 0) {
ov_options[key] = value;
} else if (value.find("MULTI:") == 0) {
ov_options[key] = value;
} else if (value.find("AUTO:") == 0) {
ov_options[key] = value;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] You have selected wrong configuration value for the key 'device_type'. "
"Select from 'CPU', 'GPU', 'NPU', 'GPU.x' where x = 0,1,2 and so on or from"
" HETERO/MULTI/AUTO options available. \n");
}
} else if (key == "device_id") {
if (value == "CPU" || value == "GPU" || value == "NPU") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported device_id is selected. Select from available options.");
}
} else if (key == "precision") {
auto device_type = ov_options["device_type"];
if (device_type.find("GPU") != std::string::npos) {
if (value == "") {
ov_options[key] = "FP16";
continue;
} else if (value == "ACCURACY" || value == "FP16" || value == "FP32") {
ov_options[key] = value;
continue;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] Unsupported inference precision is selected. "
"GPU only supported FP32 / FP16. \n");
}
} else if (device_type.find("NPU") != std::string::npos) {
if (value == "" || value == "ACCURACY" || value == "FP16") {
ov_options[key] = "FP16";
continue;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. NPU only supported FP16. \n");
}
} else if (device_type.find("CPU") != std::string::npos) {
if (value == "" || value == "ACCURACY" || value == "FP32") {
ov_options[key] = "FP32";
continue;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. CPU only supports FP32 . \n");
}
}
} else if (key == "enable_npu_fast_compile") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_npu_fast_compile' should be a boolean"
" i.e. true or false. Default value is false.\n");
}
} else if (key == "enable_opencl_throttling") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_opencl_throttling' should be a boolean"
" i.e. true or false. Default value is false.\n");
}
} else if (key == "disable_dynamic_shapes") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] The value for the key 'enable_dynamic_shapes' "
"should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "num_of_threads") {
if (std::stoi(value) <= 0) {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'num_of_threads' should be greater than 0\n");
} else {
ov_options[key] = value;
}
} else if (key == "model_priority") {
ov_options[key] = value;
} else if (key == "cache_dir") {
ov_options[key] = value;
} else if (key == "context") {
ov_options[key] = value;
} else if (key == "num_streams") {
if (std::stoi(value) <= 0 && std::stoi(value) > 8) {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'num_streams' should be in the range of 1-8 \n");
} else {
ov_options[key] = value;
}
} else if (key == "export_ep_ctx_blob") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] The value for the key 'export_ep_ctx_blob' "
"should be a boolean i.e. true or false. Default value is false.\n");
}
} else {
ORT_THROW("[ERROR] [OpenVINO] wrong key type entered."
" Choose from the following runtime key options that are available for OpenVINO."
" ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir',"
" 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n");
}
}
session_options.AppendExecutionProvider_OpenVINO_V2(ov_options);
#else
ORT_THROW("OpenVINO is not supported in this build\n");
#endif
} else if (provider_name_ == onnxruntime::kQnnExecutionProvider) {
#ifdef USE_QNN
Expand Down
Loading