From 24ad4ff2cf6b65c49e7b6e319543a0adbfb026aa Mon Sep 17 00:00:00 2001 From: tomdol Date: Wed, 31 Jul 2019 15:23:32 +0200 Subject: [PATCH 01/14] Update nGraph to 0.21 and adjust the EP --- cmake/external/ngraph.cmake | 2 +- .../ngraph/ngraph_execution_provider.cc | 32 ++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/cmake/external/ngraph.cmake b/cmake/external/ngraph.cmake index 65b7159e34bee..f993f1d08e03a 100644 --- a/cmake/external/ngraph.cmake +++ b/cmake/external/ngraph.cmake @@ -11,7 +11,7 @@ set(ngraph_SRC ${CMAKE_CURRENT_BINARY_DIR}/ngraph/src/project_ngraph) set(prebuilt_ONNX_SOURCE_DIR "${PROJECT_SOURCE_DIR}/external/onnx") set(prebuilt_ONNX_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/onnx") set(ngraph_URL "https://github.com/NervanaSystems/ngraph.git") -set(ngraph_TAG "v0.18.1") +set(ngraph_TAG "v0.21.0") # Libraries for python package. if (WIN32) diff --git a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc index 7749a21532430..f280938ab74da 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc +++ b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc @@ -182,6 +182,36 @@ static bool IsUnsupportedOpMode(const Node* node, const onnxruntime::GraphViewer if (ceil_attr != attributes.end() && ceil_attr->second.i() != 0) { return true; } + } else if (optype == "Gather") { + // disable Gather for non-floating point types + bool is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos; + is_float = is_float || node->InputDefs()[0]->Type()->find("double") != std::string::npos; + return !is_float; + } else if (optype == "Split") { + const auto& attributes = node->GetAttributes(); + const auto split_attr = attributes.find("split"); + + if (split_attr != attributes.end()) { + // split implementation contains a bug that doesn't throw for incorrect split values + // disabling temporarily until it's fixed in the next release of nGraph + const auto splits = split_attr->second.ints(); + return std::any_of(std::begin(splits), std::end(splits), + [](const auto split) { return split <= 0; }); + } + } else if (optype == "QLinearMatMul") { + const auto& a_zero_point = node->InputDefs()[2]; + const auto& b_zero_point = node->InputDefs()[5]; + const auto& y_zero_point = node->InputDefs()[7]; + + bool non_const_zero_point = false; + + // check if any of the zero points is NOT in the initializers list + non_const_zero_point |= initializers.find(a_zero_point->Name()) == initializers.end(); + non_const_zero_point |= initializers.find(b_zero_point->Name()) == initializers.end(); + non_const_zero_point |= initializers.find(y_zero_point->Name()) == initializers.end(); + + // QLinearMatMul is not supported if any of the zero points is a dynamic input + return non_const_zero_point; } //Op doesn't fall into known any of unsupported modes. @@ -292,7 +322,7 @@ static std::map> GetNgSupportedOps(const int std::map> ng_supported_ops; ng_supported_ops.emplace(kOnnxDomain, ngraph::onnx_import::get_supported_operators(onnx_opset, kOnnxDomain)); - const std::set ng_disabled_ops = {}; //Place-holder for ops not supported. + const std::set ng_disabled_ops = {"LSTM"}; //Place-holder for ops not supported. for (const auto& disabled_op : ng_disabled_ops) { ng_supported_ops.at(kOnnxDomain).erase(disabled_op); From a7534e456d05a0c10bed71d70bffb1d4b84cda48 Mon Sep 17 00:00:00 2001 From: tomdol Date: Thu, 1 Aug 2019 14:43:33 +0200 Subject: [PATCH 02/14] Share the graph initializers between custom ops --- .../core/providers/ngraph/ngraph_custom_op.cc | 47 ++++++++++------ .../core/providers/ngraph/ngraph_custom_op.h | 5 +- .../ngraph/ngraph_execution_provider.cc | 55 +++++++++++-------- .../ngraph/ngraph_execution_provider.h | 6 ++ 4 files changed, 70 insertions(+), 43 deletions(-) diff --git a/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc b/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc index 5ac6354bbb11b..8579ba6919ed2 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc +++ b/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc @@ -35,15 +35,23 @@ static bool check_ngraph_dump_ops() { #endif } -NGRAPHCustomOp::NGRAPHCustomOp(const ComputeContext* context, const ONNX_NAMESPACE::ModelProto& model_proto, - const std::shared_ptr& ng_backend) - : ng_backend_{ng_backend}, - model_proto_{model_proto} { +NGRAPHCustomOp::NGRAPHCustomOp(const ComputeContext* context, + const ONNX_NAMESPACE::ModelProto& model_proto, + const std::shared_ptr initializers, + const std::shared_ptr& ng_backend) : + ng_backend_{ng_backend}, model_proto_{model_proto} +{ allocate_func_ = context->allocate_func; release_func_ = context->release_func; allocator_ = context->allocator_handle; name_ = context->node_name; + auto graph_proto = model_proto_.mutable_graph(); + + for (const auto& initializer : initializers->tensors()) { + graph_proto->add_initializer()->CopyFrom(initializer); + } + if (check_ngraph_dump_ops()) { std::fstream dump(name_ + ".onnx", std::ios::out | std::ios::trunc | std::ios::binary); model_proto_.SerializeToOstream(&dump); @@ -59,7 +67,6 @@ NGRAPHCustomOp::~NGRAPHCustomOp() { //This method gets called in critical path of execution: Optimize void NGRAPHCustomOp::Initialize(const OrtCustomOpApi* api, OrtKernelContext* context) const { Ort::CustomOpApi ort{*api}; - LOGS_DEFAULT(INFO) << "nGraph compiling customOp: " << name_; size_t num_inputs = ort.KernelContext_GetInputCount(context); @@ -88,6 +95,9 @@ void NGRAPHCustomOp::Initialize(const OrtCustomOpApi* api, OrtKernelContext* con return; } else { auto graph_proto = model_proto_.mutable_graph(); + + LOGS_DEFAULT(INFO) << "[NGRAPHCustomOp] Compiling customOp: " << name_; + // Clear previous shapes if any and set new input shapes for (size_t i = 0; i < num_inputs; i++) { auto g_in_shape = graph_proto->mutable_input((int)i)->mutable_type()->mutable_tensor_type()->mutable_shape(); @@ -108,12 +118,12 @@ void NGRAPHCustomOp::Initialize(const OrtCustomOpApi* api, OrtKernelContext* con try { ng_function = ngraph::onnx_import::import_onnx_model(model_stream); } catch (const std::exception& exp) { - LOGS_DEFAULT(FATAL) << "[" << name_ << "] " - << "Exception while converting onnx to nGraph: " << std::string(exp.what()); + LOGS_DEFAULT(FATAL) << "[NGRAPHCustomOp] " << " - " << name_ << " - " + << "Exception while importing model to nGraph: " << std::string(exp.what()); throw; } catch (...) { - LOGS_DEFAULT(FATAL) << "[" << name_ << "] " - << "Unknown exception while converting onnx to nGraph"; + LOGS_DEFAULT(FATAL) << "[NGRAPHCustomOp] " << " - " << name_ << " - " + << "Unknown exception while importing model to nGraph"; throw; } @@ -125,9 +135,10 @@ void NGRAPHCustomOp::Initialize(const OrtCustomOpApi* api, OrtKernelContext* con try { ng_curr_exe_ = ng_backend_->compile(ng_function); } catch (const std::exception& exp) { - LOGS_DEFAULT(FATAL) << "Exception while compiling nGraph Op: " << name_ << std::string(exp.what()); + LOGS_DEFAULT(FATAL) << "[NGRAPHCustomOp] " << " - " << name_ << " - " + << "Exception while compiling ngraph::Function: " << std::string(exp.what()); } catch (...) { - LOGS_DEFAULT(FATAL) << "Unknown exception while compiling nGraph Op: " << name_; + LOGS_DEFAULT(FATAL) << "[NGRAPHCustomOp] " << " - " << name_ << " - " << "Unknown exception while compiling ngraph::Function"; } it.first->second = ng_curr_exe_; } @@ -157,9 +168,9 @@ Status NGRAPHCustomOp::Compute(const OrtCustomOpApi* api, OrtKernelContext* cont ng_inputs.emplace_back(ng_backend_->create_tensor(ng_param->get_output_element_type(0), ng_param->get_output_shape(0), input_data)); } } catch (const std::exception& exp) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Exception while copying input data to nGraph: " + std::string(exp.what())); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, name_ + ": Exception while copying input data to nGraph: " + std::string(exp.what())); } catch (...) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unknown exception while copying input data to nGraph"); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, name_ + ": Unknown exception while copying input data to nGraph"); } // Initialize output tensors @@ -176,19 +187,19 @@ Status NGRAPHCustomOp::Compute(const OrtCustomOpApi* api, OrtKernelContext* cont ng_outputs.emplace_back(ng_backend_->create_tensor(dtype, shape, output_data)); } } catch (const std::exception& exp) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Exception while creating nGraph output Tensor: " + std::string(exp.what())); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, name_ + ": Exception while creating nGraph output Tensor: " + std::string(exp.what())); } catch (...) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unknown exception while creating nGraph output Tensor"); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, name_ + ": Unknown exception while creating nGraph output Tensor"); } // Run the graph through nGraph. try { if (!ng_curr_exe_->call(ng_outputs, ng_inputs)) - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Error while executing nGraph computation"); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, name_ + ": Error while executing nGraph computation"); } catch (const std::exception& exp) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Exception while executing nGraph computation: " + std::string(exp.what())); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, name_ + ": Exception while executing nGraph computation: " + std::string(exp.what())); } catch (...) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unknown exception while executing nGraph computation"); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, name_ + ": Unknown exception while executing nGraph computation"); } return Status::OK(); diff --git a/onnxruntime/core/providers/ngraph/ngraph_custom_op.h b/onnxruntime/core/providers/ngraph/ngraph_custom_op.h index 6661fdb378e56..55609a5d45559 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_custom_op.h +++ b/onnxruntime/core/providers/ngraph/ngraph_custom_op.h @@ -25,7 +25,10 @@ namespace ngraph_ep { class NGRAPHCustomOp { public: - NGRAPHCustomOp(const ComputeContext* context, const ONNX_NAMESPACE::ModelProto& model_proto, const std::shared_ptr& ng_backend); + NGRAPHCustomOp(const ComputeContext* context, + const ONNX_NAMESPACE::ModelProto& model_proto, + const std::shared_ptr initializers, + const std::shared_ptr& ng_backend); Status Compute(const OrtCustomOpApi* api, OrtKernelContext* context) const; diff --git a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc index 7749a21532430..1cfdec9ab0757 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc +++ b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc @@ -255,21 +255,10 @@ static void AppendClusterToSubGraph(const std::vector& nodes, const onnxruntime::GraphViewer& graph_viewer, const std::vector& inputs, const std::vector& outputs, - const std::unordered_set& ng_required_initializers, std::vector>& result) { static size_t op_counter = 0; - // Create ng_required_initializers attribute of NGraphCustomOp - ONNX_NAMESPACE::AttributeProto initializers; - initializers.set_name("initializers"); - initializers.set_type(ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_TENSORS); - for (const auto& init : ng_required_initializers) { - auto tensor = initializers.add_tensors(); - *tensor = *(graph_viewer.GetAllInitializedTensors().at(init)); - } - auto meta_def = std::make_unique(); - meta_def->attributes["initializers"] = initializers; meta_def->name = "NGRAPHCustomOp_" + std::to_string(++op_counter); meta_def->domain = kNGraphDomain; meta_def->since_version = 1; @@ -277,6 +266,13 @@ static void AppendClusterToSubGraph(const std::vector& nodes, meta_def->inputs = inputs; meta_def->outputs = outputs; + //store the name of the graph this node belongs to - used to retrieve graph initializers from the cache + ONNX_NAMESPACE::AttributeProto graph_name; + graph_name.set_name("graph_name"); + graph_name.set_type(ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_STRING); + graph_name.set_s(graph_viewer.Name()); + meta_def->attributes["graph_name"] = graph_name; + std::unique_ptr sub_graph = std::make_unique(); sub_graph->nodes = nodes; sub_graph->SetMetaDef(meta_def); @@ -451,6 +447,19 @@ NGRAPHExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie const auto unsupported_nodes = GetUnsupportedNodeIndices(graph_viewer, ng_required_initializers); + // Create a shared_ptr with this graph's initializers which are required to construct nGraph's IR + if (graph_initializers_.count(graph_viewer.Name()) == 0) { + graph_initializers_[graph_viewer.Name()] = std::make_shared(); + + auto& initializers = graph_initializers_[graph_viewer.Name()]; + initializers->set_name("initializers"); + initializers->set_type(ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_TENSORS); + for (const auto& init : ng_required_initializers) { + auto tensor = initializers->add_tensors(); + *tensor = *(graph_viewer.GetAllInitializedTensors().at(init)); + } + } + //If all ops are supported, no partitioning is required. Short-circuit and avoid splitting. if (unsupported_nodes.empty()) { std::vector inputs; @@ -475,7 +484,7 @@ NGRAPHExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie [&outputs](const NodeArg* node_arg) { outputs.push_back(node_arg->Name()); }); // Create and add this graph to result. - AppendClusterToSubGraph(graph_viewer.GetNodesInTopologicalOrder(), graph_viewer, inputs, outputs, ng_required_initializers, result); + AppendClusterToSubGraph(graph_viewer.GetNodesInTopologicalOrder(), graph_viewer, inputs, outputs, result); } else { // unsupported_nodes_idx.empty() const auto ng_clusters = GetPartitionedClusters(graph_viewer.GetNodesInTopologicalOrder(), unsupported_nodes); @@ -485,7 +494,7 @@ NGRAPHExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie GetInputsOutputsOfCluster(graph_viewer, this_cluster, ng_required_initializers, cluster_inputs, cluster_outputs); if (!cluster_inputs.empty()) { - AppendClusterToSubGraph(this_cluster, graph_viewer, cluster_inputs, cluster_outputs, ng_required_initializers, result); + AppendClusterToSubGraph(this_cluster, graph_viewer, cluster_inputs, cluster_outputs, result); } } } @@ -494,9 +503,6 @@ NGRAPHExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie } static ONNX_NAMESPACE::ModelProto GetModelProtoFromFusedNode(const onnxruntime::Node* fused_node) { - const auto& attributes = fused_node->GetAttributes(); - const auto& initializers = attributes.at("initializers").tensors(); - ONNX_NAMESPACE::ModelProto model_proto; auto graph_proto = model_proto.mutable_graph(); const auto& fused_graph = fused_node->GetFunctionBody()->Body(); @@ -515,10 +521,6 @@ static ONNX_NAMESPACE::ModelProto GetModelProtoFromFusedNode(const onnxruntime:: *valueInfoProto = output->ToProto(); } - for (const auto& initializer : initializers) { - graph_proto->add_initializer()->CopyFrom(initializer); - } - auto opset = model_proto.add_opset_import(); opset->set_domain(kOnnxDomain); opset->set_version(fused_graph.DomainToVersionMap().at(kOnnxDomain)); @@ -530,14 +532,19 @@ static ONNX_NAMESPACE::ModelProto GetModelProtoFromFusedNode(const onnxruntime:: Status NGRAPHExecutionProvider::Compile(const std::vector& fused_nodes, std::vector& node_compute_funcs) { for (const auto& fused_node : fused_nodes) { - auto model_proto = GetModelProtoFromFusedNode(fused_node); - NodeComputeInfo compute_info; + const auto& attributes = fused_node->GetAttributes(); + const auto graph_name = attributes.at("graph_name").s(); + // find the shared_ptr with initializes of a graph this fused_node belongs to + auto initializers = graph_initializers_.at(graph_name); + // Local copy of backend since, class members cannot be captured. auto ngraph_backend = ng_backend_; - compute_info.create_state_func = [model_proto, ngraph_backend](ComputeContext* context, FunctionState* state) { - auto* p = new onnxruntime::ngraph_ep::NGRAPHCustomOp(context, model_proto, ngraph_backend); + compute_info.create_state_func = [model_proto = GetModelProtoFromFusedNode(fused_node), ngraph_backend, initializers] + (ComputeContext* context, FunctionState* state) + { + auto* p = new onnxruntime::ngraph_ep::NGRAPHCustomOp(context, model_proto, initializers, ngraph_backend); *state = p; return 0; }; diff --git a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h index a2af812e31e06..c1a09f33c0b4c 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h +++ b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h @@ -4,6 +4,7 @@ #pragma once #include "core/framework/execution_provider.h" +#include namespace ngraph { namespace runtime { @@ -11,6 +12,10 @@ class Backend; } } // namespace ngraph +namespace ONNX_NAMESPACE { + class AttributeProto; +} + namespace onnxruntime { // Information needed to construct nGraph execution providers. @@ -35,6 +40,7 @@ class NGRAPHExecutionProvider : public IExecutionProvider { private: std::shared_ptr ng_backend_; + mutable std::unordered_map> graph_initializers_; }; } // namespace onnxruntime From 7ff787c132adb312b4b7cfbf094ef6a4f02664d5 Mon Sep 17 00:00:00 2001 From: tomdol Date: Fri, 2 Aug 2019 11:53:03 +0200 Subject: [PATCH 03/14] Update nGraph to 0.22 and exclude Gather entirely --- cmake/external/ngraph.cmake | 2 +- .../core/providers/ngraph/ngraph_execution_provider.cc | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/cmake/external/ngraph.cmake b/cmake/external/ngraph.cmake index f993f1d08e03a..2c6b10c75e15e 100644 --- a/cmake/external/ngraph.cmake +++ b/cmake/external/ngraph.cmake @@ -11,7 +11,7 @@ set(ngraph_SRC ${CMAKE_CURRENT_BINARY_DIR}/ngraph/src/project_ngraph) set(prebuilt_ONNX_SOURCE_DIR "${PROJECT_SOURCE_DIR}/external/onnx") set(prebuilt_ONNX_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/onnx") set(ngraph_URL "https://github.com/NervanaSystems/ngraph.git") -set(ngraph_TAG "v0.21.0") +set(ngraph_TAG "v0.22.0") # Libraries for python package. if (WIN32) diff --git a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc index f280938ab74da..bf1b04ec6fb5d 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc +++ b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc @@ -182,11 +182,6 @@ static bool IsUnsupportedOpMode(const Node* node, const onnxruntime::GraphViewer if (ceil_attr != attributes.end() && ceil_attr->second.i() != 0) { return true; } - } else if (optype == "Gather") { - // disable Gather for non-floating point types - bool is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos; - is_float = is_float || node->InputDefs()[0]->Type()->find("double") != std::string::npos; - return !is_float; } else if (optype == "Split") { const auto& attributes = node->GetAttributes(); const auto split_attr = attributes.find("split"); @@ -322,7 +317,7 @@ static std::map> GetNgSupportedOps(const int std::map> ng_supported_ops; ng_supported_ops.emplace(kOnnxDomain, ngraph::onnx_import::get_supported_operators(onnx_opset, kOnnxDomain)); - const std::set ng_disabled_ops = {"LSTM"}; //Place-holder for ops not supported. + const std::set ng_disabled_ops = {"LSTM", "Gather"}; //Place-holder for ops not supported. for (const auto& disabled_op : ng_disabled_ops) { ng_supported_ops.at(kOnnxDomain).erase(disabled_op); From c1b12e02e573e5d2411a70974f290fae34adc3ae Mon Sep 17 00:00:00 2001 From: tomdol Date: Sat, 3 Aug 2019 11:02:29 +0200 Subject: [PATCH 04/14] Enable building on Windows with nGraph v0.21.1-rc.0 --- cmake/external/ngraph.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/ngraph.cmake b/cmake/external/ngraph.cmake index 2c6b10c75e15e..228151372a8b1 100644 --- a/cmake/external/ngraph.cmake +++ b/cmake/external/ngraph.cmake @@ -11,7 +11,7 @@ set(ngraph_SRC ${CMAKE_CURRENT_BINARY_DIR}/ngraph/src/project_ngraph) set(prebuilt_ONNX_SOURCE_DIR "${PROJECT_SOURCE_DIR}/external/onnx") set(prebuilt_ONNX_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/onnx") set(ngraph_URL "https://github.com/NervanaSystems/ngraph.git") -set(ngraph_TAG "v0.22.0") +set(ngraph_TAG "v0.22.1-rc.0") # Libraries for python package. if (WIN32) From 257b42a55bdd98f804d4846868542b8e3aeb4b4e Mon Sep 17 00:00:00 2001 From: tomdol Date: Sat, 3 Aug 2019 11:49:10 +0200 Subject: [PATCH 05/14] Disable the unsigned input Shrink op tests for nGraph until the next update --- onnxruntime/test/providers/cpu/nn/shrink_test.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/onnxruntime/test/providers/cpu/nn/shrink_test.cc b/onnxruntime/test/providers/cpu/nn/shrink_test.cc index 03bf0eeb159b1..d4f96530c683e 100644 --- a/onnxruntime/test/providers/cpu/nn/shrink_test.cc +++ b/onnxruntime/test/providers/cpu/nn/shrink_test.cc @@ -68,7 +68,7 @@ std::vector> GenerateUnsignedTestCases() { } template -void RunShrinkTest(const std::vector>& test_cases) { +void RunShrinkTest(const std::vector>& test_cases, const std::unordered_set& excluded_provider_types = {}) { for (const auto& test_data : test_cases) { OpTester test("Shrink", 9); @@ -82,7 +82,7 @@ void RunShrinkTest(const std::vector>& test_cases) { test.AddInput("X", test_data.input_dimensions, test_data.input_vals); test.AddOutput("Y", test_data.expected_dimensions, test_data.expected_vals); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, {}, excluded_provider_types); } } @@ -101,7 +101,7 @@ TEST(MathOpTest, ShrinkInt8Type) { TEST(MathOpTest, ShrinkUint8Type) { const auto& test_cases = GenerateUnsignedTestCases(); - RunShrinkTest(test_cases); + RunShrinkTest(test_cases, {kNGraphExecutionProvider}); } TEST(MathOpTest, ShrinkInt16Type) { @@ -111,7 +111,7 @@ TEST(MathOpTest, ShrinkInt16Type) { TEST(MathOpTest, ShrinkUint16Type) { const auto& test_cases = GenerateUnsignedTestCases(); - RunShrinkTest(test_cases); + RunShrinkTest(test_cases, {kNGraphExecutionProvider}); } TEST(MathOpTest, ShrinkInt32Type) { @@ -121,7 +121,7 @@ TEST(MathOpTest, ShrinkInt32Type) { TEST(MathOpTest, ShrinkUint32Type) { const auto& test_cases = GenerateUnsignedTestCases(); - RunShrinkTest(test_cases); + RunShrinkTest(test_cases, {kNGraphExecutionProvider}); } TEST(MathOpTest, ShrinkInt64Type) { @@ -131,7 +131,7 @@ TEST(MathOpTest, ShrinkInt64Type) { TEST(MathOpTest, ShrinkUint64Type) { const auto& test_cases = GenerateUnsignedTestCases(); - RunShrinkTest(test_cases); + RunShrinkTest(test_cases, {kNGraphExecutionProvider}); } TEST(MathOpTest, ShrinkFloatType) { From 94cc391e6920e005a56ea832434841abf922261d Mon Sep 17 00:00:00 2001 From: tomdol Date: Sun, 4 Aug 2019 12:34:59 +0200 Subject: [PATCH 06/14] Line-shortening code refactor --- .../ngraph/ngraph_execution_provider.cc | 45 +++++++++++++------ .../ngraph/ngraph_execution_provider.h | 8 ++-- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc index a440937e3821a..d1e5c1c011405 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc +++ b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc @@ -33,19 +33,35 @@ constexpr const char* NGRAPH = "nGraph"; NGRAPHExecutionProvider::NGRAPHExecutionProvider(const NGRAPHExecutionProviderInfo& info) : IExecutionProvider{onnxruntime::kNGraphExecutionProvider} { - DeviceAllocatorRegistrationInfo default_allocator_info({OrtMemTypeDefault, - [](int) { return std::make_unique(std::make_unique(NGRAPH, OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemTypeDefault)); }, - std::numeric_limits::max()}); + + ORT_ENFORCE(info.ng_backend_type == "CPU", "nGraph Execution Provider for onnxruntime currently is only supported for CPU backend."); + + auto default_allocator_factory = [](int) { + auto allocator_info = std::make_unique(NGRAPH, OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemTypeDefault); + return std::make_unique(std::move(allocator_info)); + }; + + DeviceAllocatorRegistrationInfo default_allocator_info{ + OrtMemTypeDefault, + std::move(default_allocator_factory), + std::numeric_limits::max() + }; InsertAllocator(CreateAllocator(default_allocator_info)); - DeviceAllocatorRegistrationInfo cpu_allocator_info({OrtMemTypeCPUOutput, - [](int) { return std::make_unique(std::make_unique(NGRAPH, OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemTypeCPUOutput)); }, - std::numeric_limits::max()}); - InsertAllocator(CreateAllocator(cpu_allocator_info)); + auto cpu_allocator_factory = [](int) { + auto allocator_info = std::make_unique(NGRAPH, OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemTypeCPUOutput); + return std::make_unique(std::move(allocator_info)); + }; - ORT_ENFORCE(info.ng_backend_type == "CPU", "nGraph Execution Provider for onnxruntime currently is only supported for CPU backend."); + DeviceAllocatorRegistrationInfo cpu_allocator_info{ + OrtMemTypeCPUOutput, + std::move(cpu_allocator_factory), + std::numeric_limits::max() + }; + + InsertAllocator(CreateAllocator(cpu_allocator_info)); try { ng_backend_ = ngraph::runtime::Backend::create(info.ng_backend_type); @@ -322,7 +338,8 @@ static std::map> GetNgSupportedOps(const int return ng_supported_ops; } -static std::vector GetUnsupportedNodeIndices(const GraphViewer& graph_viewer, /*out*/ std::unordered_set& ng_required_initializers) { +static std::vector +GetUnsupportedNodeIndices(const GraphViewer& graph_viewer, /*out*/ std::unordered_set& ng_required_initializers) { const auto ng_supported_ops = GetNgSupportedOps(GetOnnxOpSet(graph_viewer)); std::vector unsupported_nodes_idx; @@ -342,10 +359,12 @@ static std::vector GetUnsupportedNodeIndices(const GraphViewer& graph return unsupported_nodes_idx; } -/* Returns a vector clusters(or node_idx). For each unsupported node, the graph is split into 3 parts. - supported_cluster + (UNsupported_node + rest_of_the_graph). This functions returns vector of all supported_clusters by nGraph -*/ -static std::vector> GetPartitionedClusters(const std::vector& topological_order, const std::vector& unsupported_nodes) { +/** + * Returns a vector clusters(or node_idx). For each unsupported node, the graph is split into 3 parts. + * supported_cluster + (UNsupported_node + rest_of_the_graph). This functions returns vector of all supported_clusters by nGraph + */ +static std::vector> +GetPartitionedClusters(const std::vector& topological_order, const std::vector& unsupported_nodes) { std::vector> ng_clusters; auto prev = topological_order.begin(); diff --git a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h index c1a09f33c0b4c..2a2aa9d0c3cc2 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h +++ b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h @@ -7,10 +7,10 @@ #include namespace ngraph { -namespace runtime { -class Backend; + namespace runtime { + class Backend; + } } -} // namespace ngraph namespace ONNX_NAMESPACE { class AttributeProto; @@ -43,4 +43,4 @@ class NGRAPHExecutionProvider : public IExecutionProvider { mutable std::unordered_map> graph_initializers_; }; -} // namespace onnxruntime +} From 8a004fb57dc418d69e5620061242478e9f62d71b Mon Sep 17 00:00:00 2001 From: tomdol Date: Mon, 5 Aug 2019 12:49:58 +0200 Subject: [PATCH 07/14] Fix for the master branch merge artifact --- .../ngraph/ngraph_execution_provider.cc | 24 +++---------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc index 7cc0eed776bce..a5bf9ed18b9f5 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc +++ b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc @@ -37,7 +37,7 @@ NGRAPHExecutionProvider::NGRAPHExecutionProvider(const NGRAPHExecutionProviderIn ORT_ENFORCE(info.ng_backend_type == "CPU", "nGraph Execution Provider for onnxruntime currently is only supported for CPU backend."); auto default_allocator_factory = [](int) { - auto allocator_info = std::make_unique(NGRAPH, OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemTypeDefault); + auto allocator_info = std::make_unique(NGRAPH, OrtAllocatorType::OrtDeviceAllocator); return std::make_unique(std::move(allocator_info)); }; @@ -51,7 +51,8 @@ NGRAPHExecutionProvider::NGRAPHExecutionProvider(const NGRAPHExecutionProviderIn auto cpu_allocator_factory = [](int) { - auto allocator_info = std::make_unique(NGRAPH, OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemTypeCPUOutput); + auto allocator_info = std::make_unique( + NGRAPH, OrtAllocatorType::OrtDeviceAllocator, OrtDevice(), 0, OrtMemTypeCPUOutput); return std::make_unique(std::move(allocator_info)); }; @@ -73,25 +74,6 @@ NGRAPHExecutionProvider::NGRAPHExecutionProvider(const NGRAPHExecutionProviderIn } } -/** - * Checks if a tensor represented by srcLocation can be copied into the dstLocation tensor - * @param src_location result of Location().name call on the source tensor - * @param dst_location result of Location().name call on the destination tensor - * @return true if src and dest locations combination allows copying - */ -bool TensorCopyPossible(const std::string& src_location, const std::string& dst_location) { - // contains allowed combinations of source and destination locations for tensors copying purposes - // the first element of a pair denotes a source, the second - destination - static const std::map allowed_copy_directions = { - {NGRAPH, CPU}, {NGRAPH, NGRAPH}, {CPU, NGRAPH}}; - - // copying of tensors is allowed only if the params match any of the allowed combinations - return std::any_of(allowed_copy_directions.begin(), - allowed_copy_directions.end(), [&](const auto& copy_direction) { - return src_location == copy_direction.first && dst_location == copy_direction.second; - }); -} - // Returns true only if op is in a mode that is not currently supported static bool IsUnsupportedOpMode(const Node* node, const onnxruntime::GraphViewer& graph_viewer) { const auto& optype = node->OpType(); From 595472915ef16b6bb4266885bb647597bcf80585 Mon Sep 17 00:00:00 2001 From: tomdol Date: Mon, 5 Aug 2019 15:50:37 +0200 Subject: [PATCH 08/14] MKLDNN patches adjustment for Windows --- cmake/external/ngraph.cmake | 6 +- .../ngraph/ngraph_fix_install_error.patch | 127 ------------------ .../ngraph/ngraph_fix_library_path.patch | 33 ----- .../ngraph_fix_mkldnn_missing_symbol.patch | 64 +++++++++ 4 files changed, 67 insertions(+), 163 deletions(-) delete mode 100644 cmake/patches/ngraph/ngraph_fix_install_error.patch delete mode 100644 cmake/patches/ngraph/ngraph_fix_library_path.patch create mode 100644 cmake/patches/ngraph/ngraph_fix_mkldnn_missing_symbol.patch diff --git a/cmake/external/ngraph.cmake b/cmake/external/ngraph.cmake index fabae19812499..f944791afeeef 100644 --- a/cmake/external/ngraph.cmake +++ b/cmake/external/ngraph.cmake @@ -42,7 +42,7 @@ else() endif() # discard prior changes due to unblock incremental builds. -set(NGRAPH_PATCH_DISCARD_COMMAND cd ${ngraph_SRC} && git checkout -- .) +set(NGRAPH_PATCH_DISCARD_COMMAND cd ${ngraph_SRC} && git reset HEAD --hard && git clean -fx) if (MSVC) set(prebuilt_ONNX_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/onnx/${CMAKE_BUILD_TYPE}") @@ -54,12 +54,12 @@ if (MSVC) PREFIX ngraph GIT_REPOSITORY ${ngraph_URL} GIT_TAG ${ngraph_TAG} + GIT_CONFIG core.autocrlf=input PATCH_COMMAND ${NGRAPH_PATCH_DISCARD_COMMAND} COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/patches/ngraph/ngraph_onnx.cmake ${ngraph_SRC}/cmake/external_onnx.cmake COMMAND git apply --ignore-space-change --ignore-whitespace ${PROJECT_SOURCE_DIR}/patches/ngraph/ngraph_protobuf.patch - COMMAND git apply --ignore-space-change --ignore-whitespace ${PROJECT_SOURCE_DIR}/patches/ngraph/ngraph_fix_install_error.patch - COMMAND git apply --ignore-space-change --ignore-whitespace ${PROJECT_SOURCE_DIR}/patches/ngraph/ngraph_fix_library_path.patch COMMAND git apply --ignore-space-change --ignore-whitespace ${PROJECT_SOURCE_DIR}/patches/ngraph/ngraph_fix_memory.patch + COMMAND git apply --ignore-space-change --ignore-whitespace ${PROJECT_SOURCE_DIR}/patches/ngraph/ngraph_fix_mkldnn_missing_symbol.patch CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DNGRAPH_DEX_ONLY=ON diff --git a/cmake/patches/ngraph/ngraph_fix_install_error.patch b/cmake/patches/ngraph/ngraph_fix_install_error.patch deleted file mode 100644 index ddabbb7d86527..0000000000000 --- a/cmake/patches/ngraph/ngraph_fix_install_error.patch +++ /dev/null @@ -1,127 +0,0 @@ -From 280fbc003ea2794adb24d6a81d42db838a793dd9 Mon Sep 17 00:00:00 2001 -From: Sang Ik Lee -Date: Mon, 15 Apr 2019 16:11:27 -0700 -Subject: [PATCH] CMAKE_CFG_INTDIR does not work at install time. Use - CMAKE_INSTALL_CONFIG_NAME on Windows. - ---- - CMakeLists.txt | 7 ++++++- - cmake/external_mkldnn.cmake | 22 +++++++++++----------- - cmake/external_tbb.cmake | 4 ++-- - cmake/external_tbb_prebuilt.cmake | 6 +++--- - 4 files changed, 22 insertions(+), 17 deletions(-) - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 2a21ed3a3..a695e217f 100755 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -390,12 +390,17 @@ endif() - - set(NGRAPH_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/src/ngraph) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${NGRAPH_BUILD_DIR}) --set(NGRAPH_LIBRARY_OUTPUT_DIRECTORY ${NGRAPH_BUILD_DIR}/${CMAKE_CFG_INTDIR}) - if(WIN32) -+ set(NGRAPH_LIBRARY_OUTPUT_DIRECTORY ${NGRAPH_BUILD_DIR}/${CMAKE_CFG_INTDIR}) -+ set(NGRAPH_LIBRARY_INSTALLSRC_DIRECTORY ${NGRAPH_BUILD_DIR}/\${CMAKE_INSTALL_CONFIG_NAME}) - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${NGRAPH_BUILD_DIR}) - set(NGRAPH_ARCHIVE_OUTPUT_DIRECTORY ${NGRAPH_BUILD_DIR}/${CMAKE_CFG_INTDIR}) -+ set(NGRAPH_ARCHIVE_INSTALLSRC_DIRECTORY ${NGRAPH_BUILD_DIR}/\${CMAKE_INSTALL_CONFIG_NAME}) - set(CMAKE_PDB_OUTPUT_DIRECTORY ${NGRAPH_BUILD_DIR}) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${NGRAPH_BUILD_DIR}) -+else() -+ set(NGRAPH_LIBRARY_OUTPUT_DIRECTORY ${NGRAPH_BUILD_DIR}) -+ set(NGRAPH_LIBRARY_INSTALLSRC_DIRECTORY ${NGRAPH_BUILD_DIR}) - endif() - - set(EXTERNAL_INSTALL_DIR ${CMAKE_BINARY_DIR}/external) -diff --git a/cmake/external_mkldnn.cmake b/cmake/external_mkldnn.cmake -index 25445bf0b..7874aca76 100644 ---- a/cmake/external_mkldnn.cmake -+++ b/cmake/external_mkldnn.cmake -@@ -312,12 +312,12 @@ endif() - if(WIN32) - install( - FILES -- ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${MKLML_LIB} -- ${NGRAPH_ARCHIVE_OUTPUT_DIRECTORY}/${MKLML_IMPLIB} -- ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${OMP_LIB} -- ${NGRAPH_ARCHIVE_OUTPUT_DIRECTORY}/${OMP_IMPLIB} -- ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${MKLDNN_LIB} -- ${NGRAPH_ARCHIVE_OUTPUT_DIRECTORY}/${MKLDNN_IMPLIB} -+ ${NGRAPH_LIBRARY_INSTALLSRC_DIRECTORY}/${MKLML_LIB} -+ ${NGRAPH_ARCHIVE_INSTALLSRC_DIRECTORY}/${MKLML_IMPLIB} -+ ${NGRAPH_LIBRARY_INSTALLSRC_DIRECTORY}/${OMP_LIB} -+ ${NGRAPH_ARCHIVE_INSTALLSRC_DIRECTORY}/${OMP_IMPLIB} -+ ${NGRAPH_LIBRARY_INSTALLSRC_DIRECTORY}/${MKLDNN_LIB} -+ ${NGRAPH_ARCHIVE_INSTALLSRC_DIRECTORY}/${MKLDNN_IMPLIB} - DESTINATION - ${NGRAPH_INSTALL_LIB} - OPTIONAL -@@ -325,9 +325,9 @@ if(WIN32) - else() - install( - FILES -- ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${MKLML_LIB} -- ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${OMP_LIB} -- ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${MKLDNN_LIB} -+ ${NGRAPH_LIBRARY_INSTALLSRC_DIRECTORY}/${MKLML_LIB} -+ ${NGRAPH_LIBRARY_INSTALLSRC_DIRECTORY}/${OMP_LIB} -+ ${NGRAPH_LIBRARY_INSTALLSRC_DIRECTORY}/${MKLDNN_LIB} - DESTINATION - ${NGRAPH_INSTALL_LIB} - OPTIONAL -@@ -335,8 +335,8 @@ else() - if(NGRAPH_LIB_VERSIONING_ENABLE) - install( - FILES -- ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${MKLDNN_SHORT_LIB} -- ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${MKLDNN_FULL_LIB} -+ ${NGRAPH_LIBRARY_INSTALLSRC_DIRECTORY}/${MKLDNN_SHORT_LIB} -+ ${NGRAPH_LIBRARY_INSTALLSRC_DIRECTORY}/${MKLDNN_FULL_LIB} - DESTINATION - ${NGRAPH_INSTALL_LIB} - OPTIONAL -diff --git a/cmake/external_tbb.cmake b/cmake/external_tbb.cmake -index 761c5b3bd..6960ea929 100644 ---- a/cmake/external_tbb.cmake -+++ b/cmake/external_tbb.cmake -@@ -63,10 +63,10 @@ if(NGRAPH_TBB_ENABLE) - ${TBB_BUILD_DIR}/${TBB_LIB}.${TBB_SOVER} - DESTINATION ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}) - endif() -- install(FILES ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${TBB_LIB} -+ install(FILES ${NGRAPH_LIBRARY_INSTALLSRC_DIRECTORY}/${TBB_LIB} - DESTINATION ${NGRAPH_INSTALL_LIB}) - if(LINUX) -- install(FILES ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${TBB_LIB}.${TBB_SOVER} -+ install(FILES ${NGRAPH_LIBRARY_INSTALLSRC_DIRECTORY}/${TBB_LIB}.${TBB_SOVER} - DESTINATION ${NGRAPH_INSTALL_LIB}) - endif() - add_library(libtbb INTERFACE) -diff --git a/cmake/external_tbb_prebuilt.cmake b/cmake/external_tbb_prebuilt.cmake -index 3e1d0688f..a1cf1922a 100644 ---- a/cmake/external_tbb_prebuilt.cmake -+++ b/cmake/external_tbb_prebuilt.cmake -@@ -69,8 +69,8 @@ if (WIN32) - DEPENDEES download - ) - -- install(FILES ${NGRAPH_ARCHIVE_OUTPUT_DIRECTORY}/${TBB_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX} -- ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${TBB_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} -+ install(FILES ${NGRAPH_ARCHIVE_INSTALLSRC_DIRECTORY}/${TBB_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX} -+ ${NGRAPH_LIBRARY_INSTALLSRC_DIRECTORY}/${TBB_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} - DESTINATION ${NGRAPH_INSTALL_LIB}) - elseif(APPLE) - set(TBB_LINK_LIBS -@@ -82,7 +82,7 @@ elseif(APPLE) - COMMENT "Move tbb libraries to ngraph build directory" - ) - -- install(FILES ${NGRAPH_LIBRARY_OUTPUT_DIRECTORY}/${CMAKE_SHARED_LIBRARY_PREFIX}${TBB_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} -+ install(FILES ${NGRAPH_LIBRARY_INSTALLSRC_DIRECTORY}/${CMAKE_SHARED_LIBRARY_PREFIX}${TBB_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} - DESTINATION ${NGRAPH_INSTALL_LIB}) - endif() - --- -2.13.0.windows.1 - diff --git a/cmake/patches/ngraph/ngraph_fix_library_path.patch b/cmake/patches/ngraph/ngraph_fix_library_path.patch deleted file mode 100644 index aaa63e96e7d78..0000000000000 --- a/cmake/patches/ngraph/ngraph_fix_library_path.patch +++ /dev/null @@ -1,33 +0,0 @@ -From fcd51f874f4a96fb4ca77d762ed39ea1bf3f2c0d Mon Sep 17 00:00:00 2001 -From: Junfeng Dong -Date: Wed, 17 Apr 2019 13:42:42 -0700 -Subject: [PATCH] Fix dll library load path on Windows. - ---- - src/ngraph/runtime/backend_manager.cpp | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/src/ngraph/runtime/backend_manager.cpp b/src/ngraph/runtime/backend_manager.cpp -index eaa8fc26a..4d35c63ec 100644 ---- a/src/ngraph/runtime/backend_manager.cpp -+++ b/src/ngraph/runtime/backend_manager.cpp -@@ -123,7 +123,7 @@ unique_ptr runtime::BackendManager::create_backend(const std:: - static string find_my_file() - { - #ifdef _WIN32 -- HMODULE hModule = GetModuleHandleW(NULL); -+ HMODULE hModule = GetModuleHandleW(L"ngraph.dll"); - WCHAR wpath[MAX_PATH]; - GetModuleFileNameW(hModule, wpath, MAX_PATH); - wstring ws(wpath); -@@ -157,6 +157,7 @@ DL_HANDLE runtime::BackendManager::open_shared_library(string type) - string my_directory = file_util::get_directory(find_my_file()); - string library_path = file_util::path_join(my_directory, library_name); - #ifdef _WIN32 -+ SetDllDirectory((LPCSTR)my_directory.c_str()); - handle = LoadLibrary(library_path.c_str()); - #else - handle = dlopen(library_path.c_str(), RTLD_NOW | RTLD_GLOBAL); --- -2.13.0.windows.1 - diff --git a/cmake/patches/ngraph/ngraph_fix_mkldnn_missing_symbol.patch b/cmake/patches/ngraph/ngraph_fix_mkldnn_missing_symbol.patch new file mode 100644 index 0000000000000..96504c910003a --- /dev/null +++ b/cmake/patches/ngraph/ngraph_fix_mkldnn_missing_symbol.patch @@ -0,0 +1,64 @@ + cmake/external_mkldnn.cmake | 1 + + cmake/mkldnn_fix_missing_symbol.patch | 99 +++++++++++++++++++++++++++++++++++ + 2 files changed, 100 insertions(+) + create mode 100644 cmake/mkldnn_fix_missing_symbol.patch + +diff --git a/cmake/external_mkldnn.cmake b/cmake/external_mkldnn.cmake +index 7874aca76..bbae6d1a4 100644 +--- a/cmake/external_mkldnn.cmake ++++ b/cmake/external_mkldnn.cmake +@@ -194,7 +194,8 @@ if (WIN32) + CONFIGURE_COMMAND + PATCH_COMMAND ${MKLDNN_PATCH_REVERT_COMMAND} + COMMAND git apply --ignore-space-change --ignore-whitespace ${CMAKE_SOURCE_DIR}/cmake/${MKLDNN_PATCH_FILE} + COMMAND git apply --ignore-space-change --ignore-whitespace ${CMAKE_SOURCE_DIR}/cmake/mkldnn_fix_memory.patch ++ COMMAND git apply --ignore-space-change --ignore-whitespace ${CMAKE_SOURCE_DIR}/cmake/mkldnn_fix_missing_symbol.patch + CMAKE_GENERATOR ${CMAKE_GENERATOR} + CMAKE_GENERATOR_PLATFORM ${CMAKE_GENERATOR_PLATFORM} + CMAKE_GENERATOR_TOOLSET ${CMAKE_GENERATOR_TOOLSET} +diff --git a/cmake/mkldnn_fix_missing_symbol.patch b/cmake/mkldnn_fix_missing_symbol.patch +new file mode 100644 +index 000000000..ea1a3bd61 +--- /dev/null ++++ b/cmake/mkldnn_fix_missing_symbol.patch +@@ -0,0 +1,40 @@ ++commit d485a54ac2b07b7349dabd833961415315a18fea ++Author: Denis Samoilov ++Date: Sun Apr 14 20:11:33 2019 -0700 ++ ++ cpu: gemv: fix unresolved symbol ++ ++ Fixes #456 ++ ++diff --git a/src/cpu/gemm/gemm_driver.cpp b/src/cpu/gemm/gemm_driver.cpp ++index 0773b212..df7bc44d 100644 ++--- a/src/cpu/gemm/gemm_driver.cpp +++++ b/src/cpu/gemm/gemm_driver.cpp ++@@ -1304,10 +1304,8 @@ static mkldnn_status_t gemm_threading_driver( ++ (float *) arg->co); ++ } ++ ++- if (data_traits::data_type == data_type::s8) { ++- if (gemm_s8u8s32_jump_to_gemv_s8u8s32(arg)) { ++- return mkldnn_success; ++- } +++ if (gemm_s8u8s32_jump_to_gemv_s8u8s32(arg)) { +++ return mkldnn_success; ++ } ++ ++ int nthr = (mkldnn_in_parallel()) ? 1 : mkldnn_get_max_threads(); ++diff --git a/src/cpu/gemm/s8x8s32/jit_avx512_core_gemv_s8u8s32.cpp b/src/cpu/gemm/s8x8s32/jit_avx512_core_gemv_s8u8s32.cpp ++index 73d50e40..81646a43 100644 ++--- a/src/cpu/gemm/s8x8s32/jit_avx512_core_gemv_s8u8s32.cpp +++++ b/src/cpu/gemm/s8x8s32/jit_avx512_core_gemv_s8u8s32.cpp ++@@ -29,6 +29,10 @@ namespace cpu { ++ template ++ int gemm_s8u8s32_jump_to_gemv_s8u8s32(T *arg); ++ +++template <> +++int gemm_s8u8s32_jump_to_gemv_s8u8s32( +++ gemm_info_t *arg) { return 0; } +++ ++ template <> ++ int gemm_s8u8s32_jump_to_gemv_s8u8s32( ++ gemm_info_t *arg) { From 3fb2fc7d30a7b3d7e7079c22a7735b832441c919 Mon Sep 17 00:00:00 2001 From: tomdol Date: Tue, 6 Aug 2019 13:25:43 +0200 Subject: [PATCH 09/14] Exclude MatMulInteger for non-const zero points --- .../ngraph/ngraph_execution_provider.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc index a5bf9ed18b9f5..1ff45248a8a53 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc +++ b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc @@ -187,6 +187,22 @@ static bool IsUnsupportedOpMode(const Node* node, const onnxruntime::GraphViewer // QLinearMatMul is not supported if any of the zero points is a dynamic input return non_const_zero_point; + } else if (optype == "MatMulInteger") { + // all MatMulInteger zero points need to be constants + const auto inputs = node->InputDefs(); + if (inputs.size() == 3) { + const auto& a_zero_point = node->InputDefs()[2]; + + // not found in initializers -> not const + return initializers.find(a_zero_point->Name()) == initializers.end(); + } else if (inputs.size() == 4) { + const auto& a_zero_point = node->InputDefs()[2]; + const auto& b_zero_point = node->InputDefs()[3]; + + // not found in initializers -> not const + return initializers.find(a_zero_point->Name()) == initializers.end() || + initializers.find(b_zero_point->Name()) == initializers.end(); + } // else -> azp & bzp are 0 by default according to ONNX spec } //Op doesn't fall into known any of unsupported modes. From b47380fa22727d844baa903a038369f803559c5c Mon Sep 17 00:00:00 2001 From: tomdol Date: Tue, 6 Aug 2019 13:45:55 +0200 Subject: [PATCH 10/14] Exclude ConvInteger for non-const zero points --- .../ngraph/ngraph_execution_provider.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc index 1ff45248a8a53..7c2928ddcb498 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc +++ b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc @@ -203,6 +203,22 @@ static bool IsUnsupportedOpMode(const Node* node, const onnxruntime::GraphViewer return initializers.find(a_zero_point->Name()) == initializers.end() || initializers.find(b_zero_point->Name()) == initializers.end(); } // else -> azp & bzp are 0 by default according to ONNX spec + } else if (optype == "ConvInteger") { + // all ConvInteger zero points need to be constants + const auto inputs = node->InputDefs(); + if (inputs.size() == 3) { + const auto& x_zero_point = node->InputDefs()[2]; + + // not found in initializers -> not const + return initializers.find(x_zero_point->Name()) == initializers.end(); + } else if (inputs.size() == 4) { + const auto& x_zero_point = node->InputDefs()[2]; + const auto& w_zero_point = node->InputDefs()[3]; + + // not found in initializers -> not const + return initializers.find(x_zero_point->Name()) == initializers.end() || + initializers.find(w_zero_point->Name()) == initializers.end(); + } // else -> xzp & wzp are 0 by default according to ONNX spec } //Op doesn't fall into known any of unsupported modes. From 80bf46b740fff8edc36b756972333ebaded2c63a Mon Sep 17 00:00:00 2001 From: tomdol Date: Wed, 7 Aug 2019 12:48:39 +0200 Subject: [PATCH 11/14] Enable full Cast op support --- .../core/providers/ngraph/ngraph_execution_provider.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc index 7c2928ddcb498..9274d447c027f 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc +++ b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc @@ -129,11 +129,6 @@ static bool IsUnsupportedOpMode(const Node* node, const onnxruntime::GraphViewer return true; } } - } else if (optype == "Cast") { - //support of casting to bool in nGraph is in progress - const auto& attributes = node->GetAttributes(); - const auto to_attr = attributes.find("to"); - return to_attr->second.i() == ONNX_NAMESPACE::TensorProto::BOOL; } else if (optype == "Slice") { //Slice in opset 10 is currently not supported. //unsupported inputs: starts, ends, axes, steps From 26d5a7cb355bf1ca703f669dca1b45c0fc131575 Mon Sep 17 00:00:00 2001 From: tomdol Date: Wed, 7 Aug 2019 17:48:57 +0200 Subject: [PATCH 12/14] Use the v0.22.1 tag --- cmake/external/ngraph.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/ngraph.cmake b/cmake/external/ngraph.cmake index f944791afeeef..45aae7d44f512 100644 --- a/cmake/external/ngraph.cmake +++ b/cmake/external/ngraph.cmake @@ -11,7 +11,7 @@ set(ngraph_SRC ${CMAKE_CURRENT_BINARY_DIR}/ngraph/src/project_ngraph) set(prebuilt_ONNX_SOURCE_DIR "${PROJECT_SOURCE_DIR}/external/onnx") set(prebuilt_ONNX_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/onnx") set(ngraph_URL "https://github.com/NervanaSystems/ngraph.git") -set(ngraph_TAG "v0.22.1-rc.0") +set(ngraph_TAG "v0.22.1") # Libraries for python package. if (WIN32) From 098dd6fa5744c0058e3d2921bde63b09eace5ad8 Mon Sep 17 00:00:00 2001 From: tomdol Date: Wed, 7 Aug 2019 17:49:33 +0200 Subject: [PATCH 13/14] Skip ConvTranspose_InvalidKernelShape test for ngraph provider --- onnxruntime/test/providers/cpu/nn/conv_transpose_op_test.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onnxruntime/test/providers/cpu/nn/conv_transpose_op_test.cc b/onnxruntime/test/providers/cpu/nn/conv_transpose_op_test.cc index 00f3d97e15cd4..22e6d728de26d 100644 --- a/onnxruntime/test/providers/cpu/nn/conv_transpose_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/conv_transpose_op_test.cc @@ -240,6 +240,7 @@ TEST(ConvTransposeTest, ConvTranspose_2D_OutputShapeWithBatchSize) { TestConvTransposeOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape); } +#ifndef USE_NGRAPH TEST(ConvTransposeTest, ConvTranspose_InvalidKernelShape) { ConvTransposeOpAttributes attrs = { vector{1, 1, 1, 5}, // invalid kernel_shape, should be [1, 5] @@ -264,6 +265,7 @@ TEST(ConvTransposeTest, ConvTranspose_InvalidKernelShape) { OpTester::ExpectResult::kExpectFailure, "kernel_shape num_dims is not compatible with W num_dims. kernel_shape: {1,1,1,5} W: {1,1,1,5}"); } +#endif TEST(ConvTransposeTest, ConvTranspose_onnx) { ConvTransposeOpAttributes attrs = { From 4892dd24260603c177c75c686be9c4b8bc5c8cf1 Mon Sep 17 00:00:00 2001 From: tomdol Date: Fri, 9 Aug 2019 14:34:40 +0200 Subject: [PATCH 14/14] Create sub-graph ModelProto from fused_node --- .../core/providers/ngraph/ngraph_custom_op.cc | 7 --- .../core/providers/ngraph/ngraph_custom_op.h | 1 - .../ngraph/ngraph_execution_provider.cc | 47 +++++-------------- .../ngraph/ngraph_execution_provider.h | 5 -- 4 files changed, 11 insertions(+), 49 deletions(-) diff --git a/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc b/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc index 8579ba6919ed2..cfad088e800da 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc +++ b/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc @@ -37,7 +37,6 @@ static bool check_ngraph_dump_ops() { NGRAPHCustomOp::NGRAPHCustomOp(const ComputeContext* context, const ONNX_NAMESPACE::ModelProto& model_proto, - const std::shared_ptr initializers, const std::shared_ptr& ng_backend) : ng_backend_{ng_backend}, model_proto_{model_proto} { @@ -46,12 +45,6 @@ NGRAPHCustomOp::NGRAPHCustomOp(const ComputeContext* context, allocator_ = context->allocator_handle; name_ = context->node_name; - auto graph_proto = model_proto_.mutable_graph(); - - for (const auto& initializer : initializers->tensors()) { - graph_proto->add_initializer()->CopyFrom(initializer); - } - if (check_ngraph_dump_ops()) { std::fstream dump(name_ + ".onnx", std::ios::out | std::ios::trunc | std::ios::binary); model_proto_.SerializeToOstream(&dump); diff --git a/onnxruntime/core/providers/ngraph/ngraph_custom_op.h b/onnxruntime/core/providers/ngraph/ngraph_custom_op.h index 55609a5d45559..27659748212de 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_custom_op.h +++ b/onnxruntime/core/providers/ngraph/ngraph_custom_op.h @@ -27,7 +27,6 @@ class NGRAPHCustomOp { public: NGRAPHCustomOp(const ComputeContext* context, const ONNX_NAMESPACE::ModelProto& model_proto, - const std::shared_ptr initializers, const std::shared_ptr& ng_backend); Status Compute(const OrtCustomOpApi* api, OrtKernelContext* context) const; diff --git a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc index 9274d447c027f..60fad0071ccf6 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc +++ b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc @@ -482,19 +482,6 @@ NGRAPHExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie const auto unsupported_nodes = GetUnsupportedNodeIndices(graph_viewer, ng_required_initializers); - // Create a shared_ptr with this graph's initializers which are required to construct nGraph's IR - if (graph_initializers_.count(graph_viewer.Name()) == 0) { - graph_initializers_[graph_viewer.Name()] = std::make_shared(); - - auto& initializers = graph_initializers_[graph_viewer.Name()]; - initializers->set_name("initializers"); - initializers->set_type(ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_TENSORS); - for (const auto& init : ng_required_initializers) { - auto tensor = initializers->add_tensors(); - *tensor = *(graph_viewer.GetAllInitializedTensors().at(init)); - } - } - //If all ops are supported, no partitioning is required. Short-circuit and avoid splitting. if (unsupported_nodes.empty()) { std::vector inputs; @@ -538,28 +525,21 @@ NGRAPHExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie } static ONNX_NAMESPACE::ModelProto GetModelProtoFromFusedNode(const onnxruntime::Node* fused_node) { - ONNX_NAMESPACE::ModelProto model_proto; - auto graph_proto = model_proto.mutable_graph(); - const auto& fused_graph = fused_node->GetFunctionBody()->Body(); + const auto* node_function = fused_node->GetFunctionBody(); - for (const auto& node : fused_graph.Nodes()) { - node.ToProto(*(graph_proto->add_node())); - } + ORT_ENFORCE(node_function != nullptr, "Could not extract function body for node: ", fused_node->Name()); - for (const auto& input : fused_node->InputDefs()) { - auto valueInfoProto = graph_proto->add_input(); - *valueInfoProto = input->ToProto(); - } + const Graph& node_subgraph = node_function->Body(); + onnxruntime::Model model{node_subgraph.Name(), true}; - for (const auto& output : fused_node->OutputDefs()) { - auto valueInfoProto = graph_proto->add_output(); - *valueInfoProto = output->ToProto(); - } + ONNX_NAMESPACE::ModelProto model_proto = model.ToProto(); + model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); + + *(model_proto.mutable_graph()) = node_subgraph.ToGraphProto(); auto opset = model_proto.add_opset_import(); opset->set_domain(kOnnxDomain); - opset->set_version(fused_graph.DomainToVersionMap().at(kOnnxDomain)); - model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); + opset->set_version(node_subgraph.DomainToVersionMap().at(kOnnxDomain)); return model_proto; } @@ -569,17 +549,12 @@ Status NGRAPHExecutionProvider::Compile(const std::vector& f for (const auto& fused_node : fused_nodes) { NodeComputeInfo compute_info; - const auto& attributes = fused_node->GetAttributes(); - const auto graph_name = attributes.at("graph_name").s(); - // find the shared_ptr with initializes of a graph this fused_node belongs to - auto initializers = graph_initializers_.at(graph_name); - // Local copy of backend since, class members cannot be captured. auto ngraph_backend = ng_backend_; - compute_info.create_state_func = [model_proto = GetModelProtoFromFusedNode(fused_node), ngraph_backend, initializers] + compute_info.create_state_func = [model_proto = GetModelProtoFromFusedNode(fused_node), ngraph_backend] (ComputeContext* context, FunctionState* state) { - auto* p = new onnxruntime::ngraph_ep::NGRAPHCustomOp(context, model_proto, initializers, ngraph_backend); + auto* p = new ngraph_ep::NGRAPHCustomOp(context, model_proto, ngraph_backend); *state = p; return 0; }; diff --git a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h index 78ba490ba4ca9..daade7022d44d 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h +++ b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h @@ -12,10 +12,6 @@ namespace ngraph { } } -namespace ONNX_NAMESPACE { - class AttributeProto; -} - namespace onnxruntime { // Information needed to construct nGraph execution providers. @@ -38,7 +34,6 @@ class NGRAPHExecutionProvider : public IExecutionProvider { private: std::shared_ptr ng_backend_; - mutable std::unordered_map> graph_initializers_; }; }