From c58b508a1569e20a20ca6a0185e8a99731e504b4 Mon Sep 17 00:00:00 2001 From: Paul McDaniel Date: Wed, 9 Oct 2019 14:29:06 -0700 Subject: [PATCH 01/14] staging some in progress work --- cmake/onnxruntime_common.cmake | 2 + include/onnxruntime/core/common/version.h | 6 ++ onnxruntime/core/platform/env.h | 4 + onnxruntime/core/platform/telemetry.cc | 11 +++ onnxruntime/core/platform/telemetry.h | 52 ++++++++++++ onnxruntime/core/platform/windows/env.cc | 8 +- .../core/platform/windows/telemetry.cc | 79 +++++++++++++++++++ onnxruntime/core/platform/windows/telemetry.h | 47 +++++++++++ 8 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 include/onnxruntime/core/common/version.h create mode 100644 onnxruntime/core/platform/telemetry.cc create mode 100644 onnxruntime/core/platform/telemetry.h create mode 100644 onnxruntime/core/platform/windows/telemetry.cc create mode 100644 onnxruntime/core/platform/windows/telemetry.h diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake index 929b8fa91ef61..e473bc9d41af7 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake @@ -16,6 +16,8 @@ set(onnxruntime_common_src_patterns "${ONNXRUNTIME_ROOT}/core/platform/env.cc" "${ONNXRUNTIME_ROOT}/core/platform/env_time.h" "${ONNXRUNTIME_ROOT}/core/platform/env_time.cc" + "${ONNXRUNTIME_ROOT}/core/platform/telemetry.h" + "${ONNXRUNTIME_ROOT}/core/platform/telemetry.cc" ) if(WIN32) diff --git a/include/onnxruntime/core/common/version.h b/include/onnxruntime/core/common/version.h new file mode 100644 index 0000000000000..c2274b634d76d --- /dev/null +++ b/include/onnxruntime/core/common/version.h @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#define ONNXRUNTIME_VERSION_STRING "1.0" diff --git a/onnxruntime/core/platform/env.h b/onnxruntime/core/platform/env.h index 5543ad63c0101..7d6f37004c74b 100644 --- a/onnxruntime/core/platform/env.h +++ b/onnxruntime/core/platform/env.h @@ -26,6 +26,7 @@ limitations under the License. #include "core/common/common.h" #include "core/framework/callback.h" #include "core/platform/env_time.h" +#include "core/platform/telemetry.h" #ifndef _WIN32 #include @@ -133,6 +134,9 @@ class Env { // returns the name that LoadDynamicLibrary() can use virtual std::string FormatLibraryFileName(const std::string& name, const std::string& version) const = 0; + // \brief returns a provider that will handle telemetry on the current platform + virtual Telemetry& GetTelemetryProvider() = 0; + protected: Env(); diff --git a/onnxruntime/core/platform/telemetry.cc b/onnxruntime/core/platform/telemetry.cc new file mode 100644 index 0000000000000..9a4677018886a --- /dev/null +++ b/onnxruntime/core/platform/telemetry.cc @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/platform/telemetry.h" + +namespace onnxruntime { + +Telemetry::Telemetry() = default; + +} // namespace onnxruntime + diff --git a/onnxruntime/core/platform/telemetry.h b/onnxruntime/core/platform/telemetry.h new file mode 100644 index 0000000000000..2e360e8243598 --- /dev/null +++ b/onnxruntime/core/platform/telemetry.h @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +#include "core/common/status.h" +#include "core/common/common.h" + +namespace onnxruntime { + +/** + * Configuration information for a session. + * An interface used by the onnxruntime implementation to + * access operating system functionality for telemetry + * + * look at env.h and the Env objection which is the activation factory + * for telemetry instances + * + * All Telemetry implementations are safe for concurrent access from + * multiple threads without any external synchronization. + */ +class Telemetry { + public: + virtual ~Telemetry() = default; + + virtual void LogProcessInfo(const std::string& runtimeVersion, bool isRedist) = 0; + + virtual void LogSessionCreation(uint32_t sessionId, int64_t irVersion, const std::string& modelProducerName, + const std::string& modelProducerVersion,const std::string& modelDomain, + const std::vector& modelOpsetImports, uint32_t modelPrecision, + const std::string& modelGraphName, const std::string& modelGraphVersion, + const std::unordered_map& modelMetaData, + bool modelFromStream, const std::string& executionProviders) = 0; + + virtual void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, + const char* function, uint32_t line) = 0; + + virtual void LogRuntimePerf(uint32_t sessionId, uint32_t runTotalTimeMs) = 0; + + protected: + // don't create these, use Env::GetTelemetryProvider() + Telemetry(); + + private: + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Telemetry); +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/windows/env.cc b/onnxruntime/core/platform/windows/env.cc index 010077d07b273..9f086b5818d0b 100644 --- a/onnxruntime/core/platform/windows/env.cc +++ b/onnxruntime/core/platform/windows/env.cc @@ -25,6 +25,7 @@ limitations under the License. #include "core/common/logging/logging.h" #include "core/platform/env.h" +#include "core/platform/windows/telemetry.h" namespace onnxruntime { @@ -211,6 +212,11 @@ class WindowsEnv : public Env { ORT_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented"); } + // \brief returns a provider that will handle telemetry on the current platform + Telemetry& GetTelemetryProvider() { + return telemetryProvider_; + } + private: WindowsEnv() : GetSystemTimePreciseAsFileTime_(nullptr) { @@ -228,8 +234,8 @@ class WindowsEnv : public Env { typedef VOID(WINAPI* FnGetSystemTimePreciseAsFileTime)(LPFILETIME); FnGetSystemTimePreciseAsFileTime GetSystemTimePreciseAsFileTime_; + WindowsTelemetry telemetryProvider_; }; - } // namespace #if defined(PLATFORM_WINDOWS) diff --git a/onnxruntime/core/platform/windows/telemetry.cc b/onnxruntime/core/platform/windows/telemetry.cc new file mode 100644 index 0000000000000..840ccf3386613 --- /dev/null +++ b/onnxruntime/core/platform/windows/telemetry.cc @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/platform/windows/telemetry.h" +#include "core/common/version.h" + +// ETW includes +// need space after Windows.h to prevent clang-format re-ordering breaking the build. +// TraceLoggingProvider.h must follow Windows.h +#include + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 26440) // Warning C26440 from TRACELOGGING_DEFINE_PROVIDER +#endif + +#include +#include + +namespace onnxruntime { + +namespace { +TRACELOGGING_DEFINE_PROVIDER(telemetry_provider_handle, "Microsoft.ML.ONNXRuntime", + // {3a26b1ff-7484-7484-7484-15261f42614d} + (0x3a26b1ff, 0x7484, 0x7484, 0x74, 0x84, 0x15, 0x26, 0x1f, 0x42, 0x61, 0x4d), + TraceLoggingOptionMicrosoftTelemetry()); +} // namespace + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +WindowsTelemetry::WindowsTelemetry() { + HRESULT hr = TraceLoggingRegister(telemetry_provider_handle); + if (SUCCEEDED(hr)) { + register_succeeded_ = true; + } +} + +WindowsTelemetry::~WindowsTelemetry() { + if (register_succeeded_) { + TraceLoggingUnregister(telemetry_provider_handle); + register_succeeded_ = false; + } +} + +void WindowsTelemetry::LogProcessInfo(const std::string& runtimeVersion, bool isRedist) { + if (!register_succeeded_) + return; + + TraceLoggingWrite(telemetry_provider_handle, + "ProcessInfo", + TraceLoggingBool(true, "UTCReplace_AppSessionGuid"), + TelemetryPrivacyDataTag(PDT_ProductAndServiceUsage), + // Telemetry info + TraceLoggingKeyword(1), + TraceLoggingUInt8(0, "schemaVersion"), + TraceLoggingString(ONNXRUNTIME_VERSION_STRING, "runtimeVersion"), + TraceLoggingBool(true, "isRedist"), + TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES)); + +} + +void WindowsTelemetry::LogSessionCreation(uint32_t sessionId, int64_t irVersion, const std::string& modelProducerName, + const std::string& modelProducerVersion, const std::string& modelDomain, + const std::vector& modelOpsetImports, uint32_t modelPrecision, + const std::string& modelGraphName, const std::string& modelGraphVersion, + const std::unordered_map& modelMetaData, + bool modelFromStream, const std::string& executionProviders) { +} + +void WindowsTelemetry::LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, + const char* function, uint32_t line) { +} + +void WindowsTelemetry::LogRuntimePerf(uint32_t sessionId, uint32_t runTotalTimeMs) { +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/windows/telemetry.h b/onnxruntime/core/platform/windows/telemetry.h new file mode 100644 index 0000000000000..3de9822a458ae --- /dev/null +++ b/onnxruntime/core/platform/windows/telemetry.h @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/platform/telemetry.h" + +// Note: this needs to get moved to a release pipeline still (paulm) +// *** +#define TraceLoggingOptionMicrosoftTelemetry() \ + TraceLoggingOptionGroup(0x4f50731a, 0x89cf, 0x4782, 0xb3, 0xe0, 0xdc, 0xe8, 0xc9, 0x4, 0x76, 0xba) +#define MICROSOFT_KEYWORD_MEASURES 0x0000400000000000 // Bit 46 +#define TelemetryPrivacyDataTag(tag) TraceLoggingUInt64((tag), "PartA_PrivTags") +#define PDT_ProductAndServiceUsage 0x0000000002000000u +// *** + +namespace onnxruntime { + +/** + * derives and implments a Telemetry provider on Windows + */ +class WindowsTelemetry : public Telemetry { + + public: + + // these are allowed to be created, WindowsEnv will create one + WindowsTelemetry(); + ~WindowsTelemetry(); + + void LogProcessInfo(const std::string& runtimeVersion, bool isRedist); + + void LogSessionCreation(uint32_t sessionId, int64_t irVersion, const std::string& modelProducerName, + const std::string& modelProducerVersion, const std::string& modelDomain, + const std::vector& modelOpsetImports, uint32_t modelPrecision, + const std::string& modelGraphName, const std::string& modelGraphVersion, + const std::unordered_map& modelMetaData, + bool modelFromStream, const std::string& executionProviders); + + void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, + const char* function, uint32_t line); + + void LogRuntimePerf(uint32_t sessionId, uint32_t runTotalTimeMs); + + private: + bool register_succeeded_ = false; +}; + +} // namespace onnxruntime From 1447aeb0e1d7bb4c17ec258f416ffe5efff5155f Mon Sep 17 00:00:00 2001 From: Paul McDaniel Date: Thu, 10 Oct 2019 14:29:36 -0700 Subject: [PATCH 02/14] added some inference session changes and global error logging --- include/onnxruntime/core/common/common.h | 26 +++++--- onnxruntime/core/platform/env.h | 2 +- onnxruntime/core/platform/telemetry.cc | 8 +++ onnxruntime/core/platform/telemetry.h | 8 +-- onnxruntime/core/platform/windows/env.cc | 2 +- .../core/platform/windows/telemetry.cc | 60 ++++++++++++++----- onnxruntime/core/platform/windows/telemetry.h | 18 +++--- onnxruntime/core/session/environment.cc | 6 ++ onnxruntime/core/session/inference_session.cc | 55 +++++++++-------- onnxruntime/core/session/inference_session.h | 4 ++ 10 files changed, 130 insertions(+), 59 deletions(-) diff --git a/include/onnxruntime/core/common/common.h b/include/onnxruntime/core/common/common.h index 414ef5e6b17ba..cb997b4cdd224 100644 --- a/include/onnxruntime/core/common/common.h +++ b/include/onnxruntime/core/common/common.h @@ -74,6 +74,9 @@ using common::Status; static_cast(fn) std::vector GetStackTrace(); +// these is a helper function that gets defined by platform/Telemetry +void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, + const char* function, uint32_t line); // __PRETTY_FUNCTION__ isn't a macro on gcc, so use a check for _MSC_VER // so we only define it as one for MSVC @@ -137,16 +140,25 @@ std::vector GetStackTrace(); ORT_DISALLOW_COPY_AND_ASSIGNMENT(TypeName); \ ORT_DISALLOW_MOVE(TypeName) -#define ORT_RETURN_IF_ERROR(expr) \ - do { \ - auto _status = (expr); \ - if ((!_status.IsOK())) return _status; \ +#define ORT_RETURN_IF_ERROR_SESSIONID(expr, session_id) \ + do { \ + auto _status = (expr); \ + if ((!_status.IsOK())) { \ + LogRuntimeError(session_id, _status, __FILE__, __FUNCTION__, __LINE__); \ + return _status; \ + } \ } while (0) +#define ORT_RETURN_IF_ERROR_SESSIONID_(expr) ORT_RETURN_IF_ERROR_SESSIONID(expr, session_id_) +#define ORT_RETURN_IF_ERROR(expr) ORT_RETURN_IF_ERROR_SESSIONID(expr, 0) + #define ORT_THROW_IF_ERROR(expr) \ do { \ auto _status = (expr); \ - if ((!_status.IsOK())) ORT_THROW(_status); \ + if ((!_status.IsOK())) { \ + LogRuntimeError(0, _status, __FILE__, __FUNCTION__, __LINE__); \ + ORT_THROW(_status); \ + } \ } while (0) // use this macro when cannot early return @@ -164,8 +176,8 @@ std::vector GetStackTrace(); #define GSL_SUPPRESS(tag) #endif -inline void MakeStringInternal(std::ostringstream& /*ss*/) noexcept { -} + inline void MakeStringInternal(std::ostringstream& /*ss*/) noexcept { + } template inline void MakeStringInternal(std::ostringstream& ss, const T& t) noexcept { diff --git a/onnxruntime/core/platform/env.h b/onnxruntime/core/platform/env.h index 7d6f37004c74b..4339a0a12746e 100644 --- a/onnxruntime/core/platform/env.h +++ b/onnxruntime/core/platform/env.h @@ -135,7 +135,7 @@ class Env { virtual std::string FormatLibraryFileName(const std::string& name, const std::string& version) const = 0; // \brief returns a provider that will handle telemetry on the current platform - virtual Telemetry& GetTelemetryProvider() = 0; + virtual const Telemetry& GetTelemetryProvider() const = 0; protected: Env(); diff --git a/onnxruntime/core/platform/telemetry.cc b/onnxruntime/core/platform/telemetry.cc index 9a4677018886a..84eafa0500595 100644 --- a/onnxruntime/core/platform/telemetry.cc +++ b/onnxruntime/core/platform/telemetry.cc @@ -2,10 +2,18 @@ // Licensed under the MIT License. #include "core/platform/telemetry.h" +#include "core/platform/env.h" namespace onnxruntime { Telemetry::Telemetry() = default; +void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, + const char* function, uint32_t line) +{ + const Env& env = Env::Default(); + env.GetTelemetryProvider().LogRuntimeError(sessionId, status, file, function, line); +} + } // namespace onnxruntime diff --git a/onnxruntime/core/platform/telemetry.h b/onnxruntime/core/platform/telemetry.h index 2e360e8243598..3cb6f0f5b52c5 100644 --- a/onnxruntime/core/platform/telemetry.h +++ b/onnxruntime/core/platform/telemetry.h @@ -27,19 +27,19 @@ class Telemetry { public: virtual ~Telemetry() = default; - virtual void LogProcessInfo(const std::string& runtimeVersion, bool isRedist) = 0; + virtual void LogProcessInfo() const = 0; virtual void LogSessionCreation(uint32_t sessionId, int64_t irVersion, const std::string& modelProducerName, const std::string& modelProducerVersion,const std::string& modelDomain, const std::vector& modelOpsetImports, uint32_t modelPrecision, const std::string& modelGraphName, const std::string& modelGraphVersion, const std::unordered_map& modelMetaData, - bool modelFromStream, const std::string& executionProviders) = 0; + bool modelFromStream, const std::string& executionProviders) const = 0; virtual void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, - const char* function, uint32_t line) = 0; + const char* function, uint32_t line) const = 0; - virtual void LogRuntimePerf(uint32_t sessionId, uint32_t runTotalTimeMs) = 0; + virtual void LogRuntimePerf(uint32_t sessionId, uint32_t runTotalTimeMs) const = 0; protected: // don't create these, use Env::GetTelemetryProvider() diff --git a/onnxruntime/core/platform/windows/env.cc b/onnxruntime/core/platform/windows/env.cc index 9f086b5818d0b..2175792c88e14 100644 --- a/onnxruntime/core/platform/windows/env.cc +++ b/onnxruntime/core/platform/windows/env.cc @@ -213,7 +213,7 @@ class WindowsEnv : public Env { } // \brief returns a provider that will handle telemetry on the current platform - Telemetry& GetTelemetryProvider() { + const Telemetry& GetTelemetryProvider() const override { return telemetryProvider_; } diff --git a/onnxruntime/core/platform/windows/telemetry.cc b/onnxruntime/core/platform/windows/telemetry.cc index 840ccf3386613..6a70e9e9e5d08 100644 --- a/onnxruntime/core/platform/windows/telemetry.cc +++ b/onnxruntime/core/platform/windows/telemetry.cc @@ -30,35 +30,51 @@ TRACELOGGING_DEFINE_PROVIDER(telemetry_provider_handle, "Microsoft.ML.ONNXRuntim #pragma warning(pop) #endif +OrtMutex WindowsTelemetry::mutex_; +uint32_t WindowsTelemetry::global_register_count_ = 0; + WindowsTelemetry::WindowsTelemetry() { - HRESULT hr = TraceLoggingRegister(telemetry_provider_handle); - if (SUCCEEDED(hr)) { - register_succeeded_ = true; + std::lock_guard lock(mutex_); + if (global_register_count_ == 0) { + // TraceLoggingRegister is fancy in that you can only register once GLOBALLY for the whole process + HRESULT hr = TraceLoggingRegister(telemetry_provider_handle); + if (SUCCEEDED(hr)) { + global_register_count_ += 1; + } } } WindowsTelemetry::~WindowsTelemetry() { - if (register_succeeded_) { - TraceLoggingUnregister(telemetry_provider_handle); - register_succeeded_ = false; + std::lock_guard lock(mutex_); + if (global_register_count_ > 0) { + global_register_count_ -= 1; + if (global_register_count_ == 0) { + TraceLoggingUnregister(telemetry_provider_handle); + } } } -void WindowsTelemetry::LogProcessInfo(const std::string& runtimeVersion, bool isRedist) { - if (!register_succeeded_) +void WindowsTelemetry::LogProcessInfo() const { + if (global_register_count_ == 0) + return; + + static std::atomic process_info_logged; + + // did we already log the process info? we only need to log it once + if (process_info_logged.exchange(true)) return; TraceLoggingWrite(telemetry_provider_handle, "ProcessInfo", TraceLoggingBool(true, "UTCReplace_AppSessionGuid"), TelemetryPrivacyDataTag(PDT_ProductAndServiceUsage), + TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES), // Telemetry info - TraceLoggingKeyword(1), TraceLoggingUInt8(0, "schemaVersion"), TraceLoggingString(ONNXRUNTIME_VERSION_STRING, "runtimeVersion"), - TraceLoggingBool(true, "isRedist"), - TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES)); + TraceLoggingBool(true, "isRedist")); + process_info_logged = true; } void WindowsTelemetry::LogSessionCreation(uint32_t sessionId, int64_t irVersion, const std::string& modelProducerName, @@ -66,14 +82,30 @@ void WindowsTelemetry::LogSessionCreation(uint32_t sessionId, int64_t irVersion, const std::vector& modelOpsetImports, uint32_t modelPrecision, const std::string& modelGraphName, const std::string& modelGraphVersion, const std::unordered_map& modelMetaData, - bool modelFromStream, const std::string& executionProviders) { + bool modelFromStream, const std::string& executionProviders) const { } void WindowsTelemetry::LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, - const char* function, uint32_t line) { + const char* function, uint32_t line) const { + if (global_register_count_ == 0) + return; + + TraceLoggingWrite(telemetry_provider_handle, + "RuntimeError", + TelemetryPrivacyDataTag(PDT_ProductAndServicePerformance), + TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES), + // Telemetry info + TraceLoggingUInt8(0, "schemaVersion"), + TraceLoggingUInt32(sessionId, "sessionId"), + TraceLoggingUInt32(status.Code(), "errorCode"), + TraceLoggingUInt32(status.Category(), "errorCategory"), + TraceLoggingString(status.ErrorMessage().c_str(), "errorMessage"), + TraceLoggingString(file, "file"), + TraceLoggingString(function, "function"), + TraceLoggingInt32(line, "line")); } -void WindowsTelemetry::LogRuntimePerf(uint32_t sessionId, uint32_t runTotalTimeMs) { +void WindowsTelemetry::LogRuntimePerf(uint32_t sessionId, uint32_t runTotalTimeMs) const { } } // namespace onnxruntime diff --git a/onnxruntime/core/platform/windows/telemetry.h b/onnxruntime/core/platform/windows/telemetry.h index 3de9822a458ae..a7c5ce67a475c 100644 --- a/onnxruntime/core/platform/windows/telemetry.h +++ b/onnxruntime/core/platform/windows/telemetry.h @@ -3,14 +3,17 @@ #pragma once #include "core/platform/telemetry.h" +#include "core/platform/ort_mutex.h" +#include // Note: this needs to get moved to a release pipeline still (paulm) // *** #define TraceLoggingOptionMicrosoftTelemetry() \ TraceLoggingOptionGroup(0x4f50731a, 0x89cf, 0x4782, 0xb3, 0xe0, 0xdc, 0xe8, 0xc9, 0x4, 0x76, 0xba) -#define MICROSOFT_KEYWORD_MEASURES 0x0000400000000000 // Bit 46 +#define MICROSOFT_KEYWORD_MEASURES 0x0000400000000000 // Bit 46 #define TelemetryPrivacyDataTag(tag) TraceLoggingUInt64((tag), "PartA_PrivTags") -#define PDT_ProductAndServiceUsage 0x0000000002000000u +#define PDT_ProductAndServicePerformance 0x0000000001000000u +#define PDT_ProductAndServiceUsage 0x0000000002000000u // *** namespace onnxruntime { @@ -26,22 +29,23 @@ class WindowsTelemetry : public Telemetry { WindowsTelemetry(); ~WindowsTelemetry(); - void LogProcessInfo(const std::string& runtimeVersion, bool isRedist); + void LogProcessInfo() const override; void LogSessionCreation(uint32_t sessionId, int64_t irVersion, const std::string& modelProducerName, const std::string& modelProducerVersion, const std::string& modelDomain, const std::vector& modelOpsetImports, uint32_t modelPrecision, const std::string& modelGraphName, const std::string& modelGraphVersion, const std::unordered_map& modelMetaData, - bool modelFromStream, const std::string& executionProviders); + bool modelFromStream, const std::string& executionProviders) const override; void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, - const char* function, uint32_t line); + const char* function, uint32_t line) const override; - void LogRuntimePerf(uint32_t sessionId, uint32_t runTotalTimeMs); + void LogRuntimePerf(uint32_t sessionId, uint32_t runTotalTimeMs) const override; private: - bool register_succeeded_ = false; + static OrtMutex mutex_; + static uint32_t global_register_count_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/session/environment.cc b/onnxruntime/core/session/environment.cc index d1f9041c9253f..08cb034ec2c69 100644 --- a/onnxruntime/core/session/environment.cc +++ b/onnxruntime/core/session/environment.cc @@ -14,6 +14,8 @@ #include "core/graph/automl_ops/automl_defs.h" #endif +#include "core/platform/env.h" + namespace onnxruntime { using namespace ::onnxruntime::common; using namespace ONNX_NAMESPACE; @@ -76,6 +78,10 @@ Internal copy node Internal copy node )DOC"); + // fire off startup telemetry (this call is idempotent) + const Env& env = Env::Default(); + env.GetTelemetryProvider().LogProcessInfo(); + is_initialized_ = true; } catch (std::exception& ex) { status = Status{ONNXRUNTIME, common::RUNTIME_EXCEPTION, std::string{"Exception caught: "} + ex.what()}; diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index d52d5e404ba89..45adc7e674fa7 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -92,6 +92,8 @@ inline std::basic_string GetCurrentTimeString() { } // namespace +std::atomic InferenceSession::global_session_id_ = 1; + InferenceSession::InferenceSession(const SessionOptions& session_options, logging::LoggingManager* logging_manager) : session_options_(session_options), @@ -119,6 +121,9 @@ InferenceSession::InferenceSession(const SessionOptions& session_options, if (session_options.enable_profiling) { StartProfiling(session_options.profile_file_prefix); } + + // a monotonically increasing session id for use in telemetry + session_id_ = global_session_id_.fetch_add(1); } InferenceSession::~InferenceSession() { @@ -174,7 +179,7 @@ common::Status InferenceSession::AddCustomTransformerList(const std::vector& op_domains) { std::shared_ptr custom_registry; - ORT_RETURN_IF_ERROR(CreateCustomRegistry(op_domains, custom_registry)); + ORT_RETURN_IF_ERROR_SESSIONID_(CreateCustomRegistry(op_domains, custom_registry)); RegisterCustomRegistry(custom_registry); return Status::OK(); } @@ -207,12 +212,12 @@ common::Status InferenceSession::Load(std::function p_tmp_model; status = loader(p_tmp_model); - ORT_RETURN_IF_ERROR(status); + ORT_RETURN_IF_ERROR_SESSIONID_(status); model_ = p_tmp_model; status = DoPostLoadProcessing(*model_); - ORT_RETURN_IF_ERROR(status); + ORT_RETURN_IF_ERROR_SESSIONID_(status); // all steps complete, mark the model as loaded. is_model_loaded_ = true; @@ -349,21 +354,21 @@ common::Status InferenceSession::TransformGraph(onnxruntime::Graph& graph, // 5. insert cast nodes. // first apply global(execution provider independent), level 1(default/system/basic) graph to graph optimizations - ORT_RETURN_IF_ERROR(graph_transformer_mgr.ApplyTransformers(graph, TransformerLevel::Level1)); + ORT_RETURN_IF_ERROR_SESSIONID_(graph_transformer_mgr.ApplyTransformers(graph, TransformerLevel::Level1)); // Do partitioning based on execution providers' capability. GraphPartitioner partitioner(kernel_registry_manager, providers); - ORT_RETURN_IF_ERROR(partitioner.Partition(graph, session_state.ExportDll(), session_state.GetMutableFuncMgr())); + ORT_RETURN_IF_ERROR_SESSIONID_(partitioner.Partition(graph, session_state.ExportDll(), session_state.GetMutableFuncMgr())); // apply transformers except default transformers // Default transformers are required for correctness and they are owned and run by inference session for (int i = static_cast(TransformerLevel::Level1); i < static_cast(TransformerLevel::MaxTransformerLevel); i++) { - ORT_RETURN_IF_ERROR(graph_transformer_mgr.ApplyTransformers(graph, static_cast(i))); + ORT_RETURN_IF_ERROR_SESSIONID_(graph_transformer_mgr.ApplyTransformers(graph, static_cast(i))); } bool modified = false; // Insert cast node/s. - ORT_RETURN_IF_ERROR(insert_cast_transformer.Apply(graph, modified)); + ORT_RETURN_IF_ERROR_SESSIONID_(insert_cast_transformer.Apply(graph, modified)); // Now every node should be already assigned to an execution provider for (auto& node : graph.Nodes()) { @@ -386,7 +391,7 @@ common::Status InferenceSession::TransformGraph(onnxruntime::Graph& graph, // Insert copy node/s. MemcpyTransformer copy_transformer{provider_types, kernel_registry_manager}; - ORT_RETURN_IF_ERROR(copy_transformer.Apply(graph, modified)); + ORT_RETURN_IF_ERROR_SESSIONID_(copy_transformer.Apply(graph, modified)); return common::Status::OK(); } @@ -412,7 +417,7 @@ common::Status InferenceSession::CreateSubgraphSessionState(Graph& graph, Sessio subgraph_session_state->GetMutableFuncMgr().SetFusedFuncs(session_state.GetFuncMgr()); // recurse - ORT_RETURN_IF_ERROR(CreateSubgraphSessionState(*subgraph, *subgraph_session_state)); + ORT_RETURN_IF_ERROR_SESSIONID_(CreateSubgraphSessionState(*subgraph, *subgraph_session_state)); // add the subgraph SessionState instance to the parent graph SessionState so it can be retrieved // by Compute() via OpKernelContextInternal. @@ -448,7 +453,7 @@ common::Status InferenceSession::InitializeSubgraphSessions(Graph& graph, Sessio *subgraph_session_state, execution_providers_, kernel_registry_manager_); const auto implicit_inputs = node.ImplicitInputDefs(); - ORT_RETURN_IF_ERROR(initializer.CreatePlan(&node, &implicit_inputs, + ORT_RETURN_IF_ERROR_SESSIONID_(initializer.CreatePlan(&node, &implicit_inputs, session_options_.enable_sequential_execution)); // LOGS(*session_logger_, VERBOSE) << std::make_pair(subgraph_info.session_state->GetExecutionPlan(), @@ -458,10 +463,10 @@ common::Status InferenceSession::InitializeSubgraphSessions(Graph& graph, Sessio auto* p_op_kernel = session_state.GetMutableKernel(node.Index()); ORT_ENFORCE(p_op_kernel); auto& control_flow_kernel = dynamic_cast(*p_op_kernel); - ORT_RETURN_IF_ERROR(control_flow_kernel.SetupSubgraphExecutionInfo(session_state, name, *subgraph_session_state)); + ORT_RETURN_IF_ERROR_SESSIONID_(control_flow_kernel.SetupSubgraphExecutionInfo(session_state, name, *subgraph_session_state)); // recurse - ORT_RETURN_IF_ERROR(InitializeSubgraphSessions(subgraph, *subgraph_session_state)); + ORT_RETURN_IF_ERROR_SESSIONID_(InitializeSubgraphSessions(subgraph, *subgraph_session_state)); } } @@ -490,7 +495,7 @@ common::Status InferenceSession::Initialize() { LOGS(*session_logger_, INFO) << "Adding default CPU execution provider."; CPUExecutionProviderInfo epi{session_options_.enable_cpu_mem_arena}; auto p_cpu_exec_provider = onnxruntime::make_unique(epi); - ORT_RETURN_IF_ERROR(RegisterExecutionProvider(std::move(p_cpu_exec_provider))); + ORT_RETURN_IF_ERROR_SESSIONID_(RegisterExecutionProvider(std::move(p_cpu_exec_provider))); } if (!session_options_.enable_sequential_execution && @@ -515,37 +520,37 @@ common::Status InferenceSession::Initialize() { // The 1st ones should have already been registered via session-level API into KernelRegistryManager. // // Register 2nd registries into KernelRegistryManager. - ORT_RETURN_IF_ERROR(kernel_registry_manager_.RegisterKernels(execution_providers_)); + ORT_RETURN_IF_ERROR_SESSIONID_(kernel_registry_manager_.RegisterKernels(execution_providers_)); SessionStateInitializer session_initializer(session_options_.enable_mem_pattern, model_location_, graph, session_state_, execution_providers_, kernel_registry_manager_); // create SessionState for subgraphs as it's needed by the transformers - ORT_RETURN_IF_ERROR(CreateSubgraphSessionState(graph, session_state_)); + ORT_RETURN_IF_ERROR_SESSIONID_(CreateSubgraphSessionState(graph, session_state_)); // apply any transformations to the main graph and any subgraphs - ORT_RETURN_IF_ERROR(TransformGraph(graph, graph_transformation_mgr_, + ORT_RETURN_IF_ERROR_SESSIONID_(TransformGraph(graph, graph_transformation_mgr_, execution_providers_, kernel_registry_manager_, insert_cast_transformer_, session_state_)); // now that all the transforms are done, call Resolve on the main graph. this will recurse into the subgraphs. - ORT_RETURN_IF_ERROR(graph.Resolve()); + ORT_RETURN_IF_ERROR_SESSIONID_(graph.Resolve()); if (!session_options_.optimized_model_filepath.empty()) { if (session_options_.graph_optimization_level < TransformerLevel::Level3) { // Serialize optimized ONNX model. - ORT_RETURN_IF_ERROR(Model::Save(*model_, session_options_.optimized_model_filepath)); + ORT_RETURN_IF_ERROR_SESSIONID_(Model::Save(*model_, session_options_.optimized_model_filepath)); } else { LOGS(*session_logger_, WARNING) << "Serializing Optimized ONNX model with Graph Optimization" " level greater than 2 is not supported."; } } - ORT_RETURN_IF_ERROR(session_initializer.CreatePlan(nullptr, nullptr, session_options_.enable_sequential_execution)); + ORT_RETURN_IF_ERROR_SESSIONID_(session_initializer.CreatePlan(nullptr, nullptr, session_options_.enable_sequential_execution)); // handle any subgraphs - ORT_RETURN_IF_ERROR(InitializeSubgraphSessions(graph, session_state_)); + ORT_RETURN_IF_ERROR_SESSIONID_(InitializeSubgraphSessions(graph, session_state_)); is_inited_ = true; LOGS(*session_logger_, INFO) << "Session successfully initialized."; @@ -654,17 +659,17 @@ common::Status InferenceSession::ValidateInputs(const std::vector& auto expected_element_type = expected_type->AsTensorType()->GetElementType(); auto input_element_type = input_ml_value.Get().DataType(); - ORT_RETURN_IF_ERROR(CheckTypes(input_element_type, expected_element_type)); + ORT_RETURN_IF_ERROR_SESSIONID_(CheckTypes(input_element_type, expected_element_type)); // check for shape const auto& expected_shape = iter->second.tensor_shape; if (expected_shape.NumDimensions() > 0) { const auto& input_shape = input_ml_value.Get().Shape(); - ORT_RETURN_IF_ERROR(CheckShapes(feed_name, input_shape, expected_shape)); + ORT_RETURN_IF_ERROR_SESSIONID_(CheckShapes(feed_name, input_shape, expected_shape)); } } else { auto input_type = input_ml_value.Type(); - ORT_RETURN_IF_ERROR(CheckTypes(input_type, expected_type)); + ORT_RETURN_IF_ERROR_SESSIONID_(CheckTypes(input_type, expected_type)); } } @@ -715,8 +720,8 @@ Status InferenceSession::Run(const RunOptions& run_options, const std::vector global_session_id_; + uint32_t session_id_; }; } // namespace onnxruntime From c380add0ecef6be2cfa470ea7c8d6f5064d6aaee Mon Sep 17 00:00:00 2001 From: Paul McDaniel Date: Fri, 11 Oct 2019 13:35:00 -0700 Subject: [PATCH 03/14] next iteration --- onnxruntime/core/platform/telemetry.h | 6 ++--- .../core/platform/windows/telemetry.cc | 6 ++--- onnxruntime/core/platform/windows/telemetry.h | 6 ++--- onnxruntime/core/session/inference_session.cc | 23 ++++++++++++------- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/onnxruntime/core/platform/telemetry.h b/onnxruntime/core/platform/telemetry.h index 3cb6f0f5b52c5..1472edb715f30 100644 --- a/onnxruntime/core/platform/telemetry.h +++ b/onnxruntime/core/platform/telemetry.h @@ -31,10 +31,10 @@ class Telemetry { virtual void LogSessionCreation(uint32_t sessionId, int64_t irVersion, const std::string& modelProducerName, const std::string& modelProducerVersion,const std::string& modelDomain, - const std::vector& modelOpsetImports, uint32_t modelPrecision, - const std::string& modelGraphName, const std::string& modelGraphVersion, + const std::unordered_map& domainToVersionMap, + const std::string& modelGraphName, const std::unordered_map& modelMetaData, - bool modelFromStream, const std::string& executionProviders) const = 0; + const std::string& loadedFrom, const std::vector& executionProviderIds) const = 0; virtual void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, const char* function, uint32_t line) const = 0; diff --git a/onnxruntime/core/platform/windows/telemetry.cc b/onnxruntime/core/platform/windows/telemetry.cc index 6a70e9e9e5d08..7287952399175 100644 --- a/onnxruntime/core/platform/windows/telemetry.cc +++ b/onnxruntime/core/platform/windows/telemetry.cc @@ -79,10 +79,10 @@ void WindowsTelemetry::LogProcessInfo() const { void WindowsTelemetry::LogSessionCreation(uint32_t sessionId, int64_t irVersion, const std::string& modelProducerName, const std::string& modelProducerVersion, const std::string& modelDomain, - const std::vector& modelOpsetImports, uint32_t modelPrecision, - const std::string& modelGraphName, const std::string& modelGraphVersion, + const std::unordered_map& domainToVersionMap, + const std::string& modelGraphName, const std::unordered_map& modelMetaData, - bool modelFromStream, const std::string& executionProviders) const { + const std::string& loadedFrom, const std::vector& executionProviderIds) const { } void WindowsTelemetry::LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, diff --git a/onnxruntime/core/platform/windows/telemetry.h b/onnxruntime/core/platform/windows/telemetry.h index a7c5ce67a475c..9784010bc2462 100644 --- a/onnxruntime/core/platform/windows/telemetry.h +++ b/onnxruntime/core/platform/windows/telemetry.h @@ -33,10 +33,10 @@ class WindowsTelemetry : public Telemetry { void LogSessionCreation(uint32_t sessionId, int64_t irVersion, const std::string& modelProducerName, const std::string& modelProducerVersion, const std::string& modelDomain, - const std::vector& modelOpsetImports, uint32_t modelPrecision, - const std::string& modelGraphName, const std::string& modelGraphVersion, + const std::unordered_map& domainToVersionMap, + const std::string& modelGraphName, const std::unordered_map& modelMetaData, - bool modelFromStream, const std::string& executionProviders) const override; + const std::string& loadedFrom, const std::vector& executionProviderIds) const override; void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, const char* function, uint32_t line) const override; diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 45adc7e674fa7..8785b41db0a80 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -121,7 +121,7 @@ InferenceSession::InferenceSession(const SessionOptions& session_options, if (session_options.enable_profiling) { StartProfiling(session_options.profile_file_prefix); } - + // a monotonically increasing session id for use in telemetry session_id_ = global_session_id_.fetch_add(1); } @@ -221,6 +221,13 @@ common::Status InferenceSession::Load(std::functionIrVersion(), model_->ProducerName(), model_->ProducerVersion(), + model_->Domain(), model_->MainGraph().DomainToVersionMap(), model_->MainGraph().Name(), + model_->MetaData(), event_name, execution_providers_.GetIds()); + } catch (const std::exception& ex) { status = Status(common::ONNXRUNTIME, common::FAIL, "Exception during loading: " + std::string(ex.what())); } catch (...) { @@ -406,9 +413,9 @@ common::Status InferenceSession::CreateSubgraphSessionState(Graph& graph, Sessio ORT_ENFORCE(subgraph, "Main Graph instance should have populated all subgraphs when being resolved."); auto subgraph_session_state = onnxruntime::make_unique(execution_providers_, - session_state.GetEnableMemoryPattern(), - session_state.GetThreadPool(), - session_state.GetInterOpThreadPool()); + session_state.GetEnableMemoryPattern(), + session_state.GetThreadPool(), + session_state.GetInterOpThreadPool()); subgraph_session_state->SetProfiler(session_profiler_); subgraph_session_state->SetLogger(*session_logger_); // Pass data transfer manager to subgraph. @@ -454,7 +461,7 @@ common::Status InferenceSession::InitializeSubgraphSessions(Graph& graph, Sessio const auto implicit_inputs = node.ImplicitInputDefs(); ORT_RETURN_IF_ERROR_SESSIONID_(initializer.CreatePlan(&node, &implicit_inputs, - session_options_.enable_sequential_execution)); + session_options_.enable_sequential_execution)); // LOGS(*session_logger_, VERBOSE) << std::make_pair(subgraph_info.session_state->GetExecutionPlan(), // &*subgraph_info.session_state); @@ -530,9 +537,9 @@ common::Status InferenceSession::Initialize() { // apply any transformations to the main graph and any subgraphs ORT_RETURN_IF_ERROR_SESSIONID_(TransformGraph(graph, graph_transformation_mgr_, - execution_providers_, kernel_registry_manager_, - insert_cast_transformer_, - session_state_)); + execution_providers_, kernel_registry_manager_, + insert_cast_transformer_, + session_state_)); // now that all the transforms are done, call Resolve on the main graph. this will recurse into the subgraphs. ORT_RETURN_IF_ERROR_SESSIONID_(graph.Resolve()); From f9ea7a45a0227be08649c15680ee9c06002178ce Mon Sep 17 00:00:00 2001 From: Paul McDaniel Date: Fri, 11 Oct 2019 17:08:54 -0700 Subject: [PATCH 04/14] next round of changes. coded SessionCreation --- .../core/platform/windows/telemetry.cc | 89 +++++++++++++++++-- onnxruntime/core/platform/windows/telemetry.h | 14 +-- onnxruntime/core/session/inference_session.cc | 17 ++++ onnxruntime/core/session/inference_session.h | 10 ++- 4 files changed, 111 insertions(+), 19 deletions(-) diff --git a/onnxruntime/core/platform/windows/telemetry.cc b/onnxruntime/core/platform/windows/telemetry.cc index 7287952399175..892810003cda5 100644 --- a/onnxruntime/core/platform/windows/telemetry.cc +++ b/onnxruntime/core/platform/windows/telemetry.cc @@ -77,15 +77,74 @@ void WindowsTelemetry::LogProcessInfo() const { process_info_logged = true; } -void WindowsTelemetry::LogSessionCreation(uint32_t sessionId, int64_t irVersion, const std::string& modelProducerName, - const std::string& modelProducerVersion, const std::string& modelDomain, - const std::unordered_map& domainToVersionMap, - const std::string& modelGraphName, - const std::unordered_map& modelMetaData, - const std::string& loadedFrom, const std::vector& executionProviderIds) const { +void WindowsTelemetry::LogSessionCreation(uint32_t session_id, int64_t ir_version, const std::string& model_producer_name, + const std::string& model_producer_version, const std::string& model_domain, + const std::unordered_map& domain_to_version_map, + const std::string& model_graph_name, + const std::unordered_map& model_metadata, + const std::string& loadedFrom, const std::vector& execution_provider_ids) const { + if (global_register_count_ == 0) + return; + + // build the strings we need + + std::string domain_to_verison_string; + bool first = true; + for (auto& i : domain_to_version_map) { + if (first) { + first = false; + } else { + domain_to_verison_string += ','; + } + domain_to_verison_string += i.first; + domain_to_verison_string += '='; + domain_to_verison_string += std::to_string(i.second); + } + + std::string model_metadata_string; + first = true; + for (auto& i : model_metadata) { + if (first) { + first = false; + } else { + model_metadata_string += ','; + } + model_metadata_string += i.first; + model_metadata_string += '='; + model_metadata_string += i.second; + } + + std::string execution_provider_string; + first = true; + for (auto& i : execution_provider_ids) { + if (first) { + first = false; + } else { + execution_provider_string += ','; + } + execution_provider_string += i; + } + + TraceLoggingWrite(telemetry_provider_handle, + "SessionCreation", + TraceLoggingBool(true, "UTCReplace_AppSessionGuid"), + TelemetryPrivacyDataTag(PDT_ProductAndServiceUsage), + TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES), + // Telemetry info + TraceLoggingUInt8(0, "schemaVersion"), + TraceLoggingUInt32(session_id, "sessionId"), + TraceLoggingInt64(ir_version, "irVersion"), + TraceLoggingString(model_producer_name.c_str(), "modelProducerName"), + TraceLoggingString(model_producer_version.c_str(), "modelProducerVersion"), + TraceLoggingString(model_domain.c_str(), "modelDomain"), + TraceLoggingString(domain_to_verison_string.c_str(), "domainToVersionMap"), + TraceLoggingString(model_graph_name.c_str(), "modelGraphName"), + TraceLoggingString(model_metadata_string.c_str(), "modelMetaData"), + TraceLoggingString(loadedFrom.c_str(), "loadedFrom"), + TraceLoggingString(execution_provider_string.c_str(), "executionProviderIds")); } -void WindowsTelemetry::LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, +void WindowsTelemetry::LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file, const char* function, uint32_t line) const { if (global_register_count_ == 0) return; @@ -96,7 +155,7 @@ void WindowsTelemetry::LogRuntimeError(uint32_t sessionId, const common::Status& TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES), // Telemetry info TraceLoggingUInt8(0, "schemaVersion"), - TraceLoggingUInt32(sessionId, "sessionId"), + TraceLoggingUInt32(session_id, "sessionId"), TraceLoggingUInt32(status.Code(), "errorCode"), TraceLoggingUInt32(status.Category(), "errorCategory"), TraceLoggingString(status.ErrorMessage().c_str(), "errorMessage"), @@ -105,7 +164,19 @@ void WindowsTelemetry::LogRuntimeError(uint32_t sessionId, const common::Status& TraceLoggingInt32(line, "line")); } -void WindowsTelemetry::LogRuntimePerf(uint32_t sessionId, uint32_t runTotalTimeMs) const { +void WindowsTelemetry::LogRuntimePerf(uint32_t session_id, uint32_t total_runs_since_last, int64_t total_run_duration_since_last) const { + if (global_register_count_ == 0) + return; + + TraceLoggingWrite(telemetry_provider_handle, + "RuntimePerf", + TelemetryPrivacyDataTag(PDT_ProductAndServicePerformance), + TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES), + // Telemetry info + TraceLoggingUInt8(0, "schemaVersion"), + TraceLoggingUInt32(session_id, "sessionId"), + TraceLoggingUInt32(total_runs_since_last, "totalRuns"), + TraceLoggingInt64(total_run_duration_since_last, "totalRunDuration")); } } // namespace onnxruntime diff --git a/onnxruntime/core/platform/windows/telemetry.h b/onnxruntime/core/platform/windows/telemetry.h index 9784010bc2462..452bee4e76556 100644 --- a/onnxruntime/core/platform/windows/telemetry.h +++ b/onnxruntime/core/platform/windows/telemetry.h @@ -31,17 +31,17 @@ class WindowsTelemetry : public Telemetry { void LogProcessInfo() const override; - void LogSessionCreation(uint32_t sessionId, int64_t irVersion, const std::string& modelProducerName, - const std::string& modelProducerVersion, const std::string& modelDomain, - const std::unordered_map& domainToVersionMap, - const std::string& modelGraphName, - const std::unordered_map& modelMetaData, - const std::string& loadedFrom, const std::vector& executionProviderIds) const override; + void LogSessionCreation(uint32_t session_id, int64_t ir_version, const std::string& model_producer_name, + const std::string& model_producer_version, const std::string& model_domain, + const std::unordered_map& domain_to_version_map, + const std::string& model_graph_name, + const std::unordered_map& model_metadata, + const std::string& loadedFrom, const std::vector& execution_provider_ids) const override; void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, const char* function, uint32_t line) const override; - void LogRuntimePerf(uint32_t sessionId, uint32_t runTotalTimeMs) const override; + void LogRuntimePerf(uint32_t session_id, uint32_t total_runs_since_last, int64_t total_run_duration_since_last) const override; private: static OrtMutex mutex_; diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 8785b41db0a80..d62ac36179463 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -770,6 +770,23 @@ Status InferenceSession::Run(const RunOptions& run_options, const std::vector kDurationBetweenSending) { + // send the telemetry + const Env& env = Env::Default(); + env.GetTelemetryProvider().LogRuntimePerf(session_id_, total_runs_since_last_, total_run_duration_since_last_); + // reset counters + time_sent_last_ = std::chrono::high_resolution_clock::now(); + total_runs_since_last_ = 0; + total_run_duration_since_last_ = 0; + } + + // send out profiling events (optional) if (session_profiler_.IsEnabled()) { session_profiler_.EndTimeAndRecordEvent(profiling::SESSION_EVENT, "model_run", tp); } diff --git a/onnxruntime/core/session/inference_session.h b/onnxruntime/core/session/inference_session.h index 17e19fe9bf293..e61fe3fda7523 100644 --- a/onnxruntime/core/session/inference_session.h +++ b/onnxruntime/core/session/inference_session.h @@ -491,8 +491,12 @@ class InferenceSession { InterOpDomains interop_domains_; #endif - // a monotonically increasing session id for use in telemetry - static std::atomic global_session_id_; - uint32_t session_id_; + // used to support platform telemetry + static std::atomic global_session_id_; // a monotonically increasing session id + uint32_t session_id_; // the current session's id + uint32_t total_runs_since_last_; // the total number of Run() calls since the last report + long long total_run_duration_since_last_; // the total duration (us) of Run() calls since the last report + TimePoint time_sent_last_; // the TimePoint of the last report + const long long kDurationBetweenSending = 1000* 1000 * 60 * 10; // duration in (us). send a report every 10 mins }; } // namespace onnxruntime From ed8c95573b01d1203c7df96841110578adbcc626 Mon Sep 17 00:00:00 2001 From: Paul McDaniel Date: Tue, 15 Oct 2019 12:01:19 -0700 Subject: [PATCH 05/14] fixed build break in POSIX. added global enable/disable of telemetry to c and cxx api add docs. --- README.md | 7 +++++ docs/C_API.md | 9 ++++++ docs/Privacy.md | 4 +++ .../core/session/onnxruntime_c_api.h | 4 +++ .../core/session/onnxruntime_cxx_api.h | 3 ++ .../core/session/onnxruntime_cxx_inline.h | 10 +++++++ onnxruntime/core/platform/posix/env.cc | 6 ++++ onnxruntime/core/platform/telemetry.cc | 26 +++++++++++++++-- onnxruntime/core/platform/telemetry.h | 29 ++++++++++--------- .../core/platform/windows/telemetry.cc | 18 +++++++++--- onnxruntime/core/platform/windows/telemetry.h | 12 ++++++-- onnxruntime/core/session/onnxruntime_c_api.cc | 22 ++++++++++++++ onnxruntime/core/session/ort_apis.h | 2 ++ 13 files changed, 130 insertions(+), 22 deletions(-) create mode 100644 docs/Privacy.md diff --git a/README.md b/README.md index 6804b197af642..f52af81eac44c 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,8 @@ **[Contributions and Feedback](#contribute)** **[License](#license)** + +**[Data/Telemetry](#Data/Telemetry)** *** # Key Features ## Run any ONNX model @@ -203,3 +205,8 @@ or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any addi *** # License [MIT License](LICENSE) + +# Data/Telemetry +This project collects usage data and sends it to Microsoft to help improve our products and services. See the [privacy statement](docs/Privacy.md) for more details. + +For more information on telemetry implementation see the [developer guide](docs/C_API.md#Telemetry). \ No newline at end of file diff --git a/docs/C_API.md b/docs/C_API.md index e68a40fe4014d..3a69163289451 100644 --- a/docs/C_API.md +++ b/docs/C_API.md @@ -29,3 +29,12 @@ The example below shows a sample run using the SqueezeNet model from ONNX model zoo, including dynamically reading model inputs, outputs, shape and type information, as well as running a sample vector and fetching the resulting class probabilities for inspection. * [../csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp](../csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp) + +# Telemetry +This project collects usage data and sends it to Microsoft to help improve our products and services. Note however that no data collection is performed by default when using your private builds + +Telemetry is turned on by default when using the Official Builds. This is implemented via 'Platform Telemetry' per vendor platform providers (see telemetry.h). + +The Windows provider uses the [TraceLogging](https://docs.microsoft.com/en-us/windows/win32/tracelogging/trace-logging-about) API for its implementation. + +You can turn this off using the DisableTelemetryEvents() API. \ No newline at end of file diff --git a/docs/Privacy.md b/docs/Privacy.md new file mode 100644 index 0000000000000..808980328eb3c --- /dev/null +++ b/docs/Privacy.md @@ -0,0 +1,4 @@ +# Data Collection +The software may collect information about you and your use of the software and send it to Microsoft. Microsoft may use this information to provide services and improve our products and services. You may turn off the telemetry as described in the repository. There are also some features in the software that may enable you and Microsoft to collect data from users of your applications. If you use these features, you must comply with applicable law, including providing appropriate notices to users of your applications together with a copy of Microsoft's privacy statement. Our privacy statement is located at https://go.microsoft.com/fwlink/?LinkID=824704. You can learn more about data collection and use in the help documentation and our privacy statement. Your use of the software operates as your consent to these practices. + +For more information on telemetry implementation see the [developer guide](C_API.md#Telemetry). \ No newline at end of file diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index b911d7f58d432..aa7a2010cfe53 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -230,6 +230,10 @@ struct OrtApi { _In_ const char* logid, _Outptr_ OrtEnv** out)NO_EXCEPTION; + // Platform telemetry events are on by default since they are lightweight. You can manually turn them off. + OrtStatus*(ORT_API_CALL* EnableTelemetryEvents)(_In_ const OrtEnv* env)NO_EXCEPTION; + OrtStatus*(ORT_API_CALL* DisableTelemetryEvents)(_In_ const OrtEnv* env)NO_EXCEPTION; + // TODO: document the path separator convention? '/' vs '\' // TODO: should specify the access characteristics of model_path. Is this read only during the // execution of OrtCreateSession, or does the OrtSession retain a handle to the file/directory diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index 5e57b928df9e2..ded275c1a072b 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -107,6 +107,9 @@ struct Env : Base { Env(OrtLoggingLevel default_logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param); explicit Env(OrtEnv* p) : Base{p} {} + Env& EnableTelemetryEvents(); + Env& DisableTelemetryEvents(); + static const OrtApi* s_api; }; diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h index e7569b5739844..230f5eb509dda 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h @@ -93,6 +93,16 @@ inline Env::Env(OrtLoggingLevel default_warning_level, const char* logid, OrtLog ThrowOnError(g_api->CreateEnvWithCustomLogger(logging_function, logger_param, default_warning_level, logid, &p_)); } +inline Env& Env::EnableTelemetryEvents() { + ThrowOnError(g_api->EnableTelemetryEvents(p_)); + return *this; +} + +inline Env& Env::DisableTelemetryEvents() { + ThrowOnError(g_api->DisableTelemetryEvents(p_)); + return *this; +} + inline CustomOpDomain::CustomOpDomain(const char* domain) { ThrowOnError(g_api->CreateCustomOpDomain(domain, &p_)); } diff --git a/onnxruntime/core/platform/posix/env.cc b/onnxruntime/core/platform/posix/env.cc index 2d34bea1e2bfc..ff42d66c81626 100644 --- a/onnxruntime/core/platform/posix/env.cc +++ b/onnxruntime/core/platform/posix/env.cc @@ -266,8 +266,14 @@ class PosixEnv : public Env { return filename; } + // \brief returns a provider that will handle telemetry on the current platform + const Telemetry& GetTelemetryProvider() const override { + return telemetryProvider_; + } + private: PosixEnv() = default; + Telemetry telemetryProvider_; }; } // namespace diff --git a/onnxruntime/core/platform/telemetry.cc b/onnxruntime/core/platform/telemetry.cc index 84eafa0500595..3e837de5c8bfc 100644 --- a/onnxruntime/core/platform/telemetry.cc +++ b/onnxruntime/core/platform/telemetry.cc @@ -6,8 +6,6 @@ namespace onnxruntime { -Telemetry::Telemetry() = default; - void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, const char* function, uint32_t line) { @@ -15,5 +13,29 @@ void LogRuntimeError(uint32_t sessionId, const common::Status& status, const cha env.GetTelemetryProvider().LogRuntimeError(sessionId, status, file, function, line); } +void Telemetry::EnableTelemetryEvents() const { +} + +void Telemetry::DisableTelemetryEvents() const { +} + +void Telemetry::LogProcessInfo() const { +} + +void Telemetry::LogSessionCreation(uint32_t session_id, int64_t ir_version, const std::string& model_producer_name, + const std::string& model_producer_version, const std::string& model_domain, + const std::unordered_map& domain_to_version_map, + const std::string& model_graph_name, + const std::unordered_map& model_metadata, + const std::string& loadedFrom, const std::vector& execution_provider_ids) const { +} + +void Telemetry::LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file, + const char* function, uint32_t line) const { +} + +void Telemetry::LogRuntimePerf(uint32_t session_id, uint32_t total_runs_since_last, int64_t total_run_duration_since_last) const { +} + } // namespace onnxruntime diff --git a/onnxruntime/core/platform/telemetry.h b/onnxruntime/core/platform/telemetry.h index 1472edb715f30..323e1463e9c3a 100644 --- a/onnxruntime/core/platform/telemetry.h +++ b/onnxruntime/core/platform/telemetry.h @@ -25,25 +25,28 @@ namespace onnxruntime { */ class Telemetry { public: + // don't create these, use Env::GetTelemetryProvider() instead + // this constructor is made public so that other platform Env providers can + // use this base class as a "stub" implementation + Telemetry() = default; virtual ~Telemetry() = default; - virtual void LogProcessInfo() const = 0; + virtual void EnableTelemetryEvents() const; + virtual void DisableTelemetryEvents() const; - virtual void LogSessionCreation(uint32_t sessionId, int64_t irVersion, const std::string& modelProducerName, - const std::string& modelProducerVersion,const std::string& modelDomain, - const std::unordered_map& domainToVersionMap, - const std::string& modelGraphName, - const std::unordered_map& modelMetaData, - const std::string& loadedFrom, const std::vector& executionProviderIds) const = 0; + virtual void LogProcessInfo() const; - virtual void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, - const char* function, uint32_t line) const = 0; + virtual void LogSessionCreation(uint32_t session_id, int64_t ir_version, const std::string& model_producer_name, + const std::string& model_producer_version, const std::string& model_domain, + const std::unordered_map& domain_to_version_map, + const std::string& model_graph_name, + const std::unordered_map& model_metadata, + const std::string& loadedFrom, const std::vector& execution_provider_ids) const; - virtual void LogRuntimePerf(uint32_t sessionId, uint32_t runTotalTimeMs) const = 0; + virtual void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, + const char* function, uint32_t line) const; - protected: - // don't create these, use Env::GetTelemetryProvider() - Telemetry(); + virtual void LogRuntimePerf(uint32_t session_id, uint32_t total_runs_since_last, int64_t total_run_duration_since_last) const; private: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Telemetry); diff --git a/onnxruntime/core/platform/windows/telemetry.cc b/onnxruntime/core/platform/windows/telemetry.cc index 892810003cda5..d83868ec3495f 100644 --- a/onnxruntime/core/platform/windows/telemetry.cc +++ b/onnxruntime/core/platform/windows/telemetry.cc @@ -32,6 +32,8 @@ TRACELOGGING_DEFINE_PROVIDER(telemetry_provider_handle, "Microsoft.ML.ONNXRuntim OrtMutex WindowsTelemetry::mutex_; uint32_t WindowsTelemetry::global_register_count_ = 0; +bool WindowsTelemetry::enabled_ = false; + WindowsTelemetry::WindowsTelemetry() { std::lock_guard lock(mutex_); @@ -54,8 +56,16 @@ WindowsTelemetry::~WindowsTelemetry() { } } +void WindowsTelemetry::EnableTelemetryEvents() const { + enabled_ = true; +} + +void WindowsTelemetry::DisableTelemetryEvents() const { + enabled_ = false; +} + void WindowsTelemetry::LogProcessInfo() const { - if (global_register_count_ == 0) + if (global_register_count_ == 0 || enabled_ == false) return; static std::atomic process_info_logged; @@ -83,7 +93,7 @@ void WindowsTelemetry::LogSessionCreation(uint32_t session_id, int64_t ir_versio const std::string& model_graph_name, const std::unordered_map& model_metadata, const std::string& loadedFrom, const std::vector& execution_provider_ids) const { - if (global_register_count_ == 0) + if (global_register_count_ == 0 || enabled_ == false) return; // build the strings we need @@ -146,7 +156,7 @@ void WindowsTelemetry::LogSessionCreation(uint32_t session_id, int64_t ir_versio void WindowsTelemetry::LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file, const char* function, uint32_t line) const { - if (global_register_count_ == 0) + if (global_register_count_ == 0 || enabled_ == false) return; TraceLoggingWrite(telemetry_provider_handle, @@ -165,7 +175,7 @@ void WindowsTelemetry::LogRuntimeError(uint32_t session_id, const common::Status } void WindowsTelemetry::LogRuntimePerf(uint32_t session_id, uint32_t total_runs_since_last, int64_t total_run_duration_since_last) const { - if (global_register_count_ == 0) + if (global_register_count_ == 0 || enabled_ == false) return; TraceLoggingWrite(telemetry_provider_handle, diff --git a/onnxruntime/core/platform/windows/telemetry.h b/onnxruntime/core/platform/windows/telemetry.h index 452bee4e76556..bbc31acde37a6 100644 --- a/onnxruntime/core/platform/windows/telemetry.h +++ b/onnxruntime/core/platform/windows/telemetry.h @@ -6,12 +6,14 @@ #include "core/platform/ort_mutex.h" #include -// Note: this needs to get moved to a release pipeline still (paulm) // *** +// platform specific control bits +#ifndef TraceLoggingOptionMicrosoftTelemetry #define TraceLoggingOptionMicrosoftTelemetry() \ - TraceLoggingOptionGroup(0x4f50731a, 0x89cf, 0x4782, 0xb3, 0xe0, 0xdc, 0xe8, 0xc9, 0x4, 0x76, 0xba) + TraceLoggingOptionGroup(0000000000, 00000, 00000, 0000, 0000, 0000, 0000, 0000, 000, 0000, 0000) +#endif #define MICROSOFT_KEYWORD_MEASURES 0x0000400000000000 // Bit 46 -#define TelemetryPrivacyDataTag(tag) TraceLoggingUInt64((tag), "PartA_PrivTags") +#define TelemetryPrivacyDataTag(tag) TraceLoggingUInt64((tag), "PartA_PrivTags") #define PDT_ProductAndServicePerformance 0x0000000001000000u #define PDT_ProductAndServiceUsage 0x0000000002000000u // *** @@ -29,6 +31,9 @@ class WindowsTelemetry : public Telemetry { WindowsTelemetry(); ~WindowsTelemetry(); + void EnableTelemetryEvents() const override; + void DisableTelemetryEvents() const override; + void LogProcessInfo() const override; void LogSessionCreation(uint32_t session_id, int64_t ir_version, const std::string& model_producer_name, @@ -46,6 +51,7 @@ class WindowsTelemetry : public Telemetry { private: static OrtMutex mutex_; static uint32_t global_register_count_; + static bool enabled_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index 000e27638cc19..fe9557b6bd4ad 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -131,6 +131,25 @@ ORT_API_STATUS_IMPL(OrtApis::CreateEnv, OrtLoggingLevel default_warning_level, API_IMPL_END } +// enable platform telemetry +ORT_API_STATUS_IMPL(OrtApis::EnableTelemetryEvents, _In_ const OrtEnv* ort_env) { + API_IMPL_BEGIN + // note telemetry is controlled via the platform Env object, not the OrtEnv object instance + const Env& env = Env::Default(); + env.GetTelemetryProvider().EnableTelemetryEvents(); + return nullptr; + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtApis::DisableTelemetryEvents, _In_ const OrtEnv* ort_env) { + API_IMPL_BEGIN + // note telemetry is controlled via the platform Env object, not the OrtEnv object instance + const Env& env = Env::Default(); + env.GetTelemetryProvider().DisableTelemetryEvents(); + return nullptr; + API_IMPL_END +} + template OrtStatus* CreateTensorImpl(const int64_t* shape, size_t shape_len, OrtAllocator* allocator, std::unique_ptr* out) { @@ -1124,6 +1143,9 @@ static constexpr OrtApi ort_api_1 = { &OrtApis::CreateEnv, &OrtApis::CreateEnvWithCustomLogger, + &OrtApis::EnableTelemetryEvents, + &OrtApis::DisableTelemetryEvents, + &OrtApis::CreateSession, &OrtApis::CreateSessionFromArray, &OrtApis::Run, diff --git a/onnxruntime/core/session/ort_apis.h b/onnxruntime/core/session/ort_apis.h index 3a97753b88fea..a20145b7472ed 100644 --- a/onnxruntime/core/session/ort_apis.h +++ b/onnxruntime/core/session/ort_apis.h @@ -21,6 +21,8 @@ const char* ORT_API_CALL GetErrorMessage(_In_ const OrtStatus* status) NO_EXCEPT ORT_API_STATUS_IMPL(CreateEnv, OrtLoggingLevel default_logging_level, _In_ const char* logid, _Outptr_ OrtEnv** out) ORT_ALL_ARGS_NONNULL; ORT_API_STATUS_IMPL(CreateEnvWithCustomLogger, OrtLoggingFunction logging_function, _In_opt_ void* logger_param, OrtLoggingLevel default_warning_level, _In_ const char* logid, _Outptr_ OrtEnv** out); +ORT_API_STATUS_IMPL(EnableTelemetryEvents, _In_ const OrtEnv* env); +ORT_API_STATUS_IMPL(DisableTelemetryEvents, _In_ const OrtEnv* env); ORT_API_STATUS_IMPL(CreateSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); From e16a24d6f1674869c54b98324fe9dabd977c0fed Mon Sep 17 00:00:00 2001 From: Paul McDaniel Date: Tue, 15 Oct 2019 16:16:08 -0700 Subject: [PATCH 06/14] fixed the C# projection with the new api additions. --- csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs index 4c811e9ae340d..d8d67c7d72798 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs @@ -22,6 +22,8 @@ public struct OrtApi public IntPtr GetErrorMessage; public IntPtr CreateEnv; public IntPtr CreateEnvWithCustomLogger; + public IntPtr EnableTelemetryEvents; + public IntPtr DisableTelemetryEvents; public IntPtr CreateSession; public IntPtr CreateSessionFromArray; public IntPtr Run; From c2a47fa82a8aa7931c9b783d29711ea01b1f31d5 Mon Sep 17 00:00:00 2001 From: Paul McDaniel Date: Tue, 15 Oct 2019 16:22:49 -0700 Subject: [PATCH 07/14] fixed posix build failures --- onnxruntime/core/platform/telemetry.cc | 28 +++++++++++++++---- onnxruntime/core/session/inference_session.cc | 2 +- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/onnxruntime/core/platform/telemetry.cc b/onnxruntime/core/platform/telemetry.cc index 3e837de5c8bfc..7c587a1b5d469 100644 --- a/onnxruntime/core/platform/telemetry.cc +++ b/onnxruntime/core/platform/telemetry.cc @@ -23,18 +23,36 @@ void Telemetry::LogProcessInfo() const { } void Telemetry::LogSessionCreation(uint32_t session_id, int64_t ir_version, const std::string& model_producer_name, - const std::string& model_producer_version, const std::string& model_domain, - const std::unordered_map& domain_to_version_map, - const std::string& model_graph_name, - const std::unordered_map& model_metadata, - const std::string& loadedFrom, const std::vector& execution_provider_ids) const { + const std::string& model_producer_version, const std::string& model_domain, + const std::unordered_map& domain_to_version_map, + const std::string& model_graph_name, + const std::unordered_map& model_metadata, + const std::string& loadedFrom, const std::vector& execution_provider_ids) const { + ORT_UNUSED_PARAMETER(session_id); + ORT_UNUSED_PARAMETER(ir_version); + ORT_UNUSED_PARAMETER(model_producer_name); + ORT_UNUSED_PARAMETER(model_producer_version); + ORT_UNUSED_PARAMETER(model_domain); + ORT_UNUSED_PARAMETER(domain_to_version_map); + ORT_UNUSED_PARAMETER(model_graph_name); + ORT_UNUSED_PARAMETER(model_metadata); + ORT_UNUSED_PARAMETER(loadedFrom); + ORT_UNUSED_PARAMETER(execution_provider_ids); } void Telemetry::LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file, const char* function, uint32_t line) const { + ORT_UNUSED_PARAMETER(session_id); + ORT_UNUSED_PARAMETER(status); + ORT_UNUSED_PARAMETER(file); + ORT_UNUSED_PARAMETER(function); + ORT_UNUSED_PARAMETER(line); } void Telemetry::LogRuntimePerf(uint32_t session_id, uint32_t total_runs_since_last, int64_t total_run_duration_since_last) const { + ORT_UNUSED_PARAMETER(session_id); + ORT_UNUSED_PARAMETER(total_runs_since_last); + ORT_UNUSED_PARAMETER(total_run_duration_since_last); } } // namespace onnxruntime diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 716e6540bc1a7..007ebedf35674 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -95,7 +95,7 @@ inline std::basic_string GetCurrentTimeString() { } // namespace -std::atomic InferenceSession::global_session_id_ = 1; +std::atomic InferenceSession::global_session_id_{1}; InferenceSession::InferenceSession(const SessionOptions& session_options, logging::LoggingManager* logging_manager) From e90e6b4cc3688b326701e7b5102dea5d777b776e Mon Sep 17 00:00:00 2001 From: Paul McDaniel Date: Tue, 15 Oct 2019 17:16:01 -0700 Subject: [PATCH 08/14] another posix build break --- onnxruntime/core/session/onnxruntime_c_api.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index c1e405cc5404d..d2e743c0c497d 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -131,6 +131,7 @@ ORT_API_STATUS_IMPL(OrtApis::CreateEnv, OrtLoggingLevel default_warning_level, // enable platform telemetry ORT_API_STATUS_IMPL(OrtApis::EnableTelemetryEvents, _In_ const OrtEnv* ort_env) { API_IMPL_BEGIN + ORT_UNUSED_PARAMETER(ort_env); // note telemetry is controlled via the platform Env object, not the OrtEnv object instance const Env& env = Env::Default(); env.GetTelemetryProvider().EnableTelemetryEvents(); @@ -140,6 +141,7 @@ ORT_API_STATUS_IMPL(OrtApis::EnableTelemetryEvents, _In_ const OrtEnv* ort_env) ORT_API_STATUS_IMPL(OrtApis::DisableTelemetryEvents, _In_ const OrtEnv* ort_env) { API_IMPL_BEGIN + ORT_UNUSED_PARAMETER(ort_env); // note telemetry is controlled via the platform Env object, not the OrtEnv object instance const Env& env = Env::Default(); env.GetTelemetryProvider().DisableTelemetryEvents(); From 882f7abf924c4f2bdc5939e2c3ade5fc3dc86a7d Mon Sep 17 00:00:00 2001 From: Paul McDaniel Date: Wed, 16 Oct 2019 17:15:39 -0700 Subject: [PATCH 09/14] DML build breaks. --- include/onnxruntime/core/common/common.h | 4 ++-- .../core/providers/dml/GraphTransformers/bn_add_fusion.cc | 2 +- .../core/providers/dml/GraphTransformers/bn_mul_fusion.cc | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/onnxruntime/core/common/common.h b/include/onnxruntime/core/common/common.h index cb997b4cdd224..ef77582f77767 100644 --- a/include/onnxruntime/core/common/common.h +++ b/include/onnxruntime/core/common/common.h @@ -144,7 +144,7 @@ void LogRuntimeError(uint32_t sessionId, const common::Status& status, const cha do { \ auto _status = (expr); \ if ((!_status.IsOK())) { \ - LogRuntimeError(session_id, _status, __FILE__, __FUNCTION__, __LINE__); \ + ::onnxruntime::LogRuntimeError(session_id, _status, __FILE__, __FUNCTION__, __LINE__); \ return _status; \ } \ } while (0) @@ -156,7 +156,7 @@ void LogRuntimeError(uint32_t sessionId, const common::Status& status, const cha do { \ auto _status = (expr); \ if ((!_status.IsOK())) { \ - LogRuntimeError(0, _status, __FILE__, __FUNCTION__, __LINE__); \ + ::onnxruntime::LogRuntimeError(0, _status, __FILE__, __FUNCTION__, __LINE__); \ ORT_THROW(_status); \ } \ } while (0) diff --git a/onnxruntime/core/providers/dml/GraphTransformers/bn_add_fusion.cc b/onnxruntime/core/providers/dml/GraphTransformers/bn_add_fusion.cc index c879c4c186dbb..613b96fbe725a 100644 --- a/onnxruntime/core/providers/dml/GraphTransformers/bn_add_fusion.cc +++ b/onnxruntime/core/providers/dml/GraphTransformers/bn_add_fusion.cc @@ -97,7 +97,7 @@ bool BatchNormalizationAddFusion::SatisfyCondition(const Graph& graph, const Nod const auto& next_node = *node.OutputNodesBegin(); return !(!graph_utils::IsSupportedOptypeVersionAndDomain(next_node, "Add", {7}) || next_node.GetExecutionProviderType() != node.GetExecutionProviderType() || - next_node.GetInputEdgesCount() != 1 || graph.IsNodeOutputsInGraphOutputs(next_node)); + next_node.GetInputEdgesCount() != 1 || graph.GetNodeOutputsInGraphOutputs(next_node).empty() == false); } } // namespace onnxruntime diff --git a/onnxruntime/core/providers/dml/GraphTransformers/bn_mul_fusion.cc b/onnxruntime/core/providers/dml/GraphTransformers/bn_mul_fusion.cc index 808f66553ce86..c29f38b915841 100644 --- a/onnxruntime/core/providers/dml/GraphTransformers/bn_mul_fusion.cc +++ b/onnxruntime/core/providers/dml/GraphTransformers/bn_mul_fusion.cc @@ -109,7 +109,7 @@ bool BatchNormalizationMulFusion::SatisfyCondition(const Graph& graph, const Nod const auto& next_node = *node.OutputNodesBegin(); return !(!graph_utils::IsSupportedOptypeVersionAndDomain(next_node, "Mul", {7}) || - next_node.GetInputEdgesCount() != 1 || graph.IsNodeOutputsInGraphOutputs(next_node) || + next_node.GetInputEdgesCount() != 1 || graph.GetNodeOutputsInGraphOutputs(next_node).empty() == false || next_node.GetExecutionProviderType() != node.GetExecutionProviderType()); } From 0c453702c087e0ac5355f2bf5bd4f34c76a2bfba Mon Sep 17 00:00:00 2001 From: Paul McDaniel Date: Wed, 16 Oct 2019 17:39:17 -0700 Subject: [PATCH 10/14] code style and PR comments. --- include/onnxruntime/core/common/common.h | 2 +- onnxruntime/core/platform/telemetry.h | 2 +- onnxruntime/core/platform/windows/env.cc | 4 ++-- onnxruntime/core/platform/windows/telemetry.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/onnxruntime/core/common/common.h b/include/onnxruntime/core/common/common.h index ef77582f77767..997b17e9ab325 100644 --- a/include/onnxruntime/core/common/common.h +++ b/include/onnxruntime/core/common/common.h @@ -75,7 +75,7 @@ using common::Status; std::vector GetStackTrace(); // these is a helper function that gets defined by platform/Telemetry -void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, +void LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file, const char* function, uint32_t line); // __PRETTY_FUNCTION__ isn't a macro on gcc, so use a check for _MSC_VER diff --git a/onnxruntime/core/platform/telemetry.h b/onnxruntime/core/platform/telemetry.h index 323e1463e9c3a..a0fc42e045c49 100644 --- a/onnxruntime/core/platform/telemetry.h +++ b/onnxruntime/core/platform/telemetry.h @@ -43,7 +43,7 @@ class Telemetry { const std::unordered_map& model_metadata, const std::string& loadedFrom, const std::vector& execution_provider_ids) const; - virtual void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, + virtual void LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file, const char* function, uint32_t line) const; virtual void LogRuntimePerf(uint32_t session_id, uint32_t total_runs_since_last, int64_t total_run_duration_since_last) const; diff --git a/onnxruntime/core/platform/windows/env.cc b/onnxruntime/core/platform/windows/env.cc index 2175792c88e14..c387e9e349e94 100644 --- a/onnxruntime/core/platform/windows/env.cc +++ b/onnxruntime/core/platform/windows/env.cc @@ -214,7 +214,7 @@ class WindowsEnv : public Env { // \brief returns a provider that will handle telemetry on the current platform const Telemetry& GetTelemetryProvider() const override { - return telemetryProvider_; + return telemetry_provider_; } private: @@ -234,7 +234,7 @@ class WindowsEnv : public Env { typedef VOID(WINAPI* FnGetSystemTimePreciseAsFileTime)(LPFILETIME); FnGetSystemTimePreciseAsFileTime GetSystemTimePreciseAsFileTime_; - WindowsTelemetry telemetryProvider_; + WindowsTelemetry telemetry_provider_; }; } // namespace diff --git a/onnxruntime/core/platform/windows/telemetry.h b/onnxruntime/core/platform/windows/telemetry.h index bbc31acde37a6..def4a1e67fc75 100644 --- a/onnxruntime/core/platform/windows/telemetry.h +++ b/onnxruntime/core/platform/windows/telemetry.h @@ -43,7 +43,7 @@ class WindowsTelemetry : public Telemetry { const std::unordered_map& model_metadata, const std::string& loadedFrom, const std::vector& execution_provider_ids) const override; - void LogRuntimeError(uint32_t sessionId, const common::Status& status, const char* file, + void LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file, const char* function, uint32_t line) const override; void LogRuntimePerf(uint32_t session_id, uint32_t total_runs_since_last, int64_t total_run_duration_since_last) const override; From 8b07208a705f06afa67105b2a3158c7c25445dd6 Mon Sep 17 00:00:00 2001 From: Paul McDaniel Date: Fri, 18 Oct 2019 09:41:35 -0700 Subject: [PATCH 11/14] doc changes --- README.md | 7 +++---- docs/C_API.md | 3 +-- docs/Privacy.md | 8 +++++++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e50cf08b78776..7a519b9682577 100644 --- a/README.md +++ b/README.md @@ -174,10 +174,9 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. *** -# License -[MIT License](LICENSE) - # Data/Telemetry This project collects usage data and sends it to Microsoft to help improve our products and services. See the [privacy statement](docs/Privacy.md) for more details. -For more information on telemetry implementation see the [developer guide](docs/C_API.md#Telemetry). \ No newline at end of file +*** +# License +[MIT License](LICENSE) diff --git a/docs/C_API.md b/docs/C_API.md index 3a69163289451..92e0f2456ce1a 100644 --- a/docs/C_API.md +++ b/docs/C_API.md @@ -33,8 +33,7 @@ The example below shows a sample run using the SqueezeNet model from ONNX model # Telemetry This project collects usage data and sends it to Microsoft to help improve our products and services. Note however that no data collection is performed by default when using your private builds -Telemetry is turned on by default when using the Official Builds. This is implemented via 'Platform Telemetry' per vendor platform providers (see telemetry.h). +Telemetry is turned OFF by default while this feature is in BETA. When the feature moves from BETA to RELEASE, developers should expect telemetry to be ON by default when using the Official Builds. This is implemented via 'Platform Telemetry' per vendor platform providers (see telemetry.h). The Windows provider uses the [TraceLogging](https://docs.microsoft.com/en-us/windows/win32/tracelogging/trace-logging-about) API for its implementation. -You can turn this off using the DisableTelemetryEvents() API. \ No newline at end of file diff --git a/docs/Privacy.md b/docs/Privacy.md index 808980328eb3c..822e1da6f8bee 100644 --- a/docs/Privacy.md +++ b/docs/Privacy.md @@ -1,4 +1,10 @@ # Data Collection The software may collect information about you and your use of the software and send it to Microsoft. Microsoft may use this information to provide services and improve our products and services. You may turn off the telemetry as described in the repository. There are also some features in the software that may enable you and Microsoft to collect data from users of your applications. If you use these features, you must comply with applicable law, including providing appropriate notices to users of your applications together with a copy of Microsoft's privacy statement. Our privacy statement is located at https://go.microsoft.com/fwlink/?LinkID=824704. You can learn more about data collection and use in the help documentation and our privacy statement. Your use of the software operates as your consent to these practices. -For more information on telemetry implementation see the [developer guide](C_API.md#Telemetry). \ No newline at end of file +Note that no data collection is performed by default when using your private builds. + +Telemetry is turned OFF by default while this feature is in BETA. When the feature moves from BETA to RELEASE, developers should expect telemetry to be ON by default when using the Official Builds. This is implemented via 'Platform Telemetry' per vendor platform providers (see telemetry.h). + +The Windows provider uses the [TraceLogging](https://docs.microsoft.com/en-us/windows/win32/tracelogging/trace-logging-about) API for its implementation. + +You can turn this on or off using the Enable/DisableTelemetryEvents() C API. From f50935bbb22be9a80cd8e59b9951fd1e8f21b048 Mon Sep 17 00:00:00 2001 From: Paul McDaniel Date: Fri, 18 Oct 2019 09:49:48 -0700 Subject: [PATCH 12/14] extra spaces got in there somehow. --- include/onnxruntime/core/common/common.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/onnxruntime/core/common/common.h b/include/onnxruntime/core/common/common.h index 997b17e9ab325..2b295c9acf6b1 100644 --- a/include/onnxruntime/core/common/common.h +++ b/include/onnxruntime/core/common/common.h @@ -176,8 +176,8 @@ void LogRuntimeError(uint32_t session_id, const common::Status& status, const ch #define GSL_SUPPRESS(tag) #endif - inline void MakeStringInternal(std::ostringstream& /*ss*/) noexcept { - } +inline void MakeStringInternal(std::ostringstream& /*ss*/) noexcept { +} template inline void MakeStringInternal(std::ostringstream& ss, const T& t) noexcept { From fb8393b65d68fa89bf2f76842f6d8afc10dbb3eb Mon Sep 17 00:00:00 2001 From: Paul McDaniel Date: Fri, 18 Oct 2019 09:52:17 -0700 Subject: [PATCH 13/14] camelCase fixed --- onnxruntime/core/platform/posix/env.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/platform/posix/env.cc b/onnxruntime/core/platform/posix/env.cc index ff42d66c81626..285e751e1699b 100644 --- a/onnxruntime/core/platform/posix/env.cc +++ b/onnxruntime/core/platform/posix/env.cc @@ -268,12 +268,12 @@ class PosixEnv : public Env { // \brief returns a provider that will handle telemetry on the current platform const Telemetry& GetTelemetryProvider() const override { - return telemetryProvider_; + return telemetry_provider_; } private: PosixEnv() = default; - Telemetry telemetryProvider_; + Telemetry telemetry_provider_; }; } // namespace From beec8298018c9b1ea38cb0444a7f4f84552de97d Mon Sep 17 00:00:00 2001 From: Faith Xu Date: Fri, 18 Oct 2019 13:23:15 -0700 Subject: [PATCH 14/14] Rearrange content order (#2183) * Rearrange content order * Reorganize page * Update telemetry text * Update C_API.md * Update Privacy.md --- README.md | 10 ++++++---- docs/C_API.md | 9 ++------- docs/Privacy.md | 16 ++++++++++++---- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 7a519b9682577..27282b44d92ea 100644 --- a/README.md +++ b/README.md @@ -30,11 +30,12 @@ * [Technical Design Details](#technical-design-details) * [Extensibility Options](#extensibility-options) +**[Data/Telemetry](#Data/Telemetry)** + **[Contributions and Feedback](#contribute)** **[License](#license)** -**[Data/Telemetry](#Data/Telemetry)** *** # Key Features ## Run any ONNX model @@ -162,6 +163,10 @@ To tune performance for ONNX models, the [ONNX Go Live tool "OLive"](https://git transform](include/onnxruntime/core/optimizer/graph_transformer.h) * [Add a new rewrite rule](include/onnxruntime/core/optimizer/rewrite_rule.h) +*** +# Data/Telemetry +This project may collect usage data and send it to Microsoft to help improve our products and services. See the [privacy statement](docs/Privacy.md) for more details. + *** # Contribute We welcome contributions! Please see the [contribution guidelines](CONTRIBUTING.md). @@ -173,9 +178,6 @@ For any feedback or to report a bug, please file a [GitHub Issue](https://github This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. -*** -# Data/Telemetry -This project collects usage data and sends it to Microsoft to help improve our products and services. See the [privacy statement](docs/Privacy.md) for more details. *** # License diff --git a/docs/C_API.md b/docs/C_API.md index 92e0f2456ce1a..30a626fa102e1 100644 --- a/docs/C_API.md +++ b/docs/C_API.md @@ -30,10 +30,5 @@ The example below shows a sample run using the SqueezeNet model from ONNX model * [../csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp](../csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp) -# Telemetry -This project collects usage data and sends it to Microsoft to help improve our products and services. Note however that no data collection is performed by default when using your private builds - -Telemetry is turned OFF by default while this feature is in BETA. When the feature moves from BETA to RELEASE, developers should expect telemetry to be ON by default when using the Official Builds. This is implemented via 'Platform Telemetry' per vendor platform providers (see telemetry.h). - -The Windows provider uses the [TraceLogging](https://docs.microsoft.com/en-us/windows/win32/tracelogging/trace-logging-about) API for its implementation. - +## Telemetry +To turn on/off telemetry collection on official Windows builds, please use Enable/DisableTelemetryEvents() in the C API. See the [Privacy](./Privacy.md) page for more information on telemetry collection and Microsoft's privacy policy. diff --git a/docs/Privacy.md b/docs/Privacy.md index 822e1da6f8bee..9292d989ecfed 100644 --- a/docs/Privacy.md +++ b/docs/Privacy.md @@ -1,10 +1,18 @@ -# Data Collection +# Privacy + +## Data Collection The software may collect information about you and your use of the software and send it to Microsoft. Microsoft may use this information to provide services and improve our products and services. You may turn off the telemetry as described in the repository. There are also some features in the software that may enable you and Microsoft to collect data from users of your applications. If you use these features, you must comply with applicable law, including providing appropriate notices to users of your applications together with a copy of Microsoft's privacy statement. Our privacy statement is located at https://go.microsoft.com/fwlink/?LinkID=824704. You can learn more about data collection and use in the help documentation and our privacy statement. Your use of the software operates as your consent to these practices. -Note that no data collection is performed by default when using your private builds. +*** + +### Private Builds +No data collection is performed when using your private builds. -Telemetry is turned OFF by default while this feature is in BETA. When the feature moves from BETA to RELEASE, developers should expect telemetry to be ON by default when using the Official Builds. This is implemented via 'Platform Telemetry' per vendor platform providers (see telemetry.h). +### Official Builds +Currently telemetry is only implemented for Windows builds, but may be expanded in the future to cover other platforms. Telemetry is turned OFF by default while this feature is in BETA. When the feature moves from BETA to RELEASE, developers should expect telemetry to be ON by default when using the Official Builds. This is implemented via 'Platform Telemetry' per vendor platform providers (see telemetry.h). +#### Technical Details The Windows provider uses the [TraceLogging](https://docs.microsoft.com/en-us/windows/win32/tracelogging/trace-logging-about) API for its implementation. -You can turn this on or off using the Enable/DisableTelemetryEvents() C API. +For API usage details to turn this on/off, please check the API pages: +* [C API](./C_API.md#telemetry)