diff --git a/src/profiler/profiler.h b/src/profiler/profiler.h
index cbb357dbeecf..cd41d587c6fe 100644
--- a/src/profiler/profiler.h
+++ b/src/profiler/profiler.h
@@ -490,6 +490,12 @@ class Profiler {
 #define VTUNE_ONLY_CODE(...) /* */        /* This is undefined at the bottom of this file */
 #endif
 
+#ifdef MXNET_USE_NVTX
+#define NVTX_ONLY_CODE(...) __VA_ARGS__  /* This is undefined at the bottom of this file */
+#else
+#define NVTX_ONLY_CODE(...) /* */        /* This is undefined at the bottom of this file */
+#endif
+
 /**
  *  _____              __  _  _  _                ____  _     _            _
  * |  __ \            / _|(_)| |(_)              / __ \| |   (_)          | |
@@ -772,10 +778,7 @@ struct ProfileTask : public ProfileDuration {
     categories_.set(domain_->name());
     categories_.append(",task");
     VTUNE_ONLY_CODE(vtune_task_.reset(new vtune::VTuneTask(name, domain->dom())));
-
-#if MXNET_USE_NVTX
-    nvtx_duration_.reset(new nvtx::NVTXDuration(name));
-#endif  // MXNET_USE_NVTX
+    NVTX_ONLY_CODE(nvtx_duration_.reset(new nvtx::NVTXDuration(name)));
   }
 
   /*!
@@ -784,11 +787,7 @@ struct ProfileTask : public ProfileDuration {
   void start() override {
     start_time_ = ProfileStat::NowInMicrosec();
     VTUNE_ONLY_CODE(vtune_task_->start());
-
-#if MXNET_USE_NVTX
-    /*! \brief NVTX duration object */
-    nvtx_duration_->start();
-#endif  // MXNET_USE_NVTX
+    NVTX_ONLY_CODE(nvtx_duration_->start());
   }
 
   /*!
@@ -796,11 +795,8 @@ struct ProfileTask : public ProfileDuration {
    */
   void stop() override {
     VTUNE_ONLY_CODE(vtune_task_->stop());
+    NVTX_ONLY_CODE(nvtx_duration_->stop());
     SendStat();
-
-#if MXNET_USE_NVTX
-    nvtx_duration_->stop();
-#endif  // MXNET_USE_NVTX
   }
 
   ProfileObjectType type() const override { return kTask; }
@@ -839,10 +835,8 @@ struct ProfileTask : public ProfileDuration {
   ProfileDomain *domain_;
   /*! \brief VTune task object */
   VTUNE_ONLY_CODE(std::unique_ptr<vtune::VTuneTask> vtune_task_);
-#if MXNET_USE_NVTX
   /*! \brief NVTX duration object */
-  std::unique_ptr<nvtx::NVTXDuration> nvtx_duration_;
-#endif  // MXNET_USE_NVTX
+  NVTX_ONLY_CODE(std::unique_ptr<nvtx::NVTXDuration> nvtx_duration_);
 
  protected:
   /*! \brief Task's start tick */
@@ -861,9 +855,7 @@ struct ProfileEvent  : public ProfileDuration {
     : name_(name)
       , categories_("event") {
     VTUNE_ONLY_CODE(vtune_event_ = vtune::VTuneEvent::registry_.get(name));
-#if MXNET_USE_NVTX
-    nvtx_duration_.reset(new nvtx::NVTXDuration(name));
-#endif  // MXNET_USE_NVTX
+    NVTX_ONLY_CODE(nvtx_duration_.reset(new nvtx::NVTXDuration(name)));
   }
 
   /*!
@@ -872,10 +864,7 @@ struct ProfileEvent  : public ProfileDuration {
   void start() override {
     start_time_ = ProfileStat::NowInMicrosec();
     VTUNE_ONLY_CODE(vtune_event_->start());
-#if MXNET_USE_NVTX
-    /*! \brief NVTX duration object */
-    nvtx_duration_->start();
-#endif  // MXNET_USE_NVTX
+    NVTX_ONLY_CODE(nvtx_duration_->start());
   }
 
   /*!
@@ -924,10 +913,8 @@ struct ProfileEvent  : public ProfileDuration {
   profile_stat_string categories_;
   /*! \brief VTune event object */
   VTUNE_ONLY_CODE(vtune::VTuneEvent *vtune_event_);
-#if MXNET_USE_NVTX
-      /*! \brief NVTX duration object */
-      std::unique_ptr<nvtx::NVTXDuration> nvtx_duration_;
-#endif  // MXNET_USE_NVTX
+  /*! \brief NVTX duration object */
+  NVTX_ONLY_CODE(std::unique_ptr<nvtx::NVTXDuration> nvtx_duration_;);
 
  protected:
   /*! \brief Start time of the event */
@@ -949,11 +936,7 @@ struct ProfileFrame : public ProfileDuration {
     CHECK_NOTNULL(domain);
     categories_.set(domain_->name());
     categories_.append(",frame");
-
-#if MXNET_USE_NVTX
-    nvtx_duration_.reset(new nvtx::NVTXDuration(name));
-#endif  // MXNET_USE_NVTX
-
+    NVTX_ONLY_CODE(nvtx_duration_.reset(new nvtx::NVTXDuration(name)));
     VTUNE_ONLY_CODE(vtune_frame_.reset(new vtune::VTuneFrame(domain->dom())));
   }
 
@@ -963,10 +946,7 @@ struct ProfileFrame : public ProfileDuration {
   void start() override {
     start_time_ = ProfileStat::NowInMicrosec();
     VTUNE_ONLY_CODE(vtune_frame_->start());
-#if MXNET_USE_NVTX
-    /*! \brief NVTX duration object */
-    nvtx_duration_->start();
-#endif  // MXNET_USE_NVTX
+    NVTX_ONLY_CODE(nvtx_duration_->start());
   }
 
   /*!
@@ -1009,10 +989,8 @@ struct ProfileFrame : public ProfileDuration {
   ProfileDomain *domain_;
   /*! \brief VTune Frame object */
   VTUNE_ONLY_CODE(std::unique_ptr<vtune::VTuneFrame> vtune_frame_);
-#if MXNET_USE_NVTX
-      /*! \brief NVTX duration object */
-      std::unique_ptr<nvtx::NVTXDuration> nvtx_duration_;
-#endif  // MXNET_USE_NVTX
+  /*! \brief NVTX duration object */
+  NVTX_ONLY_CODE(std::unique_ptr<nvtx::NVTXDuration> nvtx_duration_);
 
  protected:
   /*! \brief Frame start time */