Merge pull request #9124 from ggovi/new-conddb-timestamp-selection-v1

Added selection of IOV sequence snapshot
cms-sw · May 19, 2015 · 4396563 · 4396563
2 parents 7d6410f + e5a8266
commit 4396563
Show file tree

Hide file tree

Showing 24 changed files with 305 additions and 58 deletions.
diff --git a/CondCore/CondDB/interface/IOVProxy.h b/CondCore/CondDB/interface/IOVProxy.h
@@ -75,6 +75,9 @@ namespace cond {
       // full=true load the full iovSequence 
       void load( const std::string& tag, bool full=false );
 
+      // loads in memory the tag information and the iov groups
+      void load( const std::string& tag, const boost::posix_time::ptime& snapshottime, bool full=false );
+
       // reset the data in memory and execute again the queries for the current tag 
       void reload();
 

diff --git a/CondCore/CondDB/interface/PayloadProxy.h b/CondCore/CondDB/interface/PayloadProxy.h
@@ -28,6 +28,8 @@ namespace cond {
       void setUp( Session dbSession );
 
       void loadTag( const std::string& tag );
+
+      void loadTag( const std::string& tag, const boost::posix_time::ptime& snapshotTime );
 
       void reload();
 

diff --git a/CondCore/CondDB/interface/Session.h b/CondCore/CondDB/interface/Session.h
@@ -86,10 +86,14 @@ namespace cond {
       void createDatabase();
 
       // read access to the iov sequence. 
-      // by default ( full=false ) the iovs are lazy-loaded in groups when required, with repeatable queries ( for FronTier )
-      // full=true will load the entire sequence in memory. Mainly for test/debugging.
+      // the iovs are lazy-loaded in groups when required, with repeatable queries ( for FronTier )
+      IOVProxy readIov( const std::string& tag, bool full=false );
+
+      // read access to the iov sequence. 
+      // the iovs are lazy-loaded in groups when required, with repeatable queries ( for FronTier )
       IOVProxy readIov( const std::string& tag, 
-			bool full=false );//,const boost::posix_time::ptime& snapshottime )  
+			const boost::posix_time::ptime& snapshottime,
+			bool full=false );  
 
       // 
       bool existsIov( const std::string& tag );

diff --git a/CondCore/CondDB/interface/Time.h b/CondCore/CondDB/interface/Time.h
@@ -19,6 +19,8 @@ namespace cond {
     const Time_t MAX_VAL(std::numeric_limits<Time_t>::max());
 
     const Time_t MIN_VAL(0);
+
+    static constexpr const char* const MAX_TIMESTAMP = "9999-12-31 23:59:59.000";
 
     typedef cond::UnpackedTime UnpackedTime;
 

diff --git a/CondCore/CondDB/interface/Types.h b/CondCore/CondDB/interface/Types.h
@@ -98,6 +98,14 @@ namespace cond {
     std::string execmessage;
   };
 
+  struct GTMetadata_t {
+    Time_t validity;
+    std::string description;
+    std::string release;
+    boost::posix_time::ptime insertionTime;
+    boost::posix_time::ptime snapshotTime;
+  };
+
   class GTEntry_t {
   public: 
     GTEntry_t():

diff --git a/CondCore/CondDB/interface/Utils.h b/CondCore/CondDB/interface/Utils.h
@@ -84,7 +84,7 @@ namespace cond {
     inline std::string convertoToOracleConnection(const std::string & input){
 
       // leave the connection string unmodified for sqlite
-      if( input.find("sqlite") == 0 ) return input;
+      if( input.find("sqlite") == 0 || input.find("oracle") == 0) return input;
 
       //static const boost::regex trivial("oracle://(cms_orcon_adg|cms_orcoff_prep)/([_[:alnum:]]+?)");
       static const boost::regex short_frontier("frontier://([[:alnum:]]+?)/([_[:alnum:]]+?)");

diff --git a/CondCore/CondDB/python/CondDB_cfi.py b/CondCore/CondDB/python/CondDB_cfi.py
@@ -6,7 +6,8 @@
         authenticationSystem = cms.untracked.int32(0),
         messageLevel = cms.untracked.int32(0),
     ),
-    connect = cms.string(''), ##db/schema"
+    connect = cms.string(''), 
+    snapshotTime = cms.string(''),
     dbFormat = cms.untracked.int32(0)
 )
 
diff --git a/CondCore/CondDB/src/IDbSchema.h b/CondCore/CondDB/src/IDbSchema.h
@@ -54,7 +54,10 @@ namespace cond {
 					    const boost::posix_time::ptime& snapshotTime, 
 					    std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs) = 0;
       virtual size_t selectLatest( const std::string& tag, std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs) = 0;
+      virtual size_t selectSnapshot( const std::string& tag, const boost::posix_time::ptime& snapshotTime,
+                                     std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs) = 0;
       virtual bool getLastIov( const std::string& tag, cond::Time_t& since, cond::Hash& hash ) = 0;
+      virtual bool getSnapshotLastIov( const std::string& tag, const boost::posix_time::ptime& snapshotTime, cond::Time_t& since, cond::Hash& hash ) = 0;
       virtual bool getSize( const std::string& tag, size_t& size ) = 0;
       virtual bool getSnapshotSize( const std::string& tag, const boost::posix_time::ptime& snapshotTime, size_t& size ) = 0;
       virtual void insertOne( const std::string& tag, cond::Time_t since, cond::Hash payloadHash, 

diff --git a/CondCore/CondDB/src/IOVProxy.cc b/CondCore/CondDB/src/IOVProxy.cc
@@ -16,6 +16,7 @@ namespace cond {
 
       // tag data
       std::string tag;
+      boost::posix_time::ptime snapshotTime;
       cond::TimeType timeType;
       std::string payloadType;
       cond::SynchronizationType synchronizationType = cond::OFFLINE;
@@ -121,7 +122,15 @@ namespace cond {
       return *this;
     }
 
-    void IOVProxy::load( const std::string& tag, bool full ){
+    void IOVProxy::load( const std::string& tag,
+                         bool full ){
+      boost::posix_time::ptime notime;
+      load( tag, notime, full );
+    }
+
+    void IOVProxy::load( const std::string& tag, 
+			 const boost::posix_time::ptime& snapshotTime,
+			 bool full ){
       if( !m_data.get() ) return;
 
       // clear
@@ -139,16 +148,25 @@ namespace cond {
       // now get the iov sequence when required
       if( full ) {
 	// load the full iov sequence in this case!
-	m_session->iovSchema().iovTable().selectLatest( m_data->tag, m_data->iovSequence );
+        if( snapshotTime.is_not_a_date_time() ){
+          m_session->iovSchema().iovTable().selectLatest( m_data->tag, m_data->iovSequence );
+        } else {
+          m_session->iovSchema().iovTable().selectSnapshot( m_data->tag, snapshotTime, m_data->iovSequence );
+        }
 	m_data->groupLowerIov = cond::time::MIN_VAL;
 	m_data->groupHigherIov = cond::time::MAX_VAL;
       } else {
-	m_session->iovSchema().iovTable().selectGroups( m_data->tag, m_data->sinceGroups );
+        if( snapshotTime.is_not_a_date_time() ){
+          m_session->iovSchema().iovTable().selectGroups( m_data->tag, m_data->sinceGroups );
+        } else {
+          m_session->iovSchema().iovTable().selectSnapshotGroups( m_data->tag, snapshotTime, m_data->sinceGroups );
+        }
       }
+      m_data->snapshotTime = snapshotTime;
     }
 
     void IOVProxy::reload(){
-      if(m_data.get() && !m_data->tag.empty()) load( m_data->tag );
+      if(m_data.get() && !m_data->tag.empty()) load( m_data->tag, m_data->snapshotTime );
     }
 
     void IOVProxy::reset(){
@@ -206,7 +224,11 @@ namespace cond {
 
     void IOVProxy::fetchSequence( cond::Time_t lowerGroup, cond::Time_t higherGroup ){
       m_data->iovSequence.clear();
-      m_session->iovSchema().iovTable().selectLatestByGroup( m_data->tag, lowerGroup, higherGroup, m_data->iovSequence );
+      if( m_data->snapshotTime.is_not_a_date_time() ){
+	m_session->iovSchema().iovTable().selectLatestByGroup( m_data->tag, lowerGroup, higherGroup, m_data->iovSequence );
+      } else {
+	m_session->iovSchema().iovTable().selectSnapshotByGroup( m_data->tag, lowerGroup, higherGroup, m_data->snapshotTime, m_data->iovSequence );
+      }
 
       if( m_data->iovSequence.empty() ){
 	m_data->groupLowerIov = cond::time::MAX_VAL;
@@ -297,9 +319,13 @@ namespace cond {
     cond::Iov_t IOVProxy::getLast(){
       checkTransaction( "IOVProxy::getLast" );
       cond::Iov_t ret;
-      if( m_session->iovSchema().iovTable().getLastIov( m_data->tag, ret.since, ret.payloadId ) ){
-	ret.till = cond::time::MAX_VAL;
+      bool ok = false;
+      if( m_data->snapshotTime.is_not_a_date_time() ){
+	ok = m_session->iovSchema().iovTable().getLastIov( m_data->tag, ret.since, ret.payloadId );
+      } else {
+	ok = m_session->iovSchema().iovTable().getSnapshotLastIov( m_data->tag, m_data->snapshotTime, ret.since, ret.payloadId );
       }
+      if(ok) ret.till = cond::time::MAX_VAL;
       return ret;
     }
 
@@ -310,7 +336,11 @@ namespace cond {
     int IOVProxy::sequenceSize() const {
       checkTransaction( "IOVProxy::sequenceSize" );
       size_t ret = 0;
-      m_session->iovSchema().iovTable().getSize( m_data->tag, ret );
+      if( m_data->snapshotTime.is_not_a_date_time() ){
+	m_session->iovSchema().iovTable().getSize( m_data->tag, ret );
+      } else {
+	m_session->iovSchema().iovTable().getSnapshotSize( m_data->tag, m_data->snapshotTime, ret );
+      }
       return ret;
     }
 

diff --git a/CondCore/CondDB/src/IOVSchema.cc b/CondCore/CondDB/src/IOVSchema.cc
@@ -210,6 +210,23 @@ namespace cond {
       return iovs.size()-initialSize;
     }
 
+    size_t IOV::Table::selectSnapshot( const std::string& tag,
+                                       const boost::posix_time::ptime& snapshotTime,
+                                       std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs){
+      Query< SINCE, PAYLOAD_HASH > q( m_schema );
+      q.addCondition<TAG_NAME>( tag );
+      q.addCondition<INSERTION_TIME>( snapshotTime,"<=" );
+      q.addOrderClause<SINCE>();
+      q.addOrderClause<INSERTION_TIME>( false );
+      size_t initialSize = iovs.size();
+      for ( auto row : q ) {
+        // starting from the second iov in the array, skip the rows with older timestamp                                                                            
+        if( iovs.size()-initialSize && std::get<0>(iovs.back()) == std::get<0>(row) ) continue;
+        iovs.push_back( row );
+      }
+      return iovs.size()-initialSize;
+    }
+
     bool IOV::Table::getLastIov( const std::string& tag, cond::Time_t& since, cond::Hash& hash ){
       Query< SINCE, PAYLOAD_HASH > q( m_schema );
       q.addCondition<TAG_NAME>( tag );
@@ -223,6 +240,20 @@ namespace cond {
       return false;
     }
 
+    bool IOV::Table::getSnapshotLastIov( const std::string& tag, const boost::posix_time::ptime& snapshotTime, cond::Time_t& since, cond::Hash& hash ){
+      Query< SINCE, PAYLOAD_HASH > q( m_schema );
+      q.addCondition<TAG_NAME>( tag );
+      q.addCondition<INSERTION_TIME>( snapshotTime,"<=" );
+      q.addOrderClause<SINCE>( false );
+      q.addOrderClause<INSERTION_TIME>( false );
+      for ( auto row : q ) {
+        since = std::get<0>(row);
+        hash = std::get<1>(row);
+        return true;
+      }
+      return false;
+    }
+
     bool IOV::Table::getSize( const std::string& tag, size_t& size ){
       Query< SEQUENCE_SIZE > q( m_schema );
       q.addCondition<TAG_NAME>( tag );

diff --git a/CondCore/CondDB/src/IOVSchema.h b/CondCore/CondDB/src/IOVSchema.h
@@ -120,7 +120,11 @@ namespace cond {
 				      const boost::posix_time::ptime& snapshotTime, 
 				      std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs);
 	size_t selectLatest( const std::string& tag, std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs);
+	size_t selectSnapshot( const std::string& tag,
+                               const boost::posix_time::ptime& snapshotTime,
+                               std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs);
 	bool getLastIov( const std::string& tag, cond::Time_t& since, cond::Hash& hash );
+	bool getSnapshotLastIov( const std::string& tag, const boost::posix_time::ptime& snapshotTime, cond::Time_t& since, cond::Hash& hash );
 	bool getSize( const std::string& tag, size_t& size );
         bool getSnapshotSize( const std::string& tag, const boost::posix_time::ptime& snapshotTime, size_t& size );
 	void insertOne( const std::string& tag, cond::Time_t since, cond::Hash payloadHash, const boost::posix_time::ptime& insertTime);

diff --git a/CondCore/CondDB/src/OraDbSchema.cc b/CondCore/CondDB/src/OraDbSchema.cc
@@ -210,14 +210,26 @@ namespace cond {
       return ret;      
     }
 
-    bool OraIOVTable::getLastIov( const std::string& tag, cond::Time_t& since, cond::Hash& hash ){
+    size_t OraIOVTable::selectSnapshot( const std::string& tag, const boost::posix_time::ptime&, 
+					std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs){
+      // no (easy) way to do it...                                                                                                                                  
+      return selectLatest( tag, iovs );
+    }
+
+     bool OraIOVTable::getLastIov( const std::string& tag, cond::Time_t& since, cond::Hash& hash ){
       if(!m_cache.load( tag ) || m_cache.iovSequence().size()==0 ) return false;
       cond::IOVElementProxy last = *(--m_cache.iovSequence().end());
       since = last.since();
       hash = last.token();
       return true;
     }
 
+    bool OraIOVTable::getSnapshotLastIov( const std::string& tag, const boost::posix_time::ptime&, 
+				  cond::Time_t& since, cond::Hash& hash ){
+      // no (easy) way to do it...                                                                                                                                  
+      return getLastIov( tag, since, hash );
+    }
+
     bool OraIOVTable::getSize( const std::string& tag, size_t& size ){
       if(!m_cache.load( tag )) return false;
       size = m_cache.iovSequence().size();

diff --git a/CondCore/CondDB/src/OraDbSchema.h b/CondCore/CondDB/src/OraDbSchema.h
@@ -100,7 +100,9 @@ namespace cond {
 				    const boost::posix_time::ptime& snapshotTime, 
 				    std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs);
       size_t selectLatest( const std::string& tag, std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs);
+      size_t selectSnapshot( const std::string& tag, const boost::posix_time::ptime& snapshotTime, std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs);
       bool getLastIov( const std::string& tag, cond::Time_t& since, cond::Hash& hash );
+      bool getSnapshotLastIov( const std::string& tag, const boost::posix_time::ptime& snapshotTime, cond::Time_t& since, cond::Hash& hash );
       bool getSize( const std::string& tag, size_t& size );
       bool getSnapshotSize( const std::string& tag, const boost::posix_time::ptime& snapshotTime, size_t& size );
       void insertOne( const std::string& tag, cond::Time_t since, cond::Hash payloadHash, 

diff --git a/CondCore/CondDB/src/PayloadProxy.cc b/CondCore/CondDB/src/PayloadProxy.cc
@@ -22,9 +22,16 @@ namespace cond {
       invalidateCache();
     }
 
+    void BasePayloadProxy::loadTag( const std::string& tag, const boost::posix_time::ptime& snapshotTime ){
+      m_session.transaction().start(true);
+      m_iovProxy = m_session.readIov( tag, snapshotTime );
+      m_session.transaction().commit();
+      invalidateCache();
+    }
+
     void BasePayloadProxy::reload(){
       std::string tag = m_iovProxy.tag();
-      if( !tag.empty() ) loadTag( tag );
+      if( !tag.empty() ) m_iovProxy.reload();
     }
 
     ValidityInterval BasePayloadProxy::setIntervalFor(cond::Time_t time, bool load) {

diff --git a/CondCore/CondDB/src/Session.cc b/CondCore/CondDB/src/Session.cc
@@ -85,6 +85,15 @@ namespace cond {
       return proxy;
     }
 
+    IOVProxy  Session::readIov( const std::string& tag,
+                                const boost::posix_time::ptime& snapshottime,
+                                bool full ){
+      m_session->openIovDb();
+      IOVProxy proxy( m_session );
+      proxy.load( tag, snapshottime, full );
+      return proxy;
+    }
+
     bool Session::existsIov( const std::string& tag ){
       m_session->openIovDb();
       return m_session->iovSchema().tagTable().select( tag );

diff --git a/CondCore/CondDB/test/BuildFile.xml b/CondCore/CondDB/test/BuildFile.xml
@@ -9,6 +9,8 @@
 <bin   file="testConditionDatabase_0.cpp" name="testConditionDatabase_0">
   <lib   name="testCondDBDict"/>
 </bin>
+<bin   file="testConditionDatabase_1.cpp" name="testConditionDatabase_1">
+</bin>
 <bin   file="testRootStreaming.cpp" name="testRootStreaming">
   <lib   name="testCondDBDict"/>
 </bin>

diff --git a/CondCore/CondDB/test/testConditionDatabase_0.cpp b/CondCore/CondDB/test/testConditionDatabase_0.cpp
@@ -126,7 +126,7 @@ int main (int argc, char** argv)
   int ret = 0;
   edmplugin::PluginManager::Config config;
   edmplugin::PluginManager::configure(edmplugin::standard::config());
-  std::string connectionString0("sqlite_file:cms_conditions.db");
+  std::string connectionString0("sqlite_file:cms_conditions_0.db");
   std::cout <<"## Running with CondDBV2 format..."<<std::endl;
   ret = run( connectionString0 );
   if( ret<0 ) return ret;