Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added selection of IOV sequence snapshot #9124

Merged
merged 2 commits into from
May 19, 2015
Merged

Added selection of IOV sequence snapshot #9124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
304883c
Added selection of IOV sequence snapshot
ggovi May 18, 2015
e5a8266
Changed test filenames
ggovi May 19, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CondCore/CondDB/interface/IOVProxy.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ namespace cond {
// full=true load the full iovSequence
void load( const std::string& tag, bool full=false );

// loads in memory the tag information and the iov groups
void load( const std::string& tag, const boost::posix_time::ptime& snapshottime, bool full=false );

// reset the data in memory and execute again the queries for the current tag
void reload();

Expand Down
2 changes: 2 additions & 0 deletions CondCore/CondDB/interface/PayloadProxy.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ namespace cond {
void setUp( Session dbSession );

void loadTag( const std::string& tag );

void loadTag( const std::string& tag, const boost::posix_time::ptime& snapshotTime );

void reload();

Expand Down
10 changes: 7 additions & 3 deletions CondCore/CondDB/interface/Session.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,14 @@ namespace cond {
void createDatabase();

// read access to the iov sequence.
// by default ( full=false ) the iovs are lazy-loaded in groups when required, with repeatable queries ( for FronTier )
// full=true will load the entire sequence in memory. Mainly for test/debugging.
// the iovs are lazy-loaded in groups when required, with repeatable queries ( for FronTier )
IOVProxy readIov( const std::string& tag, bool full=false );

// read access to the iov sequence.
// the iovs are lazy-loaded in groups when required, with repeatable queries ( for FronTier )
IOVProxy readIov( const std::string& tag,
bool full=false );//,const boost::posix_time::ptime& snapshottime )
const boost::posix_time::ptime& snapshottime,
bool full=false );

//
bool existsIov( const std::string& tag );
Expand Down
2 changes: 2 additions & 0 deletions CondCore/CondDB/interface/Time.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ namespace cond {
const Time_t MAX_VAL(std::numeric_limits<Time_t>::max());

const Time_t MIN_VAL(0);

static constexpr const char* const MAX_TIMESTAMP = "9999-12-31 23:59:59.000";

typedef cond::UnpackedTime UnpackedTime;

Expand Down
8 changes: 8 additions & 0 deletions CondCore/CondDB/interface/Types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,14 @@ namespace cond {
std::string execmessage;
};

struct GTMetadata_t {
Time_t validity;
std::string description;
std::string release;
boost::posix_time::ptime insertionTime;
boost::posix_time::ptime snapshotTime;
};

class GTEntry_t {
public:
GTEntry_t():
Expand Down
2 changes: 1 addition & 1 deletion CondCore/CondDB/interface/Utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ namespace cond {
inline std::string convertoToOracleConnection(const std::string & input){

// leave the connection string unmodified for sqlite
if( input.find("sqlite") == 0 ) return input;
if( input.find("sqlite") == 0 || input.find("oracle") == 0) return input;

//static const boost::regex trivial("oracle://(cms_orcon_adg|cms_orcoff_prep)/([_[:alnum:]]+?)");
static const boost::regex short_frontier("frontier://([[:alnum:]]+?)/([_[:alnum:]]+?)");
Expand Down
3 changes: 2 additions & 1 deletion CondCore/CondDB/python/CondDB_cfi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
authenticationSystem = cms.untracked.int32(0),
messageLevel = cms.untracked.int32(0),
),
connect = cms.string(''), ##db/schema"
connect = cms.string(''),
snapshotTime = cms.string(''),
dbFormat = cms.untracked.int32(0)
)

3 changes: 3 additions & 0 deletions CondCore/CondDB/src/IDbSchema.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,10 @@ namespace cond {
const boost::posix_time::ptime& snapshotTime,
std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs) = 0;
virtual size_t selectLatest( const std::string& tag, std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs) = 0;
virtual size_t selectSnapshot( const std::string& tag, const boost::posix_time::ptime& snapshotTime,
std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs) = 0;
virtual bool getLastIov( const std::string& tag, cond::Time_t& since, cond::Hash& hash ) = 0;
virtual bool getSnapshotLastIov( const std::string& tag, const boost::posix_time::ptime& snapshotTime, cond::Time_t& since, cond::Hash& hash ) = 0;
virtual bool getSize( const std::string& tag, size_t& size ) = 0;
virtual bool getSnapshotSize( const std::string& tag, const boost::posix_time::ptime& snapshotTime, size_t& size ) = 0;
virtual void insertOne( const std::string& tag, cond::Time_t since, cond::Hash payloadHash,
Expand Down
46 changes: 38 additions & 8 deletions CondCore/CondDB/src/IOVProxy.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ namespace cond {

// tag data
std::string tag;
boost::posix_time::ptime snapshotTime;
cond::TimeType timeType;
std::string payloadType;
cond::SynchronizationType synchronizationType = cond::OFFLINE;
Expand Down Expand Up @@ -121,7 +122,15 @@ namespace cond {
return *this;
}

void IOVProxy::load( const std::string& tag, bool full ){
void IOVProxy::load( const std::string& tag,
bool full ){
boost::posix_time::ptime notime;
load( tag, notime, full );
}

void IOVProxy::load( const std::string& tag,
const boost::posix_time::ptime& snapshotTime,
bool full ){
if( !m_data.get() ) return;

// clear
Expand All @@ -139,16 +148,25 @@ namespace cond {
// now get the iov sequence when required
if( full ) {
// load the full iov sequence in this case!
m_session->iovSchema().iovTable().selectLatest( m_data->tag, m_data->iovSequence );
if( snapshotTime.is_not_a_date_time() ){
m_session->iovSchema().iovTable().selectLatest( m_data->tag, m_data->iovSequence );
} else {
m_session->iovSchema().iovTable().selectSnapshot( m_data->tag, snapshotTime, m_data->iovSequence );
}
m_data->groupLowerIov = cond::time::MIN_VAL;
m_data->groupHigherIov = cond::time::MAX_VAL;
} else {
m_session->iovSchema().iovTable().selectGroups( m_data->tag, m_data->sinceGroups );
if( snapshotTime.is_not_a_date_time() ){
m_session->iovSchema().iovTable().selectGroups( m_data->tag, m_data->sinceGroups );
} else {
m_session->iovSchema().iovTable().selectSnapshotGroups( m_data->tag, snapshotTime, m_data->sinceGroups );
}
}
m_data->snapshotTime = snapshotTime;
}

void IOVProxy::reload(){
if(m_data.get() && !m_data->tag.empty()) load( m_data->tag );
if(m_data.get() && !m_data->tag.empty()) load( m_data->tag, m_data->snapshotTime );
}

void IOVProxy::reset(){
Expand Down Expand Up @@ -206,7 +224,11 @@ namespace cond {

void IOVProxy::fetchSequence( cond::Time_t lowerGroup, cond::Time_t higherGroup ){
m_data->iovSequence.clear();
m_session->iovSchema().iovTable().selectLatestByGroup( m_data->tag, lowerGroup, higherGroup, m_data->iovSequence );
if( m_data->snapshotTime.is_not_a_date_time() ){
m_session->iovSchema().iovTable().selectLatestByGroup( m_data->tag, lowerGroup, higherGroup, m_data->iovSequence );
} else {
m_session->iovSchema().iovTable().selectSnapshotByGroup( m_data->tag, lowerGroup, higherGroup, m_data->snapshotTime, m_data->iovSequence );
}

if( m_data->iovSequence.empty() ){
m_data->groupLowerIov = cond::time::MAX_VAL;
Expand Down Expand Up @@ -297,9 +319,13 @@ namespace cond {
cond::Iov_t IOVProxy::getLast(){
checkTransaction( "IOVProxy::getLast" );
cond::Iov_t ret;
if( m_session->iovSchema().iovTable().getLastIov( m_data->tag, ret.since, ret.payloadId ) ){
ret.till = cond::time::MAX_VAL;
bool ok = false;
if( m_data->snapshotTime.is_not_a_date_time() ){
ok = m_session->iovSchema().iovTable().getLastIov( m_data->tag, ret.since, ret.payloadId );
} else {
ok = m_session->iovSchema().iovTable().getSnapshotLastIov( m_data->tag, m_data->snapshotTime, ret.since, ret.payloadId );
}
if(ok) ret.till = cond::time::MAX_VAL;
return ret;
}

Expand All @@ -310,7 +336,11 @@ namespace cond {
int IOVProxy::sequenceSize() const {
checkTransaction( "IOVProxy::sequenceSize" );
size_t ret = 0;
m_session->iovSchema().iovTable().getSize( m_data->tag, ret );
if( m_data->snapshotTime.is_not_a_date_time() ){
m_session->iovSchema().iovTable().getSize( m_data->tag, ret );
} else {
m_session->iovSchema().iovTable().getSnapshotSize( m_data->tag, m_data->snapshotTime, ret );
}
return ret;
}

Expand Down
31 changes: 31 additions & 0 deletions CondCore/CondDB/src/IOVSchema.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,23 @@ namespace cond {
return iovs.size()-initialSize;
}

size_t IOV::Table::selectSnapshot( const std::string& tag,
const boost::posix_time::ptime& snapshotTime,
std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs){
Query< SINCE, PAYLOAD_HASH > q( m_schema );
q.addCondition<TAG_NAME>( tag );
q.addCondition<INSERTION_TIME>( snapshotTime,"<=" );
q.addOrderClause<SINCE>();
q.addOrderClause<INSERTION_TIME>( false );
size_t initialSize = iovs.size();
for ( auto row : q ) {
// starting from the second iov in the array, skip the rows with older timestamp
if( iovs.size()-initialSize && std::get<0>(iovs.back()) == std::get<0>(row) ) continue;
iovs.push_back( row );
}
return iovs.size()-initialSize;
}

bool IOV::Table::getLastIov( const std::string& tag, cond::Time_t& since, cond::Hash& hash ){
Query< SINCE, PAYLOAD_HASH > q( m_schema );
q.addCondition<TAG_NAME>( tag );
Expand All @@ -223,6 +240,20 @@ namespace cond {
return false;
}

bool IOV::Table::getSnapshotLastIov( const std::string& tag, const boost::posix_time::ptime& snapshotTime, cond::Time_t& since, cond::Hash& hash ){
Query< SINCE, PAYLOAD_HASH > q( m_schema );
q.addCondition<TAG_NAME>( tag );
q.addCondition<INSERTION_TIME>( snapshotTime,"<=" );
q.addOrderClause<SINCE>( false );
q.addOrderClause<INSERTION_TIME>( false );
for ( auto row : q ) {
since = std::get<0>(row);
hash = std::get<1>(row);
return true;
}
return false;
}

bool IOV::Table::getSize( const std::string& tag, size_t& size ){
Query< SEQUENCE_SIZE > q( m_schema );
q.addCondition<TAG_NAME>( tag );
Expand Down
4 changes: 4 additions & 0 deletions CondCore/CondDB/src/IOVSchema.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,11 @@ namespace cond {
const boost::posix_time::ptime& snapshotTime,
std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs);
size_t selectLatest( const std::string& tag, std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs);
size_t selectSnapshot( const std::string& tag,
const boost::posix_time::ptime& snapshotTime,
std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs);
bool getLastIov( const std::string& tag, cond::Time_t& since, cond::Hash& hash );
bool getSnapshotLastIov( const std::string& tag, const boost::posix_time::ptime& snapshotTime, cond::Time_t& since, cond::Hash& hash );
bool getSize( const std::string& tag, size_t& size );
bool getSnapshotSize( const std::string& tag, const boost::posix_time::ptime& snapshotTime, size_t& size );
void insertOne( const std::string& tag, cond::Time_t since, cond::Hash payloadHash, const boost::posix_time::ptime& insertTime);
Expand Down
14 changes: 13 additions & 1 deletion CondCore/CondDB/src/OraDbSchema.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,14 +210,26 @@ namespace cond {
return ret;
}

bool OraIOVTable::getLastIov( const std::string& tag, cond::Time_t& since, cond::Hash& hash ){
size_t OraIOVTable::selectSnapshot( const std::string& tag, const boost::posix_time::ptime&,
std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs){
// no (easy) way to do it...
return selectLatest( tag, iovs );
}

bool OraIOVTable::getLastIov( const std::string& tag, cond::Time_t& since, cond::Hash& hash ){
if(!m_cache.load( tag ) || m_cache.iovSequence().size()==0 ) return false;
cond::IOVElementProxy last = *(--m_cache.iovSequence().end());
since = last.since();
hash = last.token();
return true;
}

bool OraIOVTable::getSnapshotLastIov( const std::string& tag, const boost::posix_time::ptime&,
cond::Time_t& since, cond::Hash& hash ){
// no (easy) way to do it...
return getLastIov( tag, since, hash );
}

bool OraIOVTable::getSize( const std::string& tag, size_t& size ){
if(!m_cache.load( tag )) return false;
size = m_cache.iovSequence().size();
Expand Down
2 changes: 2 additions & 0 deletions CondCore/CondDB/src/OraDbSchema.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,9 @@ namespace cond {
const boost::posix_time::ptime& snapshotTime,
std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs);
size_t selectLatest( const std::string& tag, std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs);
size_t selectSnapshot( const std::string& tag, const boost::posix_time::ptime& snapshotTime, std::vector<std::tuple<cond::Time_t,cond::Hash> >& iovs);
bool getLastIov( const std::string& tag, cond::Time_t& since, cond::Hash& hash );
bool getSnapshotLastIov( const std::string& tag, const boost::posix_time::ptime& snapshotTime, cond::Time_t& since, cond::Hash& hash );
bool getSize( const std::string& tag, size_t& size );
bool getSnapshotSize( const std::string& tag, const boost::posix_time::ptime& snapshotTime, size_t& size );
void insertOne( const std::string& tag, cond::Time_t since, cond::Hash payloadHash,
Expand Down
9 changes: 8 additions & 1 deletion CondCore/CondDB/src/PayloadProxy.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,16 @@ namespace cond {
invalidateCache();
}

void BasePayloadProxy::loadTag( const std::string& tag, const boost::posix_time::ptime& snapshotTime ){
m_session.transaction().start(true);
m_iovProxy = m_session.readIov( tag, snapshotTime );
m_session.transaction().commit();
invalidateCache();
}

void BasePayloadProxy::reload(){
std::string tag = m_iovProxy.tag();
if( !tag.empty() ) loadTag( tag );
if( !tag.empty() ) m_iovProxy.reload();
}

ValidityInterval BasePayloadProxy::setIntervalFor(cond::Time_t time, bool load) {
Expand Down
9 changes: 9 additions & 0 deletions CondCore/CondDB/src/Session.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,15 @@ namespace cond {
return proxy;
}

IOVProxy Session::readIov( const std::string& tag,
const boost::posix_time::ptime& snapshottime,
bool full ){
m_session->openIovDb();
IOVProxy proxy( m_session );
proxy.load( tag, snapshottime, full );
return proxy;
}

bool Session::existsIov( const std::string& tag ){
m_session->openIovDb();
return m_session->iovSchema().tagTable().select( tag );
Expand Down
2 changes: 2 additions & 0 deletions CondCore/CondDB/test/BuildFile.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
<bin file="testConditionDatabase_0.cpp" name="testConditionDatabase_0">
<lib name="testCondDBDict"/>
</bin>
<bin file="testConditionDatabase_1.cpp" name="testConditionDatabase_1">
</bin>
<bin file="testRootStreaming.cpp" name="testRootStreaming">
<lib name="testCondDBDict"/>
</bin>
Expand Down
2 changes: 1 addition & 1 deletion CondCore/CondDB/test/testConditionDatabase_0.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ int main (int argc, char** argv)
int ret = 0;
edmplugin::PluginManager::Config config;
edmplugin::PluginManager::configure(edmplugin::standard::config());
std::string connectionString0("sqlite_file:cms_conditions.db");
std::string connectionString0("sqlite_file:cms_conditions_0.db");
std::cout <<"## Running with CondDBV2 format..."<<std::endl;
ret = run( connectionString0 );
if( ret<0 ) return ret;
Expand Down
Loading