Merge pull request #42967 from missirol/devel_hltUtils

add `hltPrintMenuVersions` and utilities for HLT-menu spreadsheets

HLTrigger/Configuration/scripts/hltPrintMenuVersions

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""hltPrintMenuVersions: print to stdout metadata on the history of an HLT configuration in ConfDB
+"""
+import argparse
+import os
+import cx_Oracle
+
+def getHLTMenuVersions(configName, connect_string, add_hyperlinks = True, do_twiki_edits = True):
+    # connect to ConfDB
+    conn = cx_Oracle.connect(connect_string)
+    curs = conn.cursor()
+
+    ret = []
+    # loop on the different versions of the target configuration
+    version = 0
+    while True:
+        version += 1
+        configuration = f'{configName}/V{version}'
+        # get only Config description and "release template"
+        query = 'SELECT u_confversions.description, u_softreleases.releaseTag FROM u_confversions, u_softreleases'
+        query += f" WHERE u_confversions.id_release = u_softreleases.id AND u_confversions.name='{configuration}'"
+        curs.execute(query)
+        rows = curs.fetchall()
+        if len(rows) == 0:
+            break
+        for rows in rows:
+            ## get description, if it is not empty
+            try:
+                description = rows[0].read()
+            except:
+                description = ''
+
+            ## get release template, if it is not empty
+            try:
+                releaseTemplate = rows[1]
+            except:
+                releaseTemplate = ''
+
+            ## add link to JIRA tickets
+            if add_hyperlinks:
+                posInit = description.find('CMSHLT-')
+                if posInit >= 0:
+                    posFinal = posInit+7
+                    while (posFinal < len(description) and description[posFinal].isdigit()):
+                        posFinal += 1
+                    cmshlt = description[posInit:posFinal]
+                    description = description.replace(cmshlt, f'[[https://its.cern.ch/jira/browse/{cmshlt}][{cmshlt}]]')
+
+            ## add '!' in front of capital letters (Twiki syntax)
+            if do_twiki_edits:
+                for i in range(len(description)):
+                    if description[i].isupper() and (i == 0 or description[i-1] == ' '):
+                        description = description[:i] + '!' + description[i:]
+
+            ret.append((f'{configuration} ({releaseTemplate})', description))
+
+    # anti-chronological order
+    ret.reverse()
+
+    return ret
+
+###
+### main
+###
+if __name__ == '__main__':
+
+    ### args
+    parser = argparse.ArgumentParser(
+        prog = './'+os.path.basename(__file__),
+        formatter_class = argparse.RawDescriptionHelpFormatter,
+        description = __doc__
+    )
+
+    parser.add_argument('config',
+                        type = str,
+                        help = 'Name of HLT configuration in ConfDB (without specifying its version number)')
+
+    parser.add_argument('-d', '--db',
+                        dest = 'db',
+                        action = 'store',
+                        default = 'run3',
+                        choices = ['run3', 'run2', 'dev', 'adg'],
+                        help = 'Keyword to identify the target database (must be "run3", "run2", "dev", or "adg") [default: "run3"]')
+
+    parser.add_argument('-c', '--connect-string',
+                        dest = 'connect_string',
+                        action = 'store',
+                        default = None,
+                        help = 'Argument of cx_Oracle.connect (argument of "--db" will be ignored) [default: None]')
+
+    parser.add_argument('--no-hyperlinks',
+                        dest = 'add_hyperlinks',
+                        action = 'store_false',
+                        default = True,
+                        help = 'Do not include hyperlinks [default: False]')
+
+    parser.add_argument('--no-twiki',
+                        dest = 'do_twiki_edits',
+                        action = 'store_false',
+                        default = True,
+                        help = 'Do not adapt to Twiki syntax [default: False]')
+
+    opts, opts_unknown = parser.parse_known_args()
+    ### ----
+
+    if len(opts_unknown) > 0:
+        raise RuntimeError('unsupported command-line arguments: '+str(opts_unknown))
+
+    connect_string = opts.connect_string
+
+    if connect_string != None:
+        print(f'HLT Configuration: {opts.config} (connect = "{opts.connect_string}")\n')
+    else:
+        print(f'HLT Configuration: {opts.config} (database = "{opts.db}")\n')
+        if opts.db == 'run3':
+            # Run-3 (offline) db
+            connect_string = 'cms_hlt_v3_r/convertMe!@cmsr'
+        elif opts.db == 'run2':
+            # Run-2 (offline) db
+            connect_string = 'cms_hlt_gdr_r/convertMe!@cmsr'
+        elif opts.db == 'dev':
+            # dev db
+            connect_string = 'cms_hlt_gdrdev_r/convertMe1!@cmsr'
+        elif opts.db == 'adg':
+            # ADG (read-only copy of online db)
+            connect_string = 'cms_hlt_gdr_r/convertMe!@cms_orcon_adg'
+        else:
+            raise RuntimeError(f'invalid keyword for target database: "{opts.db}"')
+
+    hltMenuVersions = getHLTMenuVersions(
+        configName = opts.config,
+        connect_string = connect_string,
+        add_hyperlinks = opts.add_hyperlinks,
+        do_twiki_edits = opts.do_twiki_edits
+    )
+
+    if len(hltMenuVersions) == 0:
+        print('No Configuration Found !!\n')
+        raise SystemExit(1)
+
+    for (configuration, description) in hltMenuVersions:
+        print(f'   * ={configuration}=: {description}')
+    print()

HLTrigger/Configuration/scripts/utils/.gitignore

@@ -0,0 +1 @@
+*.csv

HLTrigger/Configuration/scripts/utils/hltListPathsWithoutOwners

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+import os
+import json
+import argparse
+import subprocess
+
+import FWCore.ParameterSet.Config as cms
+
+import HLTrigger.Configuration.Tools.options as options
+from HLTrigger.Configuration.extend_argparse import *
+
+def getHLTProcess(config):
+    if config.menu.run:
+        configline = f'--runNumber {config.menu.run}'
+    else:
+        configline = f'--{config.menu.database} --{config.menu.version} --configName {config.menu.name}'
+
+    # cmd to download HLT configuration
+    cmdline = f'hltConfigFromDB {configline} --noedsources --noes --nooutput'
+    if config.proxy:
+        cmdline += f' --dbproxy --dbproxyhost {config.proxy_host} --dbproxyport {config.proxy_port}'
+
+    # download HLT configuration
+    proc = subprocess.Popen(cmdline, shell = True, stdin = None, stdout = subprocess.PIPE, stderr = None)
+    (out, err) = proc.communicate()
+
+    # load HLT configuration
+    try:
+        foo = {'process': None}
+        exec(out, foo)
+        process = foo['process']
+    except:
+        raise Exception(f'query did not return a valid python file:\n query="{cmdline}"')
+
+    if not isinstance(process, cms.Process):
+        raise Exception(f'query did not return a valid HLT menu:\n query="{cmdline}"')
+
+    return process
+
+def main():
+    # define an argparse parser to parse our options
+    textwidth = int( 80 )
+    try:
+        textwidth = int( os.popen("stty size", "r").read().split()[1] )
+    except:
+        pass
+    formatter = FixedWidthFormatter( HelpFormatterRespectNewlines, width = textwidth )
+
+    # read defaults
+    defaults = options.HLTProcessOptions()
+
+    parser = argparse.ArgumentParser(
+        description = 'Create outputs to announce the release of a new HLT menu.',
+        argument_default = argparse.SUPPRESS,
+        formatter_class = formatter,
+        add_help = False
+    )
+
+    # required argument
+    parser.add_argument('menu',
+                        action  = 'store',
+                        type    = options.ConnectionHLTMenu,
+                        metavar = 'MENU',
+                        help    = 'HLT menu to dump from the database. Supported formats are:\n  - /path/to/configuration[/Vn]\n  - [[{v1|v2|v3}/]{run3|run2|online|adg}:]/path/to/configuration[/Vn]\n  - run:runnumber\nThe possible converters are "v1", "v2, and "v3" (default).\nThe possible databases are "run3" (default, used for offline development), "run2" (used for accessing run2 offline development menus), "online" (used to extract online menus within Point 5) and "adg" (used to extract the online menus outside Point 5).\nIf no menu version is specified, the latest one is automatically used.\nIf "run:" is used instead, the HLT menu used for the given run number is looked up and used.\nNote other converters and databases exist as options but they are only for expert/special use.' )
+
+    # options
+    parser.add_argument('--dbproxy',
+                        dest    = 'proxy',
+                        action  = 'store_true',
+                        default = defaults.proxy,
+                        help    = 'Use a socks proxy to connect outside CERN network (default: False)' )
+    parser.add_argument('--dbproxyport',
+                        dest    = 'proxy_port',
+                        action  = 'store',
+                        metavar = 'PROXYPORT',
+                        default = defaults.proxy_port,
+                        help    = 'Port of the socks proxy (default: 8080)' )
+    parser.add_argument('--dbproxyhost',
+                        dest    = 'proxy_host',
+                        action  = 'store',
+                        metavar = 'PROXYHOST',
+                        default = defaults.proxy_host,
+                        help    = 'Host of the socks proxy (default: "localhost")' )
+
+    parser.add_argument('--metadata-json',
+                        dest    = 'metadata_json',
+                        action  = 'store',
+                        default = 'hltPathOwners.json',
+                        help    = 'Path to .json file with metadata on HLT Paths (online?, group-owners)' )
+
+    # redefine "--help" to be the last option, and use a customized message 
+    parser.add_argument('-h', '--help', 
+                        action  = 'help', 
+                        help    = 'Show this help message and exit' )
+
+    # parse command line arguments and options
+    config = parser.parse_args()
+
+    if not os.path.isfile(config.metadata_json):
+        raise RuntimeError(f'invalid path to metadata JSON file [--metadata-json]: {config.metadata_json}')
+
+    metadataDict = json.load(open(config.metadata_json))
+
+    process = getHLTProcess(config)
+
+    pathNames = sorted([pathName if '_v' not in pathName else pathName[:pathName.rfind('_v')]+'_v' for pathName, path in process.paths_().items() if not pathName.startswith('Dataset_')])
+
+    for pathName in pathNames:
+        if pathName not in metadataDict:
+            print(pathName)
+
+###
+### main
+###
+if __name__ == '__main__':
+    main()