diff --git a/docker/compose/slycat-compose/scripts/slycat-docker-compose-couchdb-load-data.py b/docker/compose/slycat-compose/scripts/slycat-docker-compose-couchdb-load-data.py index 84f0aa389..f1f17a351 100644 --- a/docker/compose/slycat-compose/scripts/slycat-docker-compose-couchdb-load-data.py +++ b/docker/compose/slycat-compose/scripts/slycat-docker-compose-couchdb-load-data.py @@ -15,24 +15,53 @@ import time parser = argparse.ArgumentParser() -parser.add_argument("--input-dir", default="/usr/src/slycat/slycat/docker/compose/slycat-compose/DB", help="Directory containing data dumped with slycat-dump.py.") -parser.add_argument("--couchdb-database", default="slycat", help="CouchDB database. Default: %(default)s") -parser.add_argument("--couchdb-host", default="couchdb", help="CouchDB host. Default: %(default)s") -parser.add_argument("--couchdb-port", type=int, default=5984, help="CouchDB port. Default: %(default)s") -parser.add_argument("--port", default="5984", help="CouchDB port. Default: %(default)s") -parser.add_argument("--admin", default="admin", help="CouchDB admin user. Default: %(default)s") -parser.add_argument("--password", default="password", help="CouchDB admin password. Default: %(default)s") +parser.add_argument( + "--input-dir", + default="/usr/src/slycat/slycat/docker/compose/slycat-compose/DB", + help="Directory containing data dumped with slycat-dump.py.", +) +parser.add_argument( + "--couchdb-database", + default="slycat", + help="CouchDB database. Default: %(default)s", +) +parser.add_argument( + "--couchdb-host", default="couchdb", help="CouchDB host. Default: %(default)s" +) +parser.add_argument( + "--couchdb-port", type=int, default=5984, help="CouchDB port. Default: %(default)s" +) +parser.add_argument( + "--port", default="5984", help="CouchDB port. Default: %(default)s" +) +parser.add_argument( + "--admin", default="admin", help="CouchDB admin user. Default: %(default)s" +) +parser.add_argument( + "--password", + default="password", + help="CouchDB admin password. Default: %(default)s", +) -parser.add_argument("--data-store", default="/var/lib/slycat/data-store", - help="Path to the hdf5 data storage directory. Default: %(default)s") +parser.add_argument( + "--data-store", + default="/var/lib/slycat/data-store", + help="Path to the hdf5 data storage directory. Default: %(default)s", +) parser.add_argument("--force", action="store_true", help="Overwrite existing data.") -parser.add_argument("--marking", default=[], nargs="+", - help="Use --marking=':' to map markings to markings. You may specify multiple maps, separated by whitespace.") +parser.add_argument( + "--marking", + default=[], + nargs="+", + help="Use --marking=':' to map markings to markings. You may specify multiple maps, separated by whitespace.", +) arguments = parser.parse_args() logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) -logging.getLogger().handlers[0].setFormatter(logging.Formatter("{} - %(levelname)s - %(message)s".format(sys.argv[0]))) +logging.getLogger().handlers[0].setFormatter( + logging.Formatter("{} - %(levelname)s - %(message)s".format(sys.argv[0])) +) # Sanity check input arguments ... markings = [marking.split(":") for marking in arguments.marking] @@ -41,20 +70,20 @@ # assuming CouchDB initialization from local process to local server creds = "" if arguments.admin != "": - creds = arguments.admin + ":" + arguments.password + "@" + creds = arguments.admin + ":" + arguments.password + "@" serverURL = "http://" + creds + arguments.couchdb_host + ":" + arguments.port + "/" logging.error("couch serverURL:%s" % serverURL) while True: - try: - couchdb_server = couchdb.Server(serverURL) - version = couchdb_server.version() - couchdb = couchdb_server[arguments.couchdb_database] - break - except Exception as e: - logging.error("Waiting for couchdb for data load.") - logging.error(e.msg) - time.sleep(2) + try: + couchdb_server = couchdb.Server(serverURL) + version = couchdb_server.version() + couchdb = couchdb_server[arguments.couchdb_database] + break + except Exception as e: + logging.error("Waiting for couchdb for data load.") + logging.error(e.msg) + time.sleep(2) # --host couchdb --admin admin --password password @@ -122,5 +151,8 @@ del couchdb[reference["_id"]] couchdb.save(reference) logging.info("Loading references Done") + except Exception: - logging.error("Not loading data resource conflict encountered data is probably already loaded") \ No newline at end of file + logging.error( + "Not loading data resource conflict encountered data is probably already loaded" + ) diff --git a/packages/slycat/web/server/cleanup.py b/packages/slycat/web/server/cleanup.py index 8f63b620f..6995d07b2 100644 --- a/packages/slycat/web/server/cleanup.py +++ b/packages/slycat/web/server/cleanup.py @@ -12,66 +12,151 @@ import time import sys + def _array_cleanup_worker(): - cherrypy.log.error("Started array cleanup worker.") - while True: - arrays.queue.get() + cherrypy.log.error("Started array cleanup worker.") while True: - try: - database = slycat.web.server.database.couchdb.connect() - #cherrypy.log.error("Array cleanup worker running.") - for file in database.view("slycat/hdf5-file-counts", group=True): - if file.value == 0: - slycat.web.server.hdf5.delete(file.key) - database.delete(database[file.key]) - cherrypy.log.error("Array cleanup worker finished.") - break - except Exception as e: - cherrypy.log.error("Array cleanup worker waiting for couchdb.") - time.sleep(2) - -_array_cleanup_worker.thread = threading.Thread(name="array-cleanup", target=_array_cleanup_worker) + arrays.queue.get() + while True: + try: + database = slycat.web.server.database.couchdb.connect() + # cherrypy.log.error("Array cleanup worker running.") + for file in database.view("slycat/hdf5-file-counts", group=True): + if file.value == 0: + slycat.web.server.hdf5.delete(file.key) + database.delete(database[file.key]) + # cherrypy.log.error("Array cleanup worker finished.") + break + except Exception as e: + cherrypy.log.error("Array cleanup worker waiting for couchdb.") + time.sleep(2) + + +_array_cleanup_worker.thread = threading.Thread( + name="array-cleanup", target=_array_cleanup_worker +) _array_cleanup_worker.thread.daemon = True + def _login_session_cleanup_worker(): - cherrypy.log.error("Started login session cleanup worker.") - while True: - try: - database = slycat.web.server.database.couchdb.connect() - #cherrypy.log.error("Login session cleanup worker running.") - cutoff = (datetime.datetime.utcnow() - cherrypy.request.app.config["slycat"]["session-timeout"]).isoformat() - for session in database.view("slycat/sessions", include_docs=True): - if session.doc["created"] < cutoff: - database.delete(session.doc) - cherrypy.log.error("Login session cleanup worker finished.") - time.sleep(datetime.timedelta(minutes=15).total_seconds()) - except Exception as e: - cherrypy.log.error("Login session cleanup worker waiting for couchdb. %s" % str(e)) - time.sleep(2) -_login_session_cleanup_worker.thread = threading.Thread(name="session-cleanup", target=_login_session_cleanup_worker) + cherrypy.log.error("Started login session cleanup worker.") + while True: + try: + database = slycat.web.server.database.couchdb.connect() + cherrypy.log.error("Login session cleanup worker running.") + cutoff = ( + datetime.datetime.now(datetime.timezone.utc) + - cherrypy.request.app.config["slycat"]["session-timeout"] + ).isoformat() + for session in database.view("slycat/sessions", include_docs=True): + if session.doc["created"] < cutoff: + database.delete(session.doc) + cherrypy.log.error("Login session cleanup worker finished.") + time.sleep(datetime.timedelta(minutes=15).total_seconds()) + except Exception as e: + cherrypy.log.error( + "Login session cleanup worker waiting for couchdb. %s" % str(e) + ) + time.sleep(2) + + +_login_session_cleanup_worker.thread = threading.Thread( + name="session-cleanup", target=_login_session_cleanup_worker +) _login_session_cleanup_worker.thread.daemon = True + def _cache_cleanup_worker(): - import cherrypy - from slycat.web.server import cache_it - cherrypy.log.error("Started server cache cleanup worker.") - while True: + import cherrypy + from slycat.web.server import cache_it + + cherrypy.log.error("Started server cache cleanup worker.") + while True: time.sleep(datetime.timedelta(minutes=15).total_seconds()) # cherrypy.log.error("[CACHE] running server cache-cleanup thread") cache_it.clean() -_cache_cleanup_worker.thread = threading.Thread(name="cache-cleanup", target=_cache_cleanup_worker) + +_cache_cleanup_worker.thread = threading.Thread( + name="cache-cleanup", target=_cache_cleanup_worker +) _cache_cleanup_worker.thread.daemon = True + +def _bookmark_cleanup_worker(): + """ + This thread daemon is designed to compile a list of bookmarks excluding any linked by references + and then test each bookmark's `last access time` against a cutoff time. If the cutoff time is greater + than the last access time the bookmark will be deleted from the system + """ + # on a thread so we need to import this + import cherrypy + + cherrypy.log.error("Started server cache cleanup worker.") + while True: + # set the run frequency + time.sleep(datetime.timedelta(days=1).total_seconds()) + if "bookmark-expiration" in cherrypy.request.app.config["slycat-web-server"]: + cutoff = ( + datetime.datetime.now(datetime.timezone.utc) + - cherrypy.request.app.config["slycat-web-server"][ + "bookmark-expiration" + ] + ).isoformat() + else: + cutoff = ( + datetime.datetime.now(datetime.timezone.utc) + - datetime.timedelta(weeks=56) + ).isoformat() + # lets make this a locked operation + with slycat.web.server.database.couchdb.db_lock: + database = slycat.web.server.database.couchdb.connect() + # build a list of bookmark ids that should not be deleted + references = [ + reference["bid"] + for reference in database.scan("slycat/references") + if "bid" in reference + ] + bookmarks_with_no_references = [ + bookmark + for bookmark in database.scan("slycat/project-bookmarks") + if bookmark["_id"] not in references + ] + count = 0 + for bookmark in bookmarks_with_no_references: + if "last_accessed" in bookmark: + if bookmark["last_accessed"] < cutoff: + database.delete(bookmark) + count = count + 1 + # no last_accessed field means its way too old so lets delete it + else: + database.delete(bookmark) + count = count + 1 + if count > 0: + cherrypy.log.error( + "[BOOKMARK-CLEANUP] bookmark-cleanup thread deleted %s bookmarks" + % str(count) + ) + + +_bookmark_cleanup_worker.thread = threading.Thread( + name="bookmark-cleanup", target=_bookmark_cleanup_worker +) +_bookmark_cleanup_worker.thread.daemon = True + + def start(): - """Called to start all of the cleanup worker threads.""" - _array_cleanup_worker.thread.start() - _login_session_cleanup_worker.thread.start() - _cache_cleanup_worker.thread.start() + """Called to start all of the cleanup worker threads.""" + _array_cleanup_worker.thread.start() + _login_session_cleanup_worker.thread.start() + _cache_cleanup_worker.thread.start() + _bookmark_cleanup_worker.thread.start() + def arrays(): - """Request a cleanup pass for unused arrays.""" - arrays.queue.put("cleanup") + """Request a cleanup pass for unused arrays.""" + arrays.queue.put("cleanup") + + arrays.queue = Queue() arrays.queue.put("cleanup") - diff --git a/packages/slycat/web/server/handlers.py b/packages/slycat/web/server/handlers.py index 054c1a038..1c92cc711 100644 --- a/packages/slycat/web/server/handlers.py +++ b/packages/slycat/web/server/handlers.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # Copyright (c) 2013, 2018 National Technology and Engineering Solutions of Sandia, LLC . Under the terms of Contract # DE-NA0003525 with National Technology and Engineering Solutions of Sandia, LLC, the U.S. Government # retains certain rights in this software. @@ -45,6 +45,7 @@ # decode base64 byte streams for file upload import base64 + class MyEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, numpy.integer): @@ -58,6 +59,7 @@ def default(self, obj): else: return super(MyEncoder, self).default(obj) + def get_sid(hostname): """ Takes a hostname address and returns the established sid value @@ -74,14 +76,32 @@ def get_sid(hostname): if host_session["hostname"] == hostname: sid = host_session["sid"] session_type = host_session["session_type"] - if(host_session["session_type"] == "ssh" and not slycat.web.server.remote.check_session(sid)): - cherrypy.log.error("error %s SID:%s Keys %s" % (slycat.web.server.remote.check_session(sid), sid, list(slycat.web.server.remote.session_cache.keys()))) + if host_session[ + "session_type" + ] == "ssh" and not slycat.web.server.remote.check_session(sid): + cherrypy.log.error( + "error %s SID:%s Keys %s" + % ( + slycat.web.server.remote.check_session(sid), + sid, + list(slycat.web.server.remote.session_cache.keys()), + ) + ) slycat.web.server.remote.delete_session(sid) del session["sessions"][index] database.save(session) raise cherrypy.HTTPError("404") - elif(host_session["session_type"] == "smb" and not slycat.web.server.smb.check_session(sid)): - cherrypy.log.error("error %s SID:%s Keys %s" % (slycat.web.server.smb.check_session(sid), sid, list(slycat.web.server.smb.session_cache.keys()))) + elif host_session[ + "session_type" + ] == "smb" and not slycat.web.server.smb.check_session(sid): + cherrypy.log.error( + "error %s SID:%s Keys %s" + % ( + slycat.web.server.smb.check_session(sid), + sid, + list(slycat.web.server.smb.session_cache.keys()), + ) + ) slycat.web.server.smb.delete_session(sid) del session["sessions"][index] database.save(session) @@ -97,14 +117,16 @@ def get_sid(hostname): def require_json_parameter(name): """ - checks to see if the parameter is in the cherrypy.request.json - and errors gracefully if it is not there - :param name: name of json param - :return: value of the json param + checks to see if the parameter is in the cherrypy.request.json + and errors gracefully if it is not there + :param name: name of json param + :return: value of the json param """ if name not in cherrypy.request.json: - cherrypy.log.error("slycat.web.server.handlers.py require_json_parameter", - "cherrypy.HTTPError 404 missing '%s' parameter." % name) + cherrypy.log.error( + "slycat.web.server.handlers.py require_json_parameter", + "cherrypy.HTTPError 404 missing '%s' parameter." % name, + ) raise cherrypy.HTTPError("400 Missing '%s' parameter." % name) return cherrypy.request.json[name] @@ -112,8 +134,10 @@ def require_json_parameter(name): def require_boolean_json_parameter(name): value = require_json_parameter(name) if value is not True and value is not False: - cherrypy.log.error("slycat.web.server.handlers.py require_boolean_json_parameter", - "cherrypy.HTTPError 400 '%s' parameter must be true or false." % name) + cherrypy.log.error( + "slycat.web.server.handlers.py require_boolean_json_parameter", + "cherrypy.HTTPError 400 '%s' parameter must be true or false." % name, + ) raise cherrypy.HTTPError("400 '%s' parameter must be true or false." % name) return value @@ -121,8 +145,10 @@ def require_boolean_json_parameter(name): def require_array_json_parameter(name): array = require_json_parameter(name) if not isinstance(array, list): - cherrypy.log.error("slycat.web.server.handlers.py require_array_json_parameter", - "cherrypy.HTTPError 400 '%s' parameter must be an array." % name) + cherrypy.log.error( + "slycat.web.server.handlers.py require_array_json_parameter", + "cherrypy.HTTPError 400 '%s' parameter must be an array." % name, + ) raise cherrypy.HTTPError("400 '%s' parameter must be an array." % name) return array @@ -132,9 +158,14 @@ def require_integer_array_json_parameter(name): try: array = [int(value) for value in array] except: - cherrypy.log.error("slycat.web.server.handlers.py require_integer_array_json_parameter", - "cherrypy.HTTPError 400 '%s' parameter must be an array of integers." % name) - raise cherrypy.HTTPError("400 '%s' parameter must be an array of integers." % name) + cherrypy.log.error( + "slycat.web.server.handlers.py require_integer_array_json_parameter", + "cherrypy.HTTPError 400 '%s' parameter must be an array of integers." + % name, + ) + raise cherrypy.HTTPError( + "400 '%s' parameter must be an array of integers." % name + ) return array @@ -142,10 +173,13 @@ def require_integer_parameter(value, name): try: return int(value) except: - cherrypy.log.error("slycat.web.server.handlers.py require_integer_parameter", - "cherrypy.HTTPError 400 '%s' parameter must be an integer." % name) + cherrypy.log.error( + "slycat.web.server.handlers.py require_integer_parameter", + "cherrypy.HTTPError 400 '%s' parameter must be an integer." % name, + ) raise cherrypy.HTTPError("400 '%s' parameter must be an integer." % name) + @cherrypy.tools.json_out(on=True) def is_user_currently_active(): """ @@ -154,17 +188,31 @@ def is_user_currently_active(): If less than 10 minutes ago, returns false. """ most_recent_time = datetime.timedelta(days=999) - time_threshold = datetime.timedelta(minutes = 10) + time_threshold = datetime.timedelta(minutes=10) database = slycat.web.server.database.couchdb.connect() for session in database.view("slycat/sessions", include_docs=True): - if session.doc["creator"] not in cherrypy.request.app.config["slycat"]["server-admins"] and (datetime.datetime.utcnow() - datetime.datetime.strptime(str(session.doc["last-active-time"]), '%Y-%m-%dT%H:%M:%S.%f')) < most_recent_time: + if ( + session.doc["creator"] + not in cherrypy.request.app.config["slycat"]["server-admins"] + and ( + datetime.datetime.utcnow() + - datetime.datetime.strptime( + str(session.doc["last-active-time"]), "%Y-%m-%dT%H:%M:%S.%f" + ) + ) + < most_recent_time + ): most_recent_time = session.doc["last-active-time"] - if (datetime.datetime.utcnow() - datetime.datetime.strptime(str(most_recent_time), '%Y-%m-%dT%H:%M:%S.%f')) >= time_threshold: + if ( + datetime.datetime.utcnow() + - datetime.datetime.strptime(str(most_recent_time), "%Y-%m-%dT%H:%M:%S.%f") + ) >= time_threshold: return True else: return False + @cherrypy.tools.json_out(on=True) def get_projects_list(_=None): """ @@ -173,11 +221,14 @@ def get_projects_list(_=None): :return: json projects list """ database = slycat.web.server.database.couchdb.connect() - projects = [project for project in database.scan("slycat/projects") if - slycat.web.server.authentication.is_project_reader( - project) or slycat.web.server.authentication.is_project_writer( - project) or slycat.web.server.authentication.is_project_administrator( - project) or slycat.web.server.authentication.is_server_administrator()] + projects = [ + project + for project in database.scan("slycat/projects") + if slycat.web.server.authentication.is_project_reader(project) + or slycat.web.server.authentication.is_project_writer(project) + or slycat.web.server.authentication.is_project_administrator(project) + or slycat.web.server.authentication.is_server_administrator() + ] projects = sorted(projects, key=lambda x: x["created"], reverse=True) return {"revision": 0, "projects": projects} @@ -196,18 +247,29 @@ def post_projects(): raise cherrypy.HTTPError("400 Missing project name.") database = slycat.web.server.database.couchdb.connect() - pid, rev = database.save({ - "type": "project", - "acl": {"administrators": [{"user": cherrypy.request.login}], "readers": [], "writers": [], "groups": []}, - "created": datetime.datetime.utcnow().isoformat(), - "creator": cherrypy.request.login, - "description": cherrypy.request.json.get("description", ""), - "name": cherrypy.request.json["name"] - }) - cherrypy.response.headers["location"] = "%s/projects/%s" % (cherrypy.request.base, pid) + pid, rev = database.save( + { + "type": "project", + "acl": { + "administrators": [{"user": cherrypy.request.login}], + "readers": [], + "writers": [], + "groups": [], + }, + "created": datetime.datetime.utcnow().isoformat(), + "creator": cherrypy.request.login, + "description": cherrypy.request.json.get("description", ""), + "name": cherrypy.request.json["name"], + } + ) + cherrypy.response.headers["location"] = "%s/projects/%s" % ( + cherrypy.request.base, + pid, + ) cherrypy.response.status = "201 Project created." return {"id": pid} + @cherrypy.tools.json_out(on=True) def get_project(pid): """ @@ -220,16 +282,19 @@ def get_project(pid): project = database.get("project", pid) slycat.web.server.authentication.require_project_reader(project) if slycat.web.server.authentication.is_server_administrator(): - project['acl']["server_administrators"] = [] - project['acl']["server_administrators"].append({'user':cherrypy.request.login}) + project["acl"]["server_administrators"] = [] + project["acl"]["server_administrators"].append({"user": cherrypy.request.login}) return project + def get_remote_host_dict(): remote_host_dict = cherrypy.request.app.config["slycat-web-server"]["remote-hosts"] remote_host_list = [] for host in remote_host_dict: if "message" in remote_host_dict[host]: - remote_host_list.append({"name": host, "message": remote_host_dict[host]["message"]}) + remote_host_list.append( + {"name": host, "message": remote_host_dict[host]["message"]} + ) else: remote_host_list.append({"name": host}) return remote_host_list @@ -238,10 +303,10 @@ def get_remote_host_dict(): @cherrypy.tools.json_in(on=True) def put_project(pid): """ - Takes json in the format of + Takes json in the format of acl: - administrators: list of object + administrators: list of object user:username writers: list of object @@ -254,7 +319,7 @@ def put_project(pid): name of the project description: string - description of the project, + description of the project, and updates the project json from this object. all top order fields are optional @@ -282,31 +347,54 @@ def put_project(pid): if "server_administrators" in cherrypy.request.json["acl"]: del cherrypy.request.json["acl"]["server_administrators"] if "administrators" not in cherrypy.request.json["acl"]: - cherrypy.log.error("slycat.web.server.handlers.py put_project", - "cherrypy.HTTPError 400 missing administrators.") + cherrypy.log.error( + "slycat.web.server.handlers.py put_project", + "cherrypy.HTTPError 400 missing administrators.", + ) raise cherrypy.HTTPError("400 missing administrators") if "writers" not in cherrypy.request.json["acl"]: - cherrypy.log.error("slycat.web.server.handlers.py put_project", - "cherrypy.HTTPError 400 missing writers.") + cherrypy.log.error( + "slycat.web.server.handlers.py put_project", + "cherrypy.HTTPError 400 missing writers.", + ) raise cherrypy.HTTPError("400 missing writers") if "readers" not in cherrypy.request.json["acl"]: - cherrypy.log.error("slycat.web.server.handlers.py put_project", - "cherrypy.HTTPError 400 missing readers.") + cherrypy.log.error( + "slycat.web.server.handlers.py put_project", + "cherrypy.HTTPError 400 missing readers.", + ) raise cherrypy.HTTPError("400 missing readers") add_acl = True for admin in cherrypy.request.json["acl"]["administrators"]: cherrypy.log.error("admin %s" % str(admin)) - if(cherrypy.request.app.config["slycat-web-server"]["directory"](admin['user']) is None): + if ( + cherrypy.request.app.config["slycat-web-server"]["directory"]( + admin["user"] + ) + is None + ): add_acl = False for reader in cherrypy.request.json["acl"]["readers"]: - if(cherrypy.request.app.config["slycat-web-server"]["directory"](reader['user']) is None): + if ( + cherrypy.request.app.config["slycat-web-server"]["directory"]( + reader["user"] + ) + is None + ): add_acl = False for writer in cherrypy.request.json["acl"]["writers"]: - if(cherrypy.request.app.config["slycat-web-server"]["directory"](writer['user']) is None): + if ( + cherrypy.request.app.config["slycat-web-server"]["directory"]( + writer["user"] + ) + is None + ): add_acl = False if not add_acl: - cherrypy.log.error("slycat.web.server.handlers.py put_project", - "cherrypy.HTTPError 403 un-verifiable user") + cherrypy.log.error( + "slycat.web.server.handlers.py put_project", + "cherrypy.HTTPError 403 un-verifiable user", + ) raise cherrypy.HTTPError("403 un-verifiable user") project["acl"] = cherrypy.request.json["acl"] if "name" in cherrypy.request.json: @@ -321,7 +409,7 @@ def put_project(pid): def delete_project(pid): """ Deletes a project and all its models. - + Arguments: pid {string} -- project id """ @@ -329,9 +417,13 @@ def delete_project(pid): couchdb = slycat.web.server.database.couchdb.connect() project = couchdb.get("project", pid) slycat.web.server.authentication.require_project_administrator(project) - for cache_object in couchdb.scan("slycat/project-cache-objects", startkey=pid, endkey=pid): + for cache_object in couchdb.scan( + "slycat/project-cache-objects", startkey=pid, endkey=pid + ): couchdb.delete(cache_object) - for reference in couchdb.scan("slycat/project-references", startkey=pid, endkey=pid): + for reference in couchdb.scan( + "slycat/project-references", startkey=pid, endkey=pid + ): couchdb.delete(reference) for bookmark in couchdb.scan("slycat/project-bookmarks", startkey=pid, endkey=pid): couchdb.delete(bookmark) @@ -350,10 +442,10 @@ def delete_project(pid): def get_project_models(pid, **kwargs): """ Returns a list of project models. - + Arguments: pid {string} -- project id - + Returns: json -- list of models in project """ @@ -361,10 +453,14 @@ def get_project_models(pid, **kwargs): project = database.get("project", pid) slycat.web.server.authentication.require_project_reader(project) - models = [model for model in database.scan("slycat/project-models", startkey=pid, endkey=pid)] + models = [ + model + for model in database.scan("slycat/project-models", startkey=pid, endkey=pid) + ] models = sorted(models, key=lambda x: x["created"], reverse=True) return models + @cherrypy.tools.json_out(on=True) def put_project_csv_data(pid, file_key, parser, mid, aids): """ @@ -386,21 +482,27 @@ def put_project_csv_data(pid, file_key, parser, mid, aids): # check for existing project_datas data types if not project_datas: cherrypy.log.error("[MICROSERVICE] The project_datas list is empty.") - raise cherrypy.HTTPError("404 There is no project data stored for this project. %s" % file_key) + raise cherrypy.HTTPError( + "404 There is no project data stored for this project. %s" % file_key + ) else: for item in project_datas: if item["project"] == pid and item["file_name"] == file_key: # find the stored data fid = item["_id"] hdf5_name = item["hdf5_name"] - hdf5_path = cherrypy.request.app.config["slycat-web-server"]["data-store"] + "/" + hdf5_name + hdf5_path = ( + cherrypy.request.app.config["slycat-web-server"]["data-store"] + + "/" + + hdf5_name + ) # Read HDF5 file for CSV data decoded_data = [] decoded_col = [] # determine the type of data we just got and if we need to extract it eg for csv files # HDF5 file path - if ('.h5' in item["file_name"]) or ('.hdf5' in item["file_name"]): + if (".h5" in item["file_name"]) or (".hdf5" in item["file_name"]): with open(hdf5_path, "rb") as fh: file_obj = fh.read() attachment.append(file_obj) @@ -408,30 +510,36 @@ def put_project_csv_data(pid, file_key, parser, mid, aids): else: with h5py.File(hdf5_path, "r") as hdf5_file: - cherrypy.log.error("project data %s:%s" % (item,str([ [key, val] for key, val in hdf5_file.items()]))) - data = list(hdf5_file['data_table']) - for col in data: - for item in col: - decoded_item = item.decode('utf-8') - decoded_col.append(decoded_item) - decoded_data.append(decoded_col) - decoded_col = [] - decoded_rows = numpy.array(decoded_data).T - - giant_csv_list = [] + cherrypy.log.error( + "project data %s:%s" + % ( + item, + str([[key, val] for key, val in hdf5_file.items()]), + ) + ) + data = list(hdf5_file["data_table"]) + for col in data: + for item in col: + decoded_item = item.decode("utf-8") + decoded_col.append(decoded_item) + decoded_data.append(decoded_col) + decoded_col = [] + decoded_rows = numpy.array(decoded_data).T + + giant_csv_list = [] + temp_row = [] + for row in decoded_rows: + for i, entry in enumerate(row): + if i == (len(row) - 1): + temp_row.append(str(entry) + "\n") + else: + temp_row.append(str(entry) + ",") + temp_row_string = "".join(temp_row) + giant_csv_list.append(temp_row_string) temp_row = [] - for row in decoded_rows: - for i, entry in enumerate(row): - if i == (len(row) - 1): - temp_row.append(str(entry) + '\n') - else: - temp_row.append(str(entry) + ',') - temp_row_string = ''.join(temp_row) - giant_csv_list.append(temp_row_string) - temp_row = [] - giant_csv_string = ''.join(giant_csv_list) - attachment.append(giant_csv_string) - + giant_csv_string = "".join(giant_csv_list) + attachment.append(giant_csv_string) + # if we didnt fined the file repspond with not found if fid is None: raise cherrypy.HTTPError("404 There was no file with name %s found." % file_key) @@ -444,14 +552,17 @@ def put_project_csv_data(pid, file_key, parser, mid, aids): model = database.get("model", mid) if isinstance(aids, str): - aids = aids.split(',') - slycat.web.server.parse_existing_file(database, parser, True, attachment, model, aids) + aids = aids.split(",") + slycat.web.server.parse_existing_file( + database, parser, True, attachment, model, aids + ) return {"Status": "Success"} + @cherrypy.tools.json_out(on=True) def get_project_file_names(pid): """ - Gets all the stored filenames on the server that can be used to make models for a + Gets all the stored filenames on the server that can be used to make models for a given project :param pid: project ID :return: array of found project data file names @@ -470,13 +581,19 @@ def get_project_file_names(pid): data.append({"file_name": project_data["file_name"]}) return data + @cherrypy.tools.json_out(on=True) def get_project_references(pid): database = slycat.web.server.database.couchdb.connect() project = database.get("project", pid) slycat.web.server.authentication.require_project_reader(project) - references = [reference for reference in database.scan("slycat/project-references", startkey=pid, endkey=pid)] + references = [ + reference + for reference in database.scan( + "slycat/project-references", startkey=pid, endkey=pid + ) + ] references = sorted(references, key=lambda x: x["created"]) return references @@ -498,17 +615,33 @@ def post_project_models(pid): model_type = require_json_parameter("model-type") allowed_model_types = list(slycat.web.server.plugin.manager.models.keys()) if model_type not in allowed_model_types: - cherrypy.log.error("slycat.web.server.handlers.py post_project_models", - "cherrypy.HTTPError allowed model types: %s" % ", ".join(allowed_model_types)) - raise cherrypy.HTTPError("400 Allowed model types: %s" % ", ".join(allowed_model_types)) + cherrypy.log.error( + "slycat.web.server.handlers.py post_project_models", + "cherrypy.HTTPError allowed model types: %s" + % ", ".join(allowed_model_types), + ) + raise cherrypy.HTTPError( + "400 Allowed model types: %s" % ", ".join(allowed_model_types) + ) marking = require_json_parameter("marking") - if marking not in cherrypy.request.app.config["slycat-web-server"]["allowed-markings"]: - cherrypy.log.error("slycat.web.server.handlers.py post_project_models", - "cherrypy.HTTPError 400 allowed marking types: %s" % ", ".join( - cherrypy.request.app.config["slycat-web-server"]["allowed-markings"])) - raise cherrypy.HTTPError("400 Allowed marking types: %s" % ", ".join( - cherrypy.request.app.config["slycat-web-server"]["allowed-markings"])) + if ( + marking + not in cherrypy.request.app.config["slycat-web-server"]["allowed-markings"] + ): + cherrypy.log.error( + "slycat.web.server.handlers.py post_project_models", + "cherrypy.HTTPError 400 allowed marking types: %s" + % ", ".join( + cherrypy.request.app.config["slycat-web-server"]["allowed-markings"] + ), + ) + raise cherrypy.HTTPError( + "400 Allowed marking types: %s" + % ", ".join( + cherrypy.request.app.config["slycat-web-server"]["allowed-markings"] + ) + ) name = require_json_parameter("name") description = cherrypy.request.json.get("description", "") @@ -536,15 +669,19 @@ def post_project_models(pid): } database.save(model) - cherrypy.response.headers["location"] = "%s/models/%s" % (cherrypy.request.base, mid) + cherrypy.response.headers["location"] = "%s/models/%s" % ( + cherrypy.request.base, + mid, + ) cherrypy.response.status = "201 Model created." return {"id": mid} + # @cherrypy.tools.json_in(on=True) # @cherrypy.tools.json_out(on=True) def create_project_data_from_pid(pid, file=None, file_name=None): """ - creates a project level data object from a project id + creates a project level data object from a project id that can be used to create new models in the current project :param file_name: artifact ID @@ -569,16 +706,25 @@ def isNumeric(some_thing): project = database.get("project", pid) slycat.web.server.authentication.require_project_writer(project) - str_data = str(file.file.read(), 'utf-8') - rows = [row for row in - csv.reader(str_data.splitlines(), delimiter=",", doublequote=True, escapechar=None, quotechar='"', - quoting=csv.QUOTE_MINIMAL, skipinitialspace=True)] + str_data = str(file.file.read(), "utf-8") + rows = [ + row + for row in csv.reader( + str_data.splitlines(), + delimiter=",", + doublequote=True, + escapechar=None, + quotechar='"', + quoting=csv.QUOTE_MINIMAL, + skipinitialspace=True, + ) + ] columns = numpy.array(rows).T for i, column in enumerate(columns): for j, entry in enumerate(column): - columns[i,j] = entry.encode("utf-8") + columns[i, j] = entry.encode("utf-8") column_names = [name.strip() for name in rows[0]] column_names = ["%eval_id"] + column_names @@ -592,7 +738,9 @@ def isNumeric(some_thing): columns[index] = numpy.array(columns[index], dtype="float64") column_types[index] = "float64" else: - stringType = "S" + str(len(columns[index][0])) # using length of first string for whole column + stringType = "S" + str( + len(columns[index][0]) + ) # using length of first string for whole column columns[index] = numpy.array(columns[index], dtype=stringType) column_types[index] = "string" except: @@ -604,14 +752,16 @@ def isNumeric(some_thing): hdf5_file_path = os.path.join(hdf5_path, hdf5_name) h5f = h5py.File((hdf5_file_path), "w") - h5f.create_dataset('data_table', data=numpy.array(columns, dtype='S')) + h5f.create_dataset("data_table", data=numpy.array(columns, dtype="S")) h5f.close() # Make the entry in couch for project data database = slycat.web.server.database.couchdb.connect() timestamp = time.time() - formatted_timestamp = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') + formatted_timestamp = datetime.datetime.fromtimestamp(timestamp).strftime( + "%Y-%m-%d %H:%M:%S" + ) did = uuid.uuid4().hex data = { @@ -619,15 +769,16 @@ def isNumeric(some_thing): "type": "project_data", "hdf5_name": hdf5_name, "file_name": formatted_timestamp + "_" + str(file_name), - "data_table": 'data-table', + "data_table": "data-table", "project": pid, - "mid": [''], + "mid": [""], "created": datetime.datetime.utcnow().isoformat(), "creator": cherrypy.request.login, } database.save(data) + def create_project_data(mid, aid, file): """ creates a project level data object that can be used to create new @@ -650,17 +801,19 @@ def isNumeric(some_thing): except ValueError: return False return True - - # If we received an HDF5 file, we don't need to create a new one. - # Just need to decode the bytes and store it as is. - if ('.h5' in aid[1]) or ('.hdf5' in aid[1]): + + # If we received an HDF5 file, we don't need to create a new one. + # Just need to decode the bytes and store it as is. + if (".h5" in aid[1]) or (".hdf5" in aid[1]): rows = [] for f in file: f_buffer = io.BytesIO(f) f_buffer.seek(0) - hdf5_path = cherrypy.request.app.config["slycat-web-server"]["data-store"] + "/" + hdf5_path = ( + cherrypy.request.app.config["slycat-web-server"]["data-store"] + "/" + ) unique_name = uuid.uuid4().hex hdf5_name = f"{unique_name}.hdf5" hdf5_file_path = os.path.join(hdf5_path, hdf5_name) @@ -679,14 +832,16 @@ def isNumeric(some_thing): project = database.get("project", model["project"]) pid = project["_id"] timestamp = time.time() - formatted_timestamp = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') + formatted_timestamp = datetime.datetime.fromtimestamp( + timestamp + ).strftime("%Y-%m-%d %H:%M:%S") did = uuid.uuid4().hex - #TODO review how we pass files to this + # TODO review how we pass files to this # our file got passed as a list by someone... if isinstance(file, list): file = file[0] - #TODO review how we pass aids to this + # TODO review how we pass aids to this # if true we are using the new file naming structure if len(aid) > 1: # looks like we got passed a list(list()) lets extract the name @@ -711,7 +866,9 @@ def isNumeric(some_thing): model["project_data"].append(did) database.save(model) database.save(data) - cherrypy.log.error("[MICROSERVICE] Added HDF5 project data %s." % data["file_name"]) + cherrypy.log.error( + "[MICROSERVICE] Added HDF5 project data %s." % data["file_name"] + ) # If we decide later to store inputs and outputs separately, this code will pull them out from the original file @@ -729,18 +886,18 @@ def isNumeric(some_thing): # We didn't get an HDF5 file, so do everything normally. else: rows = [] - split_file = file[0].split('\n') + split_file = file[0].split("\n") for row in split_file: row_list = [row] - split_row = row_list[0].split(',') + split_row = row_list[0].split(",") rows.append(split_row) columns = numpy.array(rows).T for i, column in enumerate(columns): for j, entry in enumerate(column): - columns[i,j] = entry.encode("utf-8") + columns[i, j] = entry.encode("utf-8") column_names = [name.strip() for name in rows[0]] column_names = ["%eval_id"] + column_names @@ -754,7 +911,9 @@ def isNumeric(some_thing): columns[index] = numpy.array(columns[index], dtype="float64") column_types[index] = "float64" else: - stringType = "S" + str(len(columns[index][0])) # using length of first string for whole column + stringType = "S" + str( + len(columns[index][0]) + ) # using length of first string for whole column columns[index] = numpy.array(columns[index], dtype=stringType) column_types[index] = "string" except: @@ -768,7 +927,7 @@ def isNumeric(some_thing): hdf5_file_path = os.path.join(hdf5_path, hdf5_name) h5f = h5py.File((hdf5_file_path), "w") - h5f.create_dataset('data_table', data=numpy.array(columns, dtype='S')) + h5f.create_dataset("data_table", data=numpy.array(columns, dtype="S")) h5f.close() # Make the entry in couch for project data @@ -781,14 +940,16 @@ def isNumeric(some_thing): project = database.get("project", model["project"]) pid = project["_id"] timestamp = time.time() - formatted_timestamp = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') + formatted_timestamp = datetime.datetime.fromtimestamp(timestamp).strftime( + "%Y-%m-%d %H:%M:%S" + ) did = uuid.uuid4().hex - #TODO review how we pass files to this + # TODO review how we pass files to this # our file got passed as a list by someone... if isinstance(file, list): file = file[0] - #TODO review how we pass aids to this + # TODO review how we pass aids to this # if true we are using the new file naming structure if len(aid) > 1: # looks like we got passed a list(list()) lets extract the name @@ -815,13 +976,14 @@ def isNumeric(some_thing): database.save(data) cherrypy.log.error("[MICROSERVICE] Added project data %s." % data["file_name"]) + @cherrypy.tools.json_in(on=True) @cherrypy.tools.json_out(on=True) def post_log(): """ send post json {"message":"message"} to log client errors onto the client server - :return: + :return: """ message = require_json_parameter("message") cherrypy.log.error("[CLIENT/JAVASCRIPT]:: %s" % (message)) @@ -834,9 +996,12 @@ def post_project_bookmarks(pid): database = slycat.web.server.database.couchdb.connect() project = database.get("project", pid) slycat.web.server.authentication.require_project_reader( - project) # This is intentionally out-of-the-ordinary - we explicitly allow project *readers* to store bookmarks. + project + ) # This is intentionally out-of-the-ordinary - we explicitly allow project *readers* to store bookmarks. - content = json.dumps(cherrypy.request.json, separators=(",", ":"), indent=None, sort_keys=True) + content = json.dumps( + cherrypy.request.json, separators=(",", ":"), indent=None, sort_keys=True + ) bid = hashlib.md5(pid.encode() + content.encode()).hexdigest() with slycat.web.server.database.couchdb.db_lock: try: @@ -846,13 +1011,23 @@ def post_project_bookmarks(pid): "_id": bid, "project": pid, "type": "bookmark", - "created": datetime.datetime.utcnow().isoformat(), - "last_accessed": datetime.datetime.utcnow().isoformat() + "created": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "last_accessed": datetime.datetime.now( + datetime.timezone.utc + ).isoformat(), } database.save(doc) - database.put_attachment(doc, filename="bookmark", content_type="application/json", content=content) - - cherrypy.response.headers["location"] = "%s/bookmarks/%s" % (cherrypy.request.base, bid) + database.put_attachment( + doc, + filename="bookmark", + content_type="application/json", + content=content, + ) + + cherrypy.response.headers["location"] = "%s/bookmarks/%s" % ( + cherrypy.request.base, + bid, + ) cherrypy.response.status = "201 Bookmark stored." return {"id": bid} @@ -879,7 +1054,10 @@ def post_project_references(pid): } database.save(reference) - cherrypy.response.headers["location"] = "%s/references/%s" % (cherrypy.request.base, rid) + cherrypy.response.headers["location"] = "%s/references/%s" % ( + cherrypy.request.base, + rid, + ) cherrypy.response.status = "201 Reference created." return {"id": rid} @@ -901,14 +1079,15 @@ def put_reference(rid): cherrypy.response.status = "201 Reference updated." + @cherrypy.tools.json_out(on=True) def get_model(mid, **kwargs): """ Returns a model. - + Arguments: mid {string} -- model id - + Returns: json -- represents a model """ @@ -918,14 +1097,15 @@ def get_model(mid, **kwargs): slycat.web.server.authentication.require_project_reader(project) return model + @cherrypy.tools.json_out(on=True) def get_project_data(did, **kwargs): """ Returns a project data. - + Arguments: did {string} -- data id - + Returns: json -- represents a project data """ @@ -935,6 +1115,7 @@ def get_project_data(did, **kwargs): slycat.web.server.authentication.require_project_reader(project) return project_data + @cherrypy.tools.json_out(on=True) def get_project_data_parameter(did, param, **kwargs): """ @@ -943,7 +1124,7 @@ def get_project_data_parameter(did, param, **kwargs): Arguments: did {string} -- data id param {string} -- name of parameter to return - + Returns: json -- represents a project data parameter """ @@ -951,14 +1132,19 @@ def get_project_data_parameter(did, param, **kwargs): project_data = database.get("project_data", did) project = database.get("project", project_data["project"]) slycat.web.server.authentication.require_project_reader(project) - + try: - return slycat.web.server.get_project_data_parameter(database, project_data, param) + return slycat.web.server.get_project_data_parameter( + database, project_data, param + ) except KeyError as e: - cherrypy.log.error("slycat.web.server.handlers.py get_project_data_parameter", - "cherrypy.HTTPError 404 unknown parameter: %s" % param) + cherrypy.log.error( + "slycat.web.server.handlers.py get_project_data_parameter", + "cherrypy.HTTPError 404 unknown parameter: %s" % param, + ) raise cherrypy.HTTPError("404 Unknown parameter: %s" % param) + @cherrypy.tools.json_out(on=True) def get_project_data_in_model(mid, **kwargs): """ @@ -979,6 +1165,7 @@ def get_project_data_in_model(mid, **kwargs): else: return [] + def delete_project_data(did, **kwargs): """ Delete a project data record from couchdb. @@ -1024,10 +1211,10 @@ def model_command(mid, type, command, **kwargs): mid {string} -- model id type {string} -- Unique command category. command {string} -- Custom command name. - + Raises: cherrypy.HTTPError: 400 Unknown command: - + Returns: any -- whatever the registered command """ @@ -1038,11 +1225,14 @@ def model_command(mid, type, command, **kwargs): key = (cherrypy.request.method, type, command) if key in slycat.web.server.plugin.manager.model_commands: - return slycat.web.server.plugin.manager.model_commands[key](database, model, cherrypy.request.method, type, - command, **kwargs) + return slycat.web.server.plugin.manager.model_commands[key]( + database, model, cherrypy.request.method, type, command, **kwargs + ) - cherrypy.log.error("slycat.web.server.handlers.py model_command", - "cherrypy.HTTPError 400 unknown command: %s" % command) + cherrypy.log.error( + "slycat.web.server.handlers.py model_command", + "cherrypy.HTTPError 400 unknown command: %s" % command, + ) raise cherrypy.HTTPError("400 Unknown command: %s" % command) @@ -1056,11 +1246,14 @@ def model_sensitive_command(mid, type, command): key = (cherrypy.request.method, type, command) if key in slycat.web.server.plugin.manager.model_commands: - return slycat.web.server.plugin.manager.model_commands[key](database, model, cherrypy.request.method, type, - command, **kwargs) + return slycat.web.server.plugin.manager.model_commands[key]( + database, model, cherrypy.request.method, type, command, **kwargs + ) - cherrypy.log.error("slycat.web.server.handlers.py model_sensitive_command", - "cherrypy.HTTPError 400 unknown command: %s" % command) + cherrypy.log.error( + "slycat.web.server.handlers.py model_sensitive_command", + "cherrypy.HTTPError 400 unknown command: %s" % command, + ) raise cherrypy.HTTPError("400 Unknown command: %s" % command) @@ -1074,20 +1267,34 @@ def put_model(mid): slycat.web.server.authentication.require_project_writer(project) save_model = False - model = database.get('model', model["_id"]) + model = database.get("model", model["_id"]) for key, value in cherrypy.request.json.items(): - if key not in ["name", "description", "state", "result", "progress", "message", "started", "finished", - "marking", "bookmark"]: - cherrypy.log.error("slycat.web.server.handlers.py put_model", - "cherrypy.HTTPError 400 unknown model parameter: %s" % key) + if key not in [ + "name", + "description", + "state", + "result", + "progress", + "message", + "started", + "finished", + "marking", + "bookmark", + ]: + cherrypy.log.error( + "slycat.web.server.handlers.py put_model", + "cherrypy.HTTPError 400 unknown model parameter: %s" % key, + ) raise cherrypy.HTTPError("400 Unknown model parameter: %s" % key) if key in ["started", "finished"]: try: datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%f") except: - cherrypy.log.error("slycat.web.server.handlers.py put_model", - "cherrypy.HTTPError 400 timestamp fields must use ISO-8601.") + cherrypy.log.error( + "slycat.web.server.handlers.py put_model", + "cherrypy.HTTPError 400 timestamp fields must use ISO-8601.", + ) raise cherrypy.HTTPError("400 Timestamp fields must use ISO-8601.") if value != model.get(key): @@ -1096,10 +1303,10 @@ def put_model(mid): # Revisit this, how booksmark IDs get added to model - #if key == "bookmark": + # if key == "bookmark": # model[key].append(value) # save_model = True - #else: + # else: # model[key] = value # save_model = True @@ -1118,25 +1325,40 @@ def post_model_finish(mid): # check state of the model if model["state"] != "waiting": - cherrypy.log.error("slycat.web.server.handlers.py post_model_finish", - "cherrypy.HTTPError 400 only waiting models can be finished.") + cherrypy.log.error( + "slycat.web.server.handlers.py post_model_finish", + "cherrypy.HTTPError 400 only waiting models can be finished.", + ) raise cherrypy.HTTPError("400 Only waiting models can be finished.") # check if the model type exists if model["model-type"] not in list(slycat.web.server.plugin.manager.models.keys()): - cherrypy.log.error("slycat.web.server.handlers.py post_model_finish", - "cherrypy.HTTPError 500 cannot finish unknown model type.") + cherrypy.log.error( + "slycat.web.server.handlers.py post_model_finish", + "cherrypy.HTTPError 500 cannot finish unknown model type.", + ) raise cherrypy.HTTPError("500 Cannot finish unknown model type.") # - slycat.web.server.update_model(database, model, state="running", started=datetime.datetime.utcnow().isoformat(), - progress=0.0) - slycat.web.server.plugin.manager.models[model["model-type"]]["finish"](database, model) + slycat.web.server.update_model( + database, + model, + state="running", + started=datetime.datetime.utcnow().isoformat(), + progress=0.0, + ) + slycat.web.server.plugin.manager.models[model["model-type"]]["finish"]( + database, model + ) cherrypy.response.status = "202 Finishing model." -def post_model_files(mid, input=None, files=None, sids=None, paths=None, aids=None, parser=None, **kwargs): +def post_model_files( + mid, input=None, files=None, sids=None, paths=None, aids=None, parser=None, **kwargs +): if input is None: - cherrypy.log.error("slycat.web.server.handlers.py post_model_files", - "cherrypy.HTTPError 400 required input parameter is missing.") + cherrypy.log.error( + "slycat.web.server.handlers.py post_model_files", + "cherrypy.HTTPError 400 required input parameter is missing.", + ) raise cherrypy.HTTPError("400 Required input parameter is missing.") input = True if input == "true" else False @@ -1150,22 +1372,32 @@ def post_model_files(mid, input=None, files=None, sids=None, paths=None, aids=No if not isinstance(paths, list): paths = [paths] if len(sids) != len(paths): - cherrypy.log.error("slycat.web.server.handlers.py post_model_files", - "cherrypy.HTTPError 400 sids and paths parameter must have the same length.") - raise cherrypy.HTTPError("400 sids and paths parameters must have the same length.") + cherrypy.log.error( + "slycat.web.server.handlers.py post_model_files", + "cherrypy.HTTPError 400 sids and paths parameter must have the same length.", + ) + raise cherrypy.HTTPError( + "400 sids and paths parameters must have the same length." + ) files = [] for sid, path in zip(sids, paths): with slycat.web.server.remote.get_session(sid) as session: filename = "%s@%s:%s" % (session.username, session.hostname, path) if stat.S_ISDIR(session.sftp.stat(path).st_mode): - cherrypy.log.error("slycat.web.server.handlers.py post_model_files", - "cherrypy.HTTPError 400 cannot load directory %s." % filename) + cherrypy.log.error( + "slycat.web.server.handlers.py post_model_files", + "cherrypy.HTTPError 400 cannot load directory %s." % filename, + ) raise cherrypy.HTTPError("400 Cannot load directory %s." % filename) files.append(session.sftp.file(path).read()) else: - cherrypy.log.error("slycat.web.server.handlers.py post_model_files", - "cherrypy.HTTPError 400 must supply files parameter, or sids and path parameters.") - raise cherrypy.HTTPError("400 Must supply files parameter, or sids and paths parameters.") + cherrypy.log.error( + "slycat.web.server.handlers.py post_model_files", + "cherrypy.HTTPError 400 must supply files parameter, or sids and path parameters.", + ) + raise cherrypy.HTTPError( + "400 Must supply files parameter, or sids and paths parameters." + ) if aids is None: aids = [] @@ -1173,12 +1405,16 @@ def post_model_files(mid, input=None, files=None, sids=None, paths=None, aids=No aids = [aids] if parser is None: - cherrypy.log.error("slycat.web.server.handlers.py post_model_files", - "cherrypy.HTTPError 400 required parser parameter is missing.") + cherrypy.log.error( + "slycat.web.server.handlers.py post_model_files", + "cherrypy.HTTPError 400 required parser parameter is missing.", + ) raise cherrypy.HTTPError("400 Required parser parameter is missing.") if parser not in slycat.web.server.plugin.manager.parsers: - cherrypy.log.error("slycat.web.server.handlers.py post_model_files", - "cherrypy.HTTPError 400 unknown parser plugin: %s" % parser) + cherrypy.log.error( + "slycat.web.server.handlers.py post_model_files", + "cherrypy.HTTPError 400 unknown parser plugin: %s" % parser, + ) raise cherrypy.HTTPError("400 Unknown parser plugin: %s." % parser) database = slycat.web.server.database.couchdb.connect() @@ -1187,12 +1423,16 @@ def post_model_files(mid, input=None, files=None, sids=None, paths=None, aids=No slycat.web.server.authentication.require_project_writer(project) try: - slycat.web.server.plugin.manager.parsers[parser]["parse"](database, model, input, files, aids, **kwargs) + slycat.web.server.plugin.manager.parsers[parser]["parse"]( + database, model, input, files, aids, **kwargs + ) # create_project_data(mid, aids, files) except Exception as e: cherrypy.log.error("handles Exception parsing posted files: %s" % e) - cherrypy.log.error("slycat.web.server.handlers.py post_model_files", - "cherrypy.HTTPError 400 %s" % str(e)) + cherrypy.log.error( + "slycat.web.server.handlers.py post_model_files", + "cherrypy.HTTPError 400 %s" % str(e), + ) raise cherrypy.HTTPError("400 %s" % str(e)) @@ -1208,16 +1448,24 @@ def post_uploads(): parser = require_json_parameter("parser") aids = require_json_parameter("aids") if parser not in slycat.web.server.plugin.manager.parsers: - cherrypy.log.error("slycat.web.server.handlers.py post_uploads", - "cherrypy.HTTPError 400 unknown parser plugin: %s." % parser) + cherrypy.log.error( + "slycat.web.server.handlers.py post_uploads", + "cherrypy.HTTPError 400 unknown parser plugin: %s." % parser, + ) raise cherrypy.HTTPError("400 Unknown parser plugin: %s." % parser) - kwargs = {key: value for key, value in list(cherrypy.request.json.items()) if - key not in ["mid", "input", "parser", "aids"]} + kwargs = { + key: value + for key, value in list(cherrypy.request.json.items()) + if key not in ["mid", "input", "parser", "aids"] + } uid = slycat.web.server.upload.create_session(mid, input, parser, aids, kwargs) - cherrypy.response.headers["location"] = "%s/uploads/%s" % (cherrypy.request.base, uid) + cherrypy.response.headers["location"] = "%s/uploads/%s" % ( + cherrypy.request.base, + uid, + ) cherrypy.response.status = "201 Upload started." return {"id": uid} @@ -1232,7 +1480,7 @@ def put_upload_file_part(uid, fid, pid, file=None, hostname=None, path=None): # check for byte stream if type(file) == str: data = base64.b64decode(file) - + # otherwise it's a file stream else: data = file.file.read() @@ -1250,9 +1498,13 @@ def put_upload_file_part(uid, fid, pid, file=None, hostname=None, path=None): with slycat.web.server.upload.get_session(uid) as session: session.put_upload_file_part(fid, pid, data) else: - cherrypy.log.error("slycat.web.server.handlers.py put_upload_file_part", - "cherrypy.HTTPError 400 must supply file parameter, or sid and path parameters.") - raise cherrypy.HTTPError("400 Must supply file parameter, or sid and path parameters.") + cherrypy.log.error( + "slycat.web.server.handlers.py put_upload_file_part", + "cherrypy.HTTPError 400 must supply file parameter, or sid and path parameters.", + ) + raise cherrypy.HTTPError( + "400 Must supply file parameter, or sid and path parameters." + ) @cherrypy.tools.json_in(on=True) @@ -1268,7 +1520,7 @@ def post_upload_finished(uid): # check that useProjectData is present before using # otherwise, default to False useProjectData = False - if 'useProjectData' in cherrypy.request.json: + if "useProjectData" in cherrypy.request.json: useProjectData = require_boolean_json_parameter("useProjectData") with slycat.web.server.upload.get_session(uid) as session: @@ -1277,16 +1529,16 @@ def post_upload_finished(uid): def delete_upload(uid): """ - Delete an upload session used to upload files for storage as model artifacts. - This function must be called once the client no longer needs the session, - whether the upload(s) have been completed successfully or the + Delete an upload session used to upload files for storage as model artifacts. + This function must be called once the client no longer needs the session, + whether the upload(s) have been completed successfully or the client is cancelling an incomplete session. status 204 The upload session and any temporary storage have been deleted. status 409 - The upload session cannot be deleted, because parsing is in progress. + The upload session cannot be deleted, because parsing is in progress. Try again later. :param uid: upload sessin id @@ -1303,18 +1555,28 @@ def open_id_authenticate(**params): :param params: openid params as a dictionary :return: not used """ - cherrypy.log.error("++ open_id_authenticate starting, incoming params = %s" % params) + cherrypy.log.error( + "++ open_id_authenticate starting, incoming params = %s" % params + ) # check for openid in the config for security - if slycat.web.server.config["slycat-web-server"]["authentication"]["plugin"] != "slycat-openid-authentication": + if ( + slycat.web.server.config["slycat-web-server"]["authentication"]["plugin"] + != "slycat-openid-authentication" + ): raise cherrypy.HTTPError(404) cherrypy.log.error("++ open_id_authenticate incoming params = %s" % params) current_url = urlparse(cherrypy.url() + "?" + cherrypy.request.query_string) - - kerberos_principal = params['openid.ext2.value.Authuser'] + + kerberos_principal = params["openid.ext2.value.Authuser"] auth_user = kerberos_principal.split("@")[0] - cherrypy.log.error("++ open_id_authenticate: setting auth_user = %s, calling create_single_sign_on_session" % auth_user) - slycat.web.server.create_single_sign_on_session(slycat.web.server.check_https_get_remote_ip(), auth_user) + cherrypy.log.error( + "++ open_id_authenticate: setting auth_user = %s, calling create_single_sign_on_session" + % auth_user + ) + slycat.web.server.create_single_sign_on_session( + slycat.web.server.check_https_get_remote_ip(), auth_user + ) raise cherrypy.HTTPRedirect("https://" + current_url.netloc + "/projects") @@ -1341,8 +1603,11 @@ def login(): response_url = slycat.web.server.response_url() # Get the client ip, which might be forwarded by a proxy. - remote_ip = cherrypy.request.headers.get( - "x-forwarded-for") if "x-forwarded-for" in cherrypy.request.headers else cherrypy.request.rem + remote_ip = ( + cherrypy.request.headers.get("x-forwarded-for") + if "x-forwarded-for" in cherrypy.request.headers + else cherrypy.request.rem + ) # see if we have a registered password checking function from our config password_check = slycat.web.server.get_password_function() @@ -1359,7 +1624,8 @@ def login(): # cherrypy.log.error("user %s at %s failed authentication" % (user_name, remote_ip)) cherrypy.response.status = "404 no auth found!!!" - return {'success': success, 'target': response_url} + return {"success": success, "target": response_url} + def logout(): """ @@ -1373,15 +1639,17 @@ def logout(): # expire the old cookies cherrypy.response.cookie["slycatauth"] = sid - cherrypy.response.cookie["slycatauth"]['expires'] = 0 + cherrypy.response.cookie["slycatauth"]["expires"] = 0 cherrypy.response.cookie["slycattimeout"] = "timeout" - cherrypy.response.cookie["slycattimeout"]['expires'] = 0 + cherrypy.response.cookie["slycattimeout"]["expires"] = 0 couchdb = slycat.web.server.database.couchdb.connect() session = couchdb.get("session", sid) if session is not None: - cherrypy.response.status = "200 session deleted." + str(couchdb.delete(session)) + cherrypy.response.status = "200 session deleted." + str( + couchdb.delete(session) + ) else: cherrypy.response.status = "400 Bad Request no session to delete." else: @@ -1389,58 +1657,63 @@ def logout(): except Exception as e: raise cherrypy.HTTPError("400 Bad Request") + @cherrypy.tools.json_out(on=True) def clean_project_data(): - """ - cleans out project data that is not being pointed at - by a parameter space model, and cleans up models that - are not parameter space but point to project data - """ - slycat.web.server.authentication.require_server_administrator() - couchdb = slycat.web.server.database.couchdb.connect() - # get a view list of all pd ids - for row in couchdb.view("slycat/project_datas"): - pd_doc = couchdb.get(type="project_data",id=row.id) - delete_pd = True - # go through model list in pd and start cleaning - for model_id in pd_doc['mid']: - model_doc = couchdb.get(type="model",id=model_id) - if model_doc["model-type"] == "parameter-image": - delete_pd = False - # clean up models that don't need pd - elif "project_data" in model_doc: - cherrypy.log.error("Removing PD from none parameter-space model") - del model_doc["project_data"] - couchdb.save(model_doc) - if delete_pd: - # delete the bad project data - couchdb.delete(pd_doc) - cherrypy.log.error("Deleting PD::: %s" % str(row.id)) - return {"status":"ok"} + """ + cleans out project data that is not being pointed at + by a parameter space model, and cleans up models that + are not parameter space but point to project data + """ + slycat.web.server.authentication.require_server_administrator() + couchdb = slycat.web.server.database.couchdb.connect() + # get a view list of all pd ids + for row in couchdb.view("slycat/project_datas"): + pd_doc = couchdb.get(type="project_data", id=row.id) + delete_pd = True + # go through model list in pd and start cleaning + for model_id in pd_doc["mid"]: + model_doc = couchdb.get(type="model", id=model_id) + if model_doc["model-type"] == "parameter-image": + delete_pd = False + # clean up models that don't need pd + elif "project_data" in model_doc: + cherrypy.log.error("Removing PD from none parameter-space model") + del model_doc["project_data"] + couchdb.save(model_doc) + if delete_pd: + # delete the bad project data + couchdb.delete(pd_doc) + cherrypy.log.error("Deleting PD::: %s" % str(row.id)) + return {"status": "ok"} + @cherrypy.tools.json_out(on=True) def clear_ssh_sessions(): - """ - clears out of the ssh session for the current user - """ - try: - if "slycatauth" in cherrypy.request.cookie: - sid = cherrypy.request.cookie["slycatauth"].value - couchdb = slycat.web.server.database.couchdb.connect() - session = couchdb.get("session", sid) - cherrypy.log.error("ssh and smb sessions cleared for user session: %s" % session) - cherrypy.response.status = "200" - if session is not None: - for ssh_session in session["sessions"]: - cherrypy.log.error("sessions %s" % ssh_session["sid"]) - slycat.web.server.remote.delete_session(ssh_session["sid"]) - session['sessions'] = [] - couchdb.save(session) - else: - cherrypy.response.status = "403 Forbidden" - except Exception: - raise cherrypy.HTTPError("400 Bad Request") - return {"status":"ok"} + """ + clears out of the ssh session for the current user + """ + try: + if "slycatauth" in cherrypy.request.cookie: + sid = cherrypy.request.cookie["slycatauth"].value + couchdb = slycat.web.server.database.couchdb.connect() + session = couchdb.get("session", sid) + cherrypy.log.error( + "ssh and smb sessions cleared for user session: %s" % session + ) + cherrypy.response.status = "200" + if session is not None: + for ssh_session in session["sessions"]: + cherrypy.log.error("sessions %s" % ssh_session["sid"]) + slycat.web.server.remote.delete_session(ssh_session["sid"]) + session["sessions"] = [] + couchdb.save(session) + else: + cherrypy.response.status = "403 Forbidden" + except Exception: + raise cherrypy.HTTPError("400 Bad Request") + return {"status": "ok"} + @cherrypy.tools.json_in(on=True) def put_model_inputs(mid): @@ -1453,12 +1726,15 @@ def put_model_inputs(mid): deep_copy = cherrypy.request.json.get("deep-copy", False) source = database.get("model", sid) if source["project"] != model["project"]: - cherrypy.log.error("slycat.web.server.handlers.py put_model_inputs", - "cherrypy.HTTPError 400 cannot duplicate a model from another project.") + cherrypy.log.error( + "slycat.web.server.handlers.py put_model_inputs", + "cherrypy.HTTPError 400 cannot duplicate a model from another project.", + ) raise cherrypy.HTTPError("400 Cannot duplicate a model from another project.") slycat.web.server.put_model_inputs(database, model, source, deep_copy) + @cherrypy.tools.json_in(on=True) def put_project_data_parameter(did, aid): database = slycat.web.server.database.couchdb.connect() @@ -1472,7 +1748,9 @@ def put_project_data_parameter(did, aid): require_input = require_boolean_json_parameter("input") with slycat.web.server.database.couchdb.db_lock: try: - slycat.web.server.put_project_data_parameter(database, project_data, aid, value, require_input) + slycat.web.server.put_project_data_parameter( + database, project_data, aid, value, require_input + ) except Exception: time.sleep(1) database = slycat.web.server.database.couchdb.connect() @@ -1482,7 +1760,9 @@ def put_project_data_parameter(did, aid): value = require_json_parameter("value") require_input = require_boolean_json_parameter("input") - slycat.web.server.put_project_data_parameter(database, project_data, aid, value, require_input) + slycat.web.server.put_project_data_parameter( + database, project_data, aid, value, require_input + ) @cherrypy.tools.json_in(on=True) @@ -1496,7 +1776,9 @@ def put_model_parameter(mid, aid): require_input = require_boolean_json_parameter("input") with slycat.web.server.database.couchdb.db_lock: try: - slycat.web.server.put_model_parameter(database, model, aid, value, require_input) + slycat.web.server.put_model_parameter( + database, model, aid, value, require_input + ) except Exception: time.sleep(1) database = slycat.web.server.database.couchdb.connect() @@ -1506,14 +1788,17 @@ def put_model_parameter(mid, aid): value = require_json_parameter("value") require_input = require_boolean_json_parameter("input") - slycat.web.server.put_model_parameter(database, model, aid, value, require_input) + slycat.web.server.put_model_parameter( + database, model, aid, value, require_input + ) + def delete_model_parameter(mid, aid): """ - delete a model artifact + delete a model artifact :param mid: model Id :param aid: artifact id - :return: + :return: """ database = slycat.web.server.database.couchdb.connect() model = database.get("model", mid) @@ -1549,7 +1834,9 @@ def put_model_arrayset_array(mid, aid, array): array_index = int(array) attributes = cherrypy.request.json["attributes"] dimensions = cherrypy.request.json["dimensions"] - slycat.web.server.put_model_array(database, model, aid, array_index, attributes, dimensions) + slycat.web.server.put_model_array( + database, model, aid, array_index, attributes, dimensions + ) def put_model_arrayset_data(mid, aid, hyperchunks, data, byteorder=None): @@ -1557,14 +1844,20 @@ def put_model_arrayset_data(mid, aid, hyperchunks, data, byteorder=None): try: hyperchunks = slycat.hyperchunks.parse(hyperchunks) except: - cherrypy.log.error("slycat.web.server.handlers.py put_model_arrayset_data", - "cherrypy.HTTPError 400 not a valid hyperchunks specification.") + cherrypy.log.error( + "slycat.web.server.handlers.py put_model_arrayset_data", + "cherrypy.HTTPError 400 not a valid hyperchunks specification.", + ) raise cherrypy.HTTPError("400 Not a valid hyperchunks specification.") if byteorder is not None and byteorder not in ["big", "little"]: - cherrypy.log.error("slycat.web.server.handlers.py put_model_arrayset_data", - "cherrypy.HTTPError 400 optional byteorder argument must be big or little.") - raise cherrypy.HTTPError("400 optional byteorder argument must be big or little.") + cherrypy.log.error( + "slycat.web.server.handlers.py put_model_arrayset_data", + "cherrypy.HTTPError 400 optional byteorder argument must be big or little.", + ) + raise cherrypy.HTTPError( + "400 optional byteorder argument must be big or little." + ) # Handle the request. database = slycat.web.server.database.couchdb.connect() @@ -1572,7 +1865,9 @@ def put_model_arrayset_data(mid, aid, hyperchunks, data, byteorder=None): project = database.get("project", model["project"]) slycat.web.server.authentication.require_project_writer(project) - slycat.web.server.update_model(database, model, message="Storing data to array set %s." % (aid)) + slycat.web.server.update_model( + database, model, message="Storing data to array set %s." % (aid) + ) if byteorder is None: data = json.load(data.file) @@ -1581,51 +1876,82 @@ def put_model_arrayset_data(mid, aid, hyperchunks, data, byteorder=None): with slycat.web.server.hdf5.lock: with slycat.web.server.hdf5.open(model["artifact:%s" % aid], "r+") as file: hdf5_arrayset = slycat.hdf5.ArraySet(file) - for array in slycat.hyperchunks.arrays(hyperchunks, hdf5_arrayset.array_count()): + for array in slycat.hyperchunks.arrays( + hyperchunks, hdf5_arrayset.array_count() + ): hdf5_array = hdf5_arrayset[array.index] for attribute in array.attributes(len(hdf5_array.attributes)): - if not isinstance(attribute.expression, slycat.hyperchunks.grammar.AttributeIndex): - raise cherrypy.HTTPError("400 Cannot assign data to computed attributes.") + if not isinstance( + attribute.expression, slycat.hyperchunks.grammar.AttributeIndex + ): + raise cherrypy.HTTPError( + "400 Cannot assign data to computed attributes." + ) for hyperslice in attribute.hyperslices(): - #cherrypy.log.error( + # cherrypy.log.error( # "Writing %s/%s/%s/%s" % (aid, array.index, attribute.expression.index, hyperslice)) # We have to convert our hyperslice into a shape with explicit extents so we can compute # how many bytes to extract from the input data. if hyperslice == (Ellipsis,): - data_shape = [dimension["end"] - dimension["begin"] for dimension in hdf5_array.dimensions] + data_shape = [ + dimension["end"] - dimension["begin"] + for dimension in hdf5_array.dimensions + ] else: data_shape = [] - for hyperslice_dimension, array_dimension in zip(hyperslice, hdf5_array.dimensions): + for hyperslice_dimension, array_dimension in zip( + hyperslice, hdf5_array.dimensions + ): if isinstance(hyperslice_dimension, numbers.Integral): data_shape.append(1) elif isinstance(hyperslice_dimension, type(Ellipsis)): - data_shape.append(array_dimension["end"] - array_dimension["begin"]) + data_shape.append( + array_dimension["end"] + - array_dimension["begin"] + ) elif isinstance(hyperslice_dimension, slice): # TODO: Handle step start, stop, step = hyperslice_dimension.indices( - array_dimension["end"] - array_dimension["begin"]) + array_dimension["end"] + - array_dimension["begin"] + ) data_shape.append(stop - start) else: - cherrypy.log.error("slycat.web.server.handlers.py put_model_arrayset_data", - "Unexpected hyperslice: %s" % hyperslice_dimension) - raise ValueError("Unexpected hyperslice: %s" % hyperslice_dimension) + cherrypy.log.error( + "slycat.web.server.handlers.py put_model_arrayset_data", + "Unexpected hyperslice: %s" + % hyperslice_dimension, + ) + raise ValueError( + "Unexpected hyperslice: %s" + % hyperslice_dimension + ) # Convert data to an array ... - data_type = slycat.hdf5.dtype(hdf5_array.attributes[attribute.expression.index]["type"]) + data_type = slycat.hdf5.dtype( + hdf5_array.attributes[attribute.expression.index]["type"] + ) data_size = numpy.prod(data_shape) if byteorder is None: - hyperslice_data = numpy.array(next(data_iterator), dtype=data_type).reshape(data_shape) + hyperslice_data = numpy.array( + next(data_iterator), dtype=data_type + ).reshape(data_shape) elif byteorder == sys.byteorder: - hyperslice_data = numpy.fromfile(data.file, dtype=data_type, count=data_size).reshape( - data_shape) + hyperslice_data = numpy.fromfile( + data.file, dtype=data_type, count=data_size + ).reshape(data_shape) else: - cherrypy.log.error("slycat.web.server.handlers.py put_model_arrayset_data", - "Not implemented error.") + cherrypy.log.error( + "slycat.web.server.handlers.py put_model_arrayset_data", + "Not implemented error.", + ) raise NotImplementedError() - hdf5_array.set_data(attribute.expression.index, hyperslice, hyperslice_data) + hdf5_array.set_data( + attribute.expression.index, hyperslice, hyperslice_data + ) def delete_project_data_in_model(did, mid): @@ -1651,6 +1977,7 @@ def delete_project_data_in_model(did, mid): cherrypy.response.status = "204 Project data removed from model." + def delete_model_in_project_data(mid, did): """ Removes a model id from project data. @@ -1674,6 +2001,7 @@ def delete_model_in_project_data(mid, did): cherrypy.response.status = "204 Model removed from project data." + def delete_model(mid): couchdb = slycat.web.server.database.couchdb.connect() try: @@ -1681,7 +2009,9 @@ def delete_model(mid): project = couchdb.get("project", model["project"]) slycat.web.server.authentication.require_project_writer(project) - for project_data in couchdb.scan("slycat/project_datas", startkey=model["project"], endkey=model["project"]): + for project_data in couchdb.scan( + "slycat/project_datas", startkey=model["project"], endkey=model["project"] + ): updated = False with slycat.web.server.get_project_data_lock(project_data["_id"]): for index, pd_mid in enumerate(project_data["mid"]): @@ -1693,7 +2023,7 @@ def delete_model(mid): couchdb.delete(model) except Exception as e: - cherrypy.log.error('exception raised in delete model: %s' % str(e)) + cherrypy.log.error("exception raised in delete model: %s" % str(e)) slycat.web.server.cleanup.arrays() cherrypy.response.status = "204 Model deleted." @@ -1711,18 +2041,18 @@ def delete_reference(rid): def get_project_cache_object(pid, key): """ - Retrieves an object from a project’s cache. - Cache objects are opaque binary blobs that - may contain arbitrary data, plus an explicit + Retrieves an object from a project’s cache. + Cache objects are opaque binary blobs that + may contain arbitrary data, plus an explicit content type. - + Arguments: pid {string} -- project id key {string} -- string key for obj - + Raises: cherrypy.HTTPError: 404 not found - + Returns: attachment from database """ @@ -1731,8 +2061,12 @@ def get_project_cache_object(pid, key): slycat.web.server.authentication.require_project_reader(project) lookup = pid + "-" + key - for cache_object in database.scan("slycat/project-key-cache-objects", startkey=lookup, endkey=lookup): - cherrypy.response.headers["content-type"] = cache_object["_attachments"]["content"]["content_type"] + for cache_object in database.scan( + "slycat/project-key-cache-objects", startkey=lookup, endkey=lookup + ): + cherrypy.response.headers["content-type"] = cache_object["_attachments"][ + "content" + ]["content_type"] return database.get_attachment(cache_object, "content") raise cherrypy.HTTPError(404) @@ -1741,11 +2075,11 @@ def get_project_cache_object(pid, key): def delete_project_cache_object(pid, key): """ Deletes an object from the project cache. - + Arguments: pid {string} -- project id key {string} -- key in the cache - + Raises: cherrypy.HTTPError: 404 not found """ @@ -1754,12 +2088,17 @@ def delete_project_cache_object(pid, key): slycat.web.server.authentication.require_project_writer(project) lookup = pid + "-" + key - for cache_object in couchdb.scan("slycat/project-key-cache-objects", startkey=lookup, endkey=lookup): + for cache_object in couchdb.scan( + "slycat/project-key-cache-objects", startkey=lookup, endkey=lookup + ): couchdb.delete(cache_object) cherrypy.response.status = "204 Cache object deleted." return - cherrypy.log.error("slycat.web.server.handlers.py delete_project_cache_object", "cherrypy.HTTPError 404") + cherrypy.log.error( + "slycat.web.server.handlers.py delete_project_cache_object", + "cherrypy.HTTPError 404", + ) raise cherrypy.HTTPError(404) @@ -1774,7 +2113,9 @@ def delete_project_cache(pid): project = couchdb.get("project", pid) slycat.web.server.authentication.require_project_administrator(project) - for cache_object in couchdb.scan("slycat/project-cache-objects", startkey=pid, endkey=pid): + for cache_object in couchdb.scan( + "slycat/project-cache-objects", startkey=pid, endkey=pid + ): couchdb.delete(cache_object) cherrypy.response.status = "204 Cache objects deleted." @@ -1784,7 +2125,9 @@ def get_model_array_attribute_chunk(mid, aid, array, attribute, **arguments): try: attribute = int(attribute) except: - raise cherrypy.HTTPError("400 Malformed attribute argument must be a zero-based integer attribute index.") + raise cherrypy.HTTPError( + "400 Malformed attribute argument must be a zero-based integer attribute index." + ) try: ranges = [int(spec) for spec in arguments["ranges"].split(",")] @@ -1792,12 +2135,15 @@ def get_model_array_attribute_chunk(mid, aid, array, attribute, **arguments): ranges = list(zip(i, i)) except: raise cherrypy.HTTPError( - "400 Malformed ranges argument must be a comma separated collection of half-open index ranges.") + "400 Malformed ranges argument must be a comma separated collection of half-open index ranges." + ) byteorder = arguments.get("byteorder", None) if byteorder is not None: if byteorder not in ["little", "big"]: - raise cherrypy.HTTPError("400 Malformed byteorder argument must be 'little' or 'big'.") + raise cherrypy.HTTPError( + "400 Malformed byteorder argument must be 'little' or 'big'." + ) accept = cherrypy.lib.cptools.accept(["application/octet-stream"]) else: accept = cherrypy.lib.cptools.accept(["application/json"]) @@ -1823,10 +2169,14 @@ def get_model_array_attribute_chunk(mid, aid, array, attribute, **arguments): if not (0 <= attribute and attribute < len(hdf5_array.attributes)): raise cherrypy.HTTPError("400 Attribute argument out-of-range.") if len(ranges) != hdf5_array.ndim: - raise cherrypy.HTTPError("400 Ranges argument doesn't contain the correct number of dimensions.") - - ranges = [(max(dimension["begin"], range[0]), min(dimension["end"], range[1])) for dimension, range in - zip(hdf5_array.dimensions, ranges)] + raise cherrypy.HTTPError( + "400 Ranges argument doesn't contain the correct number of dimensions." + ) + + ranges = [ + (max(dimension["begin"], range[0]), min(dimension["end"], range[1])) + for dimension, range in zip(hdf5_array.dimensions, ranges) + ] index = tuple([slice(begin, end) for begin, end in ranges]) attribute_type = hdf5_array.attributes[attribute]["type"] @@ -1859,20 +2209,23 @@ def get_model_arrayset_metadata(mid, aid, **kwargs): them may be an extremely expensive operation, since they involve full scans through their respective attributes. Because of this, callers are encouraged to retrieve statistics only when needed. - + Arguments: mid {string} -- model id aid {string} -- artifact id - + Raises: cherrypy.HTTPError: 404 cherrypy.HTTPError: 400 aid is not an array artifact. cherrypy.HTTPError: 400 Not a valid hyperchunks specification. - + Returns: json -- statistical results of arrayset """ - cherrypy.log.error("GET arrayset metadata mid:%s aid:%s kwargs:%s" % (mid, aid, list(kwargs.keys()))) + cherrypy.log.error( + "GET arrayset metadata mid:%s aid:%s kwargs:%s" + % (mid, aid, list(kwargs.keys())) + ) database = slycat.web.server.database.couchdb.connect() model = database.get("model", mid) project = database.get("project", model["project"]) @@ -1880,90 +2233,126 @@ def get_model_arrayset_metadata(mid, aid, **kwargs): artifact = model.get("artifact:%s" % aid, None) if artifact is None: - cherrypy.log.error("slycat.web.server.handlers.py get_model_arrayset_metadata", - "cherrypy.HTTPError 404 artifact is None for aid: %s" % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_arrayset_metadata", + "cherrypy.HTTPError 404 artifact is None for aid: %s" % aid, + ) raise cherrypy.HTTPError(404) artifact_type = model["artifact-types"][aid] if artifact_type not in ["hdf5"]: - cherrypy.log.error("slycat.web.server.handlers.py get_model_arrayset_metadata", - "cherrypy.HTTPError 400 %s is not an array artifact." % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_arrayset_metadata", + "cherrypy.HTTPError 400 %s is not an array artifact." % aid, + ) raise cherrypy.HTTPError("400 %s is not an array artifact." % aid) try: - arrays = slycat.hyperchunks.parse(kwargs["arrays"]) if "arrays" in kwargs else None + arrays = ( + slycat.hyperchunks.parse(kwargs["arrays"]) if "arrays" in kwargs else None + ) except: - cherrypy.log.error("slycat.web.server.handlers.py get_model_arrayset_metadata", - "cherrypy.HTTPError 400 not a valid hyperchunks specification.") + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_arrayset_metadata", + "cherrypy.HTTPError 400 not a valid hyperchunks specification.", + ) raise cherrypy.HTTPError("400 Not a valid hyperchunks specification.") try: - statistics = slycat.hyperchunks.parse(kwargs["statistics"]) if "statistics" in kwargs else None + statistics = ( + slycat.hyperchunks.parse(kwargs["statistics"]) + if "statistics" in kwargs + else None + ) except: - cherrypy.log.error("slycat.web.server.handlers.py get_model_arrayset_metadata", - "cherrypy.HTTPError 400 not a valid hyperchunks specification.") + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_arrayset_metadata", + "cherrypy.HTTPError 400 not a valid hyperchunks specification.", + ) raise cherrypy.HTTPError("400 Not a valid hyperchunks specification.") try: - unique = slycat.hyperchunks.parse(kwargs["unique"]) if "unique" in kwargs else None + unique = ( + slycat.hyperchunks.parse(kwargs["unique"]) if "unique" in kwargs else None + ) except: - cherrypy.log.error("slycat.web.server.handlers.py get_model_arrayset_metadata", - "cherrypy.HTTPError 400 not a valid hyperchunks specification.") + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_arrayset_metadata", + "cherrypy.HTTPError 400 not a valid hyperchunks specification.", + ) raise cherrypy.HTTPError("400 Not a valid hyperchunks specification.") - cherrypy.log.error("GET arrayset metadata arrays:%s stats:%s unique:%s" % (arrays, statistics, unique)) - results = slycat.web.server.get_model_arrayset_metadata(database, model, aid, arrays, statistics, unique) - #cherrypy.log.error("GOT RESULTS") + cherrypy.log.error( + "GET arrayset metadata arrays:%s stats:%s unique:%s" + % (arrays, statistics, unique) + ) + results = slycat.web.server.get_model_arrayset_metadata( + database, model, aid, arrays, statistics, unique + ) + # cherrypy.log.error("GOT RESULTS") if "unique" in results: for unique in results["unique"]: # Maybe due to caching, sometimes the result comes back as a 'list' - if isinstance(results["unique"][0]['values'][0], list): - unique["values"] = [list_of_values for list_of_values in unique["values"]] + if isinstance(results["unique"][0]["values"][0], list): + unique["values"] = [ + list_of_values for list_of_values in unique["values"] + ] # Other times it's a 'numpy.ndarray' else: - unique["values"] = [array.tolist() for array in unique["values"]] + unique["values"] = [array.tolist() for array in unique["values"]] # have to load and dump to clean out all the non json serializable numpy arrays return json.loads(json.dumps(results, cls=MyEncoder)) def get_model_arrayset_data(mid, aid, hyperchunks, byteorder=None): """ - Retrieve data stored in arrayset darray attributes. The caller - may request data stored using any combination of + Retrieve data stored in arrayset darray attributes. The caller + may request data stored using any combination of arrays, attributes, and hyperslices. - + Arguments: mid {string} -- model id aid {string} -- artifact id - hyperchunks {string} -- The request must contain a parameter - hyperchunks that specifies the arrays, attributes, + hyperchunks {string} -- The request must contain a parameter + hyperchunks that specifies the arrays, attributes, and hyperslices to be returned, in Hyperchunks format. - + Keyword Arguments: - byteorder {string} -- optional byteorder argument must + byteorder {string} -- optional byteorder argument must be big or little. (default: {None}) - + Raises: cherrypy.HTTPError: 400 aid is not an array artifact. cherrypy.HTTPError: 400 Not a valid hyperchunks specification cherrypy.HTTPError: 400 optional byteorder argument must be big or little. cherrypy.HTTPError: 404 aid not found - + Returns: octet-stream -- application/octet-stream """ cherrypy.log.error( - "GET Model Arrayset Data: arrayset %s hyperchunks %s byteorder %s" % (aid, hyperchunks, byteorder)) + "GET Model Arrayset Data: arrayset %s hyperchunks %s byteorder %s" + % (aid, hyperchunks, byteorder) + ) try: hyperchunks = slycat.hyperchunks.parse(hyperchunks) except Exception as e: - cherrypy.log.error("slycat.web.server.handlers.py get_model_arrayset_data", - "cherrypy.HTTPError 400 not a valid hyperchunks specification.") - raise cherrypy.HTTPError("400 Not a valid hyperchunks specification.%s%s" % (e, traceback.print_exc())) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_arrayset_data", + "cherrypy.HTTPError 400 not a valid hyperchunks specification.", + ) + raise cherrypy.HTTPError( + "400 Not a valid hyperchunks specification.%s%s" + % (e, traceback.print_exc()) + ) if byteorder is not None: if byteorder not in ["big", "little"]: - cherrypy.log.error("slycat.web.server.handlers.py get_model_arrayset_data", - "cherrypy.HTTPError 400 optional byteorder argument must be big or little.") - raise cherrypy.HTTPError("400 optional byteorder argument must be big or little.") + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_arrayset_data", + "cherrypy.HTTPError 400 optional byteorder argument must be big or little.", + ) + raise cherrypy.HTTPError( + "400 optional byteorder argument must be big or little." + ) accept = cherrypy.lib.cptools.accept(["application/octet-stream"]) else: accept = cherrypy.lib.cptools.accept(["application/json"]) @@ -1976,12 +2365,17 @@ def get_model_arrayset_data(mid, aid, hyperchunks, byteorder=None): artifact = model.get("artifact:%s" % aid, None) if artifact is None: - cherrypy.log.error("slycat.web.server.handlers.py get_model_arrayset_data", "cherrypy.HTTPError 404") + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_arrayset_data", + "cherrypy.HTTPError 404", + ) raise cherrypy.HTTPError(404) artifact_type = model["artifact-types"][aid] if artifact_type not in ["hdf5"]: - cherrypy.log.error("slycat.web.server.handlers.py get_model_arrayset_data", - "cherrypy.HTTPError 400 %s is not an array artifact." % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_arrayset_data", + "cherrypy.HTTPError 400 %s is not an array artifact." % aid, + ) raise cherrypy.HTTPError("400 %s is not an array artifact." % aid) def mask_nans(array): @@ -1990,12 +2384,21 @@ def mask_nans(array): return numpy.ma.masked_where(numpy.isnan(array), array) except: return array + def content(): if byteorder is None: - yield json.dumps([mask_nans(hyperslice).tolist() for hyperslice in - slycat.web.server.get_model_arrayset_data(database, model, aid, hyperchunks)]).encode() + yield json.dumps( + [ + mask_nans(hyperslice).tolist() + for hyperslice in slycat.web.server.get_model_arrayset_data( + database, model, aid, hyperchunks + ) + ] + ).encode() else: - for hyperslice in slycat.web.server.get_model_arrayset_data(database, model, aid, hyperchunks): + for hyperslice in slycat.web.server.get_model_arrayset_data( + database, model, aid, hyperchunks + ): if sys.byteorder != byteorder: yield hyperslice.byteswap().tostring(order="C") else: @@ -2010,14 +2413,14 @@ def content(): @cherrypy.tools.json_in(on=True) def post_model_arrayset_data(mid, aid): """ - get the arrayset data based on aid, mid, byteorder, and hyperchunks + get the arrayset data based on aid, mid, byteorder, and hyperchunks - requires hyperchunks to be included in the json payload + requires hyperchunks to be included in the json payload - :param mid: model id - :param aid: artifact id - :return: stream of data - """ + :param mid: model id + :param aid: artifact id + :return: stream of data + """ # try and grab hyperchunk hyperchunks = None byteorder = None @@ -2027,25 +2430,35 @@ def post_model_arrayset_data(mid, aid): byteorder = cherrypy.request.json["byteorder"] except Exception as e: byteorder = None - #cherrypy.log.error("no byteorder provided moving on") + # cherrypy.log.error("no byteorder provided moving on") # parse the hyperchunks cherrypy.log.error( - "GET Model Arrayset Data: arrayset %s hyperchunks %s byteorder %s" % (aid, hyperchunks, byteorder)) + "GET Model Arrayset Data: arrayset %s hyperchunks %s byteorder %s" + % (aid, hyperchunks, byteorder) + ) try: hyperchunks = slycat.hyperchunks.parse(hyperchunks) except: - cherrypy.log.error("slycat.web.server.handlers.py get_model_arrayset_data", - "cherrypy.HTTPError 400 not a valid hyperchunks specification.") + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_arrayset_data", + "cherrypy.HTTPError 400 not a valid hyperchunks specification.", + ) raise cherrypy.HTTPError("400 Not a valid hyperchunks specification.") - cherrypy.log.error("GET Model Arrayset Data: arrayset %s hyperchunks calculated" % (aid)) + cherrypy.log.error( + "GET Model Arrayset Data: arrayset %s hyperchunks calculated" % (aid) + ) # if byteorder is not None: if byteorder not in ["big", "little"]: - cherrypy.log.error("slycat.web.server.handlers.py get_model_arrayset_data", - "cherrypy.HTTPError 400 optional byteorder argument must be big or little.") - raise cherrypy.HTTPError("400 optional byteorder argument must be big or little.") + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_arrayset_data", + "cherrypy.HTTPError 400 optional byteorder argument must be big or little.", + ) + raise cherrypy.HTTPError( + "400 optional byteorder argument must be big or little." + ) accept = cherrypy.lib.cptools.accept(["application/octet-stream"]) else: accept = cherrypy.lib.cptools.accept(["application/json"]) @@ -2058,12 +2471,17 @@ def post_model_arrayset_data(mid, aid): artifact = model.get("artifact:%s" % aid, None) if artifact is None: - cherrypy.log.error("slycat.web.server.handlers.py get_model_arrayset_data", "cherrypy.HTTPError 404") + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_arrayset_data", + "cherrypy.HTTPError 404", + ) raise cherrypy.HTTPError(404) artifact_type = model["artifact-types"][aid] if artifact_type not in ["hdf5"]: - cherrypy.log.error("slycat.web.server.handlers.py get_model_arrayset_data", - "cherrypy.HTTPError 400 %s is not an array artifact." % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_arrayset_data", + "cherrypy.HTTPError 400 %s is not an array artifact." % aid, + ) raise cherrypy.HTTPError("400 %s is not an array artifact." % aid) def mask_nans(array): @@ -2077,37 +2495,56 @@ def content(): if "include_nans" in cherrypy.request.json: include_nans = cherrypy.request.json["include_nans"] if byteorder is None: - yield json.dumps([mask_nans(hyperslice).tolist() for hyperslice in - slycat.web.server.get_model_arrayset_data(database, model, aid, hyperchunks)]).encode() + yield json.dumps( + [ + mask_nans(hyperslice).tolist() + for hyperslice in slycat.web.server.get_model_arrayset_data( + database, model, aid, hyperchunks + ) + ] + ).encode() else: - for hyperslice in slycat.web.server.get_model_arrayset_data(database, model, aid, hyperchunks): + for hyperslice in slycat.web.server.get_model_arrayset_data( + database, model, aid, hyperchunks + ): if sys.byteorder != byteorder: yield hyperslice.byteswap().tostring(order="C") else: yield hyperslice.tostring(order="C") - cherrypy.log.error("GET Model Arrayset Data: arrayset %s starting to get content" % (aid)) + cherrypy.log.error( + "GET Model Arrayset Data: arrayset %s starting to get content" % (aid) + ) return content() # TODO: streaming was turned off because this is now a post, in the future we may wan to turn this back on to increase speed # post_model_arrayset_data._cp_config = {"response.stream" : True} + def validate_table_rows(rows): try: rows = [spec.split("-") for spec in rows.split(",")] - rows = [(int(spec[0]), int(spec[1]) if len(spec) == 2 else int(spec[0]) + 1) for spec in rows] + rows = [ + (int(spec[0]), int(spec[1]) if len(spec) == 2 else int(spec[0]) + 1) + for spec in rows + ] rows = numpy.concatenate([numpy.arange(begin, end) for begin, end in rows]) return rows except: - cherrypy.log.error("slycat.web.server.handlers.py validate_table_rows", - "cherrypy.HTTPError 400 malformed rows argument must be a comma separated collection of row indices or half-open index ranges.") + cherrypy.log.error( + "slycat.web.server.handlers.py validate_table_rows", + "cherrypy.HTTPError 400 malformed rows argument must be a comma separated collection of row indices or half-open index ranges.", + ) raise cherrypy.HTTPError( - "400 Malformed rows argument must be a comma separated collection of row indices or half-open index ranges.") + "400 Malformed rows argument must be a comma separated collection of row indices or half-open index ranges." + ) if numpy.any(rows < 0): - cherrypy.log.error("slycat.web.server.handlers.py validate_table_rows", - "cherrypy.HTTPError 400 row values must be non-negative.") + cherrypy.log.error( + "slycat.web.server.handlers.py validate_table_rows", + "cherrypy.HTTPError 400 row values must be non-negative.", + ) raise cherrypy.HTTPError("400 Row values must be non-negative.") return rows @@ -2116,18 +2553,28 @@ def validate_table_rows(rows): def validate_table_columns(columns): try: columns = [spec.split("-") for spec in columns.split(",")] - columns = [(int(spec[0]), int(spec[1]) if len(spec) == 2 else int(spec[0]) + 1) for spec in columns] - columns = numpy.concatenate([numpy.arange(begin, end) for begin, end in columns]) + columns = [ + (int(spec[0]), int(spec[1]) if len(spec) == 2 else int(spec[0]) + 1) + for spec in columns + ] + columns = numpy.concatenate( + [numpy.arange(begin, end) for begin, end in columns] + ) columns = columns[columns >= 0] except Exception as e: cherrypy.log.error(str(e)) - cherrypy.log.error("slycat.web.server.handlers.py validate_table_columns", - "cherrypy.HTTPError 400 malformed columns argument must be a comma separated collection of column indices or half-open index ranges.") + cherrypy.log.error( + "slycat.web.server.handlers.py validate_table_columns", + "cherrypy.HTTPError 400 malformed columns argument must be a comma separated collection of column indices or half-open index ranges.", + ) raise cherrypy.HTTPError( - "400 Malformed columns argument must be a comma separated collection of column indices or half-open index ranges.") + "400 Malformed columns argument must be a comma separated collection of column indices or half-open index ranges." + ) if numpy.any(columns < 0): - cherrypy.log.error("slycat.web.server.handlers.py validate_table_columns", - "cherrypy.HTTPError 400 column values must be non-negative.") + cherrypy.log.error( + "slycat.web.server.handlers.py validate_table_columns", + "cherrypy.HTTPError 400 column values must be non-negative.", + ) raise cherrypy.HTTPError("400 Column values must be non-negative.") return columns @@ -2138,31 +2585,44 @@ def validate_table_sort(sort): sort = [spec.split(":") for spec in sort.split(",")] sort = [(column, order) for column, order in sort] except: - cherrypy.log.error("slycat.web.server.handlers.py validate_table_sort", - "cherrypy.HTTPError 400 malformed order argument must be a comma separated collection of column:order tuples.") + cherrypy.log.error( + "slycat.web.server.handlers.py validate_table_sort", + "cherrypy.HTTPError 400 malformed order argument must be a comma separated collection of column:order tuples.", + ) raise cherrypy.HTTPError( - "400 Malformed order argument must be a comma separated collection of column:order tuples.") + "400 Malformed order argument must be a comma separated collection of column:order tuples." + ) try: sort = [(int(column), order) for column, order in sort] except: - cherrypy.log.error("slycat.web.server.handlers.py validate_table_sort", - "cherrypy.HTTPError 400 sort column must be an integer.") + cherrypy.log.error( + "slycat.web.server.handlers.py validate_table_sort", + "cherrypy.HTTPError 400 sort column must be an integer.", + ) raise cherrypy.HTTPError("400 Sort column must be an integer.") for column, order in sort: if column < 0: - cherrypy.log.error("slycat.web.server.handlers.py validate_table_sort", - "cherrypy.HTTPError 400 sort column must be non-negative.") + cherrypy.log.error( + "slycat.web.server.handlers.py validate_table_sort", + "cherrypy.HTTPError 400 sort column must be non-negative.", + ) raise cherrypy.HTTPError("400 Sort column must be non-negative.") if order not in ["ascending", "descending"]: - cherrypy.log.error("slycat.web.server.handlers.py validate_table_sort", - "cherrypy.HTTPError 400 sort-order must be 'ascending' or 'descending'") - raise cherrypy.HTTPError("400 Sort-order must be 'ascending' or 'descending'.") + cherrypy.log.error( + "slycat.web.server.handlers.py validate_table_sort", + "cherrypy.HTTPError 400 sort-order must be 'ascending' or 'descending'", + ) + raise cherrypy.HTTPError( + "400 Sort-order must be 'ascending' or 'descending'." + ) if len(sort) != 1: - cherrypy.log.error("slycat.web.server.handlers.py validate_table_sort", - "cherrypy.HTTPError 400 currently, only one column can be sorted.") + cherrypy.log.error( + "slycat.web.server.handlers.py validate_table_sort", + "cherrypy.HTTPError 400 currently, only one column can be sorted.", + ) raise cherrypy.HTTPError("400 Currently, only one column can be sorted.") return sort @@ -2171,9 +2631,13 @@ def validate_table_sort(sort): def validate_table_byteorder(byteorder): if byteorder is not None: if byteorder not in ["little", "big"]: - cherrypy.log.error("slycat.web.server.handlers.py validate_table_sort", - "cherrypy.HTTPError 400 malformed byteorder argument must be 'little' or 'big'.") - raise cherrypy.HTTPError("400 Malformed byteorder argument must be 'little' or 'big'.") + cherrypy.log.error( + "slycat.web.server.handlers.py validate_table_sort", + "cherrypy.HTTPError 400 malformed byteorder argument must be 'little' or 'big'.", + ) + raise cherrypy.HTTPError( + "400 Malformed byteorder argument must be 'little' or 'big'." + ) accept = cherrypy.lib.cptools.accept(["application/octet-stream"]) else: accept = cherrypy.lib.cptools.accept(["application/json"]) @@ -2191,8 +2655,10 @@ def get_table_sort_index(file, metadata, array_index, sort, index): index_key = "array/%s/index/%s" % (array_index, sort_column) if index_key not in file: # cherrypy.log.error("Caching array index for file %s array %s attribute %s" % (file.filename, array_index, sort_column)) - sort_index = numpy.argsort(slycat.hdf5.ArraySet(file)[array_index].get_data(sort_column)[...], - kind="mergesort") + sort_index = numpy.argsort( + slycat.hdf5.ArraySet(file)[array_index].get_data(sort_column)[...], + kind="mergesort", + ) file[index_key] = sort_index else: # cherrypy.log.error("Loading cached sort index.") @@ -2208,13 +2674,17 @@ def get_table_metadata(file, array_index, index): array = arrayset[array_index] if array.ndim != 1: - cherrypy.log.error("slycat.web.server.handlers.py get_table_metadata", - "cherrypy.HTTPError 400 not a table (1D array) artifact.") + cherrypy.log.error( + "slycat.web.server.handlers.py get_table_metadata", + "cherrypy.HTTPError 400 not a table (1D array) artifact.", + ) raise cherrypy.HTTPError("400 Not a table (1D array) artifact.") dimensions = array.dimensions attributes = array.attributes - statistics = [array.get_statistics(attribute) for attribute in range(len(attributes))] + statistics = [ + array.get_statistics(attribute) for attribute in range(len(attributes)) + ] metadata = { "row-count": dimensions[0]["end"] - dimensions[0]["begin"], @@ -2222,7 +2692,7 @@ def get_table_metadata(file, array_index, index): "column-names": [attribute["name"] for attribute in attributes], "column-types": [attribute["type"] for attribute in attributes], "column-min": [attribute["min"] for attribute in statistics], - "column-max": [attribute["max"] for attribute in statistics] + "column-max": [attribute["max"] for attribute in statistics], } if index is not None: @@ -2244,25 +2714,32 @@ def get_model_table_metadata(mid, aid, array, index=None): artifact = model.get("artifact:%s" % aid, None) if artifact is None: - cherrypy.log.error("slycat.web.server.handlers.py get_model_table_metadata", - "cherrypy.HTTPError 404 artifact is None for aid: %s" % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_table_metadata", + "cherrypy.HTTPError 404 artifact is None for aid: %s" % aid, + ) raise cherrypy.HTTPError(404) artifact_type = model["artifact-types"][aid] if artifact_type not in ["hdf5"]: - cherrypy.log.error("slycat.web.server.handlers.py get_model_table_metadata", - "cherrypy.HTTPError 400 %s is not an array artifact." % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_table_metadata", + "cherrypy.HTTPError 400 %s is not an array artifact." % aid, + ) raise cherrypy.HTTPError("400 %s is not an array artifact." % aid) with slycat.web.server.hdf5.lock: - with slycat.web.server.hdf5.open(artifact, - "r+") as file: # We have to open the file with writing enabled because the statistics cache may need to be updated. + with slycat.web.server.hdf5.open( + artifact, "r+" + ) as file: # We have to open the file with writing enabled because the statistics cache may need to be updated. metadata = get_table_metadata(file, array, index) # have to load and dump to clean out all the non json serializable numpy arrays return json.loads(json.dumps(metadata, cls=MyEncoder)) @cherrypy.tools.json_out(on=True) -def get_model_table_chunk(mid, aid, array, rows=None, columns=None, index=None, sort=None): +def get_model_table_chunk( + mid, aid, array, rows=None, columns=None, index=None, sort=None +): rows = validate_table_rows(rows) columns = validate_table_columns(columns) sort = validate_table_sort(sort) @@ -2274,13 +2751,17 @@ def get_model_table_chunk(mid, aid, array, rows=None, columns=None, index=None, artifact = model.get("artifact:%s" % aid, None) if artifact is None: - cherrypy.log.error("slycat.web.server.handlers.py get_model_table_chunk", - "cherrypy.HTTPError 404 artifact is None for aid: %s" % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_table_chunk", + "cherrypy.HTTPError 404 artifact is None for aid: %s" % aid, + ) raise cherrypy.HTTPError(404) artifact_type = model["artifact-types"][aid] if artifact_type not in ["hdf5"]: - cherrypy.log.error("slycat.web.server.handlers.py get_model_table_chunk", - "cherrypy.HTTPError 400 %s is not an array artifact." % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_table_chunk", + "cherrypy.HTTPError 400 %s is not an array artifact." % aid, + ) raise cherrypy.HTTPError("400 %s is not an array artifact." % aid) with slycat.web.server.hdf5.lock: @@ -2290,14 +2771,18 @@ def get_model_table_chunk(mid, aid, array, rows=None, columns=None, index=None, # Constrain end <= count along both dimensions rows = rows[rows < metadata["row-count"]] if numpy.any(columns >= metadata["column-count"]): - cherrypy.log.error("slycat.web.server.handlers.py get_model_table_chunk", - "cherrypy.HTTPError 400 column out-of-range.") + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_table_chunk", + "cherrypy.HTTPError 400 column out-of-range.", + ) raise cherrypy.HTTPError("400 Column out-of-range.") if sort is not None: for column, order in sort: if column >= metadata["column-count"]: - cherrypy.log.error("slycat.web.server.handlers.py get_model_table_chunk", - "400 sort column out-of-range.") + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_table_chunk", + "400 sort column out-of-range.", + ) raise cherrypy.HTTPError("400 Sort column out-of-range.") # Retrieve the data @@ -2311,24 +2796,35 @@ def get_model_table_chunk(mid, aid, array, rows=None, columns=None, index=None, if index is not None and column == metadata["column-count"] - 1: values = sort_slice.tolist() else: - values = slycat.hdf5.ArraySet(file)[array].get_data(column)[sort_slice[slice_index].tolist()][ - slice_reverse_index].tolist() + values = ( + slycat.hdf5.ArraySet(file)[array] + .get_data(column)[sort_slice[slice_index].tolist()][ + slice_reverse_index + ] + .tolist() + ) if meta_type in ["float32", "float64"]: - values = [None if numpy.isnan(value) else value for value in values] + values = [ + None if numpy.isnan(value) else value for value in values + ] data.append(values) result = { "rows": rows.tolist(), "columns": columns.tolist(), - "column-names": [metadata["column-names"][column] for column in columns], + "column-names": [ + metadata["column-names"][column] for column in columns + ], "data": data, - "sort": sort + "sort": sort, } return json.loads(json.dumps(result, cls=MyEncoder)) -def get_model_table_sorted_indices(mid, aid, array, rows=None, index=None, sort=None, byteorder=None): +def get_model_table_sorted_indices( + mid, aid, array, rows=None, index=None, sort=None, byteorder=None +): rows = validate_table_rows(rows) sort = validate_table_sort(sort) byteorder = validate_table_byteorder(byteorder) @@ -2340,13 +2836,17 @@ def get_model_table_sorted_indices(mid, aid, array, rows=None, index=None, sort= artifact = model.get("artifact:%s" % aid, None) if artifact is None: - cherrypy.log.error("slycat.web.server.handlers.py get_model_table_sorted_indices", - "cherrypy.HTTPError 404 artifact is None for aid: %s" % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_table_sorted_indices", + "cherrypy.HTTPError 404 artifact is None for aid: %s" % aid, + ) raise cherrypy.HTTPError(404) artifact_type = model["artifact-types"][aid] if artifact_type not in ["hdf5"]: - cherrypy.log.error("slycat.web.server.handlers.py get_model_table_sorted_indices", - "cherrypy.HTTPError 400 %s is not an array artifact." % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_table_sorted_indices", + "cherrypy.HTTPError 400 %s is not an array artifact." % aid, + ) raise cherrypy.HTTPError("400 %s is not an array artifact." % aid) with slycat.web.server.hdf5.lock: @@ -2358,13 +2858,17 @@ def get_model_table_sorted_indices(mid, aid, array, rows=None, index=None, sort= if sort is not None: for column, order in sort: if column >= metadata["column-count"]: - cherrypy.log.error("slycat.web.server.handlers.py get_model_table_sorted_indices", - "cherrypy.HTTPError 400 sort column out-of-range.") + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_table_sorted_indices", + "cherrypy.HTTPError 400 sort column out-of-range.", + ) raise cherrypy.HTTPError("400 Sort column out-of-range.") # Retrieve the data ... sort_index = get_table_sort_index(file, metadata, array, sort, index) - slice_val = numpy.argsort(sort_index, kind="mergesort")[rows].astype("int32") + slice_val = numpy.argsort(sort_index, kind="mergesort")[rows].astype( + "int32" + ) if byteorder is None: return json.dumps(slice_val.tolist()) @@ -2375,7 +2879,9 @@ def get_model_table_sorted_indices(mid, aid, array, rows=None, index=None, sort= return slice_val.tostring(order="C") -def get_model_table_unsorted_indices(mid, aid, array, rows=None, index=None, sort=None, byteorder=None): +def get_model_table_unsorted_indices( + mid, aid, array, rows=None, index=None, sort=None, byteorder=None +): rows = validate_table_rows(rows) sort = validate_table_sort(sort) byteorder = validate_table_byteorder(byteorder) @@ -2387,13 +2893,17 @@ def get_model_table_unsorted_indices(mid, aid, array, rows=None, index=None, sor artifact = model.get("artifact:%s" % aid, None) if artifact is None: - cherrypy.log.error("slycat.web.server.handlers.py get_model_table_unsorted_indices", - "cherrypy.HTTPError 404 artifact is None for aid: %s" % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_table_unsorted_indices", + "cherrypy.HTTPError 404 artifact is None for aid: %s" % aid, + ) raise cherrypy.HTTPError(404) artifact_type = model["artifact-types"][aid] if artifact_type not in ["hdf5"]: - cherrypy.log.error("slycat.web.server.handlers.py get_model_table_unsorted_indices", - "cherrypy.HTTPError 400 %s is not an array artifact." % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_table_unsorted_indices", + "cherrypy.HTTPError 400 %s is not an array artifact." % aid, + ) raise cherrypy.HTTPError("400 %s is not an array artifact." % aid) with slycat.web.server.hdf5.lock: @@ -2405,8 +2915,10 @@ def get_model_table_unsorted_indices(mid, aid, array, rows=None, index=None, sor if sort is not None: for column, order in sort: if column >= metadata["column-count"]: - cherrypy.log.error("slycat.web.server.handlers.py get_model_table_unsorted_indices", - "cherrypy.HTTPError 400 sort column out-of-range.") + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_table_unsorted_indices", + "cherrypy.HTTPError 400 sort column out-of-range.", + ) raise cherrypy.HTTPError("400 Sort column out-of-range.") # Generate a database query @@ -2423,19 +2935,19 @@ def get_model_table_unsorted_indices(mid, aid, array, rows=None, index=None, sor def get_model_file(mid, aid): - """ + """ Retrieves a file artifact from a model. File artifacts are effectively binary blobs that may contain arbitrary data with an explicit content type. - + Arguments: mid {string} -- model id aid {string} -- artifact id - + Raises: cherrypy.HTTPError: 404 cherrypy.HTTPError: 400 aid is not a file artifact. - + Returns: any -- file """ @@ -2446,17 +2958,23 @@ def get_model_file(mid, aid): artifact = model.get("artifact:%s" % aid, None) if artifact is None: - cherrypy.log.error("slycat.web.server.handlers.py get_model_file", - "cherrypy.HTTPError 404 artifact is None for aid: %s" % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_file", + "cherrypy.HTTPError 404 artifact is None for aid: %s" % aid, + ) raise cherrypy.HTTPError(404) artifact_type = model["artifact-types"][aid] if artifact_type != "file": - cherrypy.log.error("slycat.web.server.handlers.py get_model_file", - "cherrypy.HTTPError 400 %s is not a file artifact." % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_file", + "cherrypy.HTTPError 400 %s is not a file artifact." % aid, + ) raise cherrypy.HTTPError("400 %s is not a file artifact." % aid) fid = artifact - cherrypy.response.headers["content-type"] = model["_attachments"][fid]["content_type"] + cherrypy.response.headers["content-type"] = model["_attachments"][fid][ + "content_type" + ] return database.get_attachment(mid, fid) @@ -2465,14 +2983,14 @@ def get_model_parameter(mid, aid): """ Retrieves a model parameter (name / value pair) artifact. The result is a JSON expression and may be arbitrarily complex. - + Arguments: mid {string} -- model id aid {string} -- artifact id - + Raises: cherrypy.HTTPError: 404 Unknown artifact - + Returns: json """ @@ -2484,18 +3002,20 @@ def get_model_parameter(mid, aid): try: return slycat.web.server.get_model_parameter(database, model, aid) except KeyError as e: - cherrypy.log.error("slycat.web.server.handlers.py get_model_parameter", - "cherrypy.HTTPError 404 unknown artifact: %s" % aid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_model_parameter", + "cherrypy.HTTPError 404 unknown artifact: %s" % aid, + ) raise cherrypy.HTTPError("404 Unknown artifact: %s" % aid) def get_bookmark(bid): """ Retrieves a bookmark - an arbitrary collection of client state. - + Arguments: bid {string} -- bookmark id - + Returns: json -- representation of client state """ @@ -2520,14 +3040,14 @@ def get_bookmark(bid): def get_user(uid, time): """ Retrieve directory information for a given user. - + Arguments: uid {string} -- users id time {int} -- time int to prevent caching - + Raises: cherrypy.HTTPError: 404 user not found - + Returns: json -- user info """ @@ -2536,8 +3056,10 @@ def get_user(uid, time): uid = cherrypy.request.login user = cherrypy.request.app.config["slycat-web-server"]["directory"](uid) if user is None: - cherrypy.log.error("slycat.web.server.handlers.py get_user", - "cherrypy.HTTPError 404 user is None for uid: %s" % uid) + cherrypy.log.error( + "slycat.web.server.handlers.py get_user", + "cherrypy.HTTPError 404 user is None for uid: %s" % uid, + ) raise cherrypy.HTTPError(404) # Add the uid to the record, since the caller may not know it. user["uid"] = uid @@ -2548,14 +3070,14 @@ def get_user(uid, time): def get_model_statistics(mid): """ gets statistics on the model - + Arguments: mid {string} -- model ID time {int} -- time int to prevent caching - + Raises: cherrypy.HTTPError: 404 user not found - + Returns: mid {string} -- mid hdf5_file_size {float} -- hdf5_file_size @@ -2576,34 +3098,48 @@ def get_model_statistics(mid): # amount of time it took to make the model if "finished" in model and model["finished"] is not None: delta_creation_time = ( - datetime.datetime.strptime(model["finished"], "%Y-%m-%dT%H:%M:%S.%f") - datetime.datetime.strptime( - model["created"], - "%Y-%m-%dT%H:%M:%S.%f")).total_seconds() + datetime.datetime.strptime(model["finished"], "%Y-%m-%dT%H:%M:%S.%f") + - datetime.datetime.strptime(model["created"], "%Y-%m-%dT%H:%M:%S.%f") + ).total_seconds() else: delta_creation_time = 0 if "job_running_time" in model and "artifact:job_submit_time" in model: delta_queue_time = ( - datetime.datetime.strptime(model["job_running_time"], "%Y-%m-%dT%H:%M:%S.%f") - datetime.datetime.strptime( - model["artifact:job_submit_time"], "%Y-%m-%dT%H:%M:%S.%f")).total_seconds() + datetime.datetime.strptime( + model["job_running_time"], "%Y-%m-%dT%H:%M:%S.%f" + ) + - datetime.datetime.strptime( + model["artifact:job_submit_time"], "%Y-%m-%dT%H:%M:%S.%f" + ) + ).total_seconds() else: delta_queue_time = 0 if "job_completed_time" in model and "job_running_time" in model: delta_running_time = ( - datetime.datetime.strptime(model["job_completed_time"], - "%Y-%m-%dT%H:%M:%S.%f") - datetime.datetime.strptime( - model["job_running_time"], "%Y-%m-%dT%H:%M:%S.%f")).total_seconds() + datetime.datetime.strptime( + model["job_completed_time"], "%Y-%m-%dT%H:%M:%S.%f" + ) + - datetime.datetime.strptime( + model["job_running_time"], "%Y-%m-%dT%H:%M:%S.%f" + ) + ).total_seconds() delta_model_compute_time = ( - datetime.datetime.strptime(model["finished"], "%Y-%m-%dT%H:%M:%S.%f") - datetime.datetime.strptime( - model["model_compute_time"], "%Y-%m-%dT%H:%M:%S.%f")).total_seconds() + datetime.datetime.strptime(model["finished"], "%Y-%m-%dT%H:%M:%S.%f") + - datetime.datetime.strptime( + model["model_compute_time"], "%Y-%m-%dT%H:%M:%S.%f" + ) + ).total_seconds() elif "job_completed_time" in model: delta_running_time = ( - datetime.datetime.strptime(model["job_completed_time"], - "%Y-%m-%dT%H:%M:%S.%f") - datetime.datetime.strptime( - model["finished"], "%Y-%m-%dT%H:%M:%S.%f")).total_seconds() + datetime.datetime.strptime( + model["job_completed_time"], "%Y-%m-%dT%H:%M:%S.%f" + ) + - datetime.datetime.strptime(model["finished"], "%Y-%m-%dT%H:%M:%S.%f") + ).total_seconds() delta_model_compute_time = 0 - + else: delta_running_time = 0 delta_model_compute_time = 0 @@ -2619,14 +3155,18 @@ def get_model_statistics(mid): pulling_time = 0 # get hdf5 root dir - hdf5_root_directory = cherrypy.tree.apps[""].config["slycat-web-server"]["data-store"] + hdf5_root_directory = cherrypy.tree.apps[""].config["slycat-web-server"][ + "data-store" + ] # get models hdf5 footprint hdf5_file_size = 0 for key, value in model["artifact-types"].items(): if value == "hdf5": array = model["artifact:%s" % key] - hdf5_file_path = os.path.join(hdf5_root_directory, array[0:2], array[2:4], array[4:6], array + ".hdf5") + hdf5_file_path = os.path.join( + hdf5_root_directory, array[0:2], array[2:4], array[4:6], array + ".hdf5" + ) hdf5_file_size += os.path.getsize(hdf5_file_path) # calc total model server data footprint @@ -2640,7 +3180,11 @@ def get_model_statistics(mid): total_hdf5_server_size += os.path.getsize(fp) try: - server_cache_size = float(sys.getsizeof(_pickle.dumps(slycat.web.server.server_cache.cache))) / 1024.0 / 1024.0 + server_cache_size = ( + float(sys.getsizeof(_pickle.dumps(slycat.web.server.server_cache.cache))) + / 1024.0 + / 1024.0 + ) except: server_cache_size = 0 @@ -2653,14 +3197,20 @@ def get_model_statistics(mid): "model": model, "delta_creation_time": float(delta_creation_time) / 60, "couchdb_doc_size": sys.getsizeof(model) / 1024.0 / 1024.0, - "hdf5_footprint": 100.0 * (float(hdf5_file_size) / float(total_hdf5_server_size)), + "hdf5_footprint": 100.0 + * (float(hdf5_file_size) / float(total_hdf5_server_size)), "job_pending_time": float(delta_queue_time) / 60, "job_running_time": float(delta_running_time) / 60, "model_compute_time": float(model_compute_time) / 60, "pulling_time": float(pulling_time) / 60, - "analysis_computation_time": 0.0 if "analysis_computation_time" not in model else float( - model["analysis_computation_time"]), - "db_creation_time": 0.0 if "db_creation_time" not in model else float(model["db_creation_time"]) + "analysis_computation_time": ( + 0.0 + if "analysis_computation_time" not in model + else float(model["analysis_computation_time"]) + ), + "db_creation_time": ( + 0.0 if "db_creation_time" not in model else float(model["db_creation_time"]) + ), } @@ -2668,10 +3218,10 @@ def get_model_statistics(mid): @cherrypy.tools.json_out(on=True) def post_remotes(): """ - Given username, hostname, password as a json payload - establishes a session with the remote host and attaches - it to the users session - :return: {"sid":sid, "status":boolean, msg:""} + Given username, hostname, password as a json payload + establishes a session with the remote host and attaches + it to the users session + :return: {"sid":sid, "status":boolean, msg:""} """ username = cherrypy.request.json["username"] hostname = cherrypy.request.json["hostname"] @@ -2680,77 +3230,103 @@ def post_remotes(): # (they don't exist for rsa-cert auth) if username == None: username = cherrypy.request.login - + msg = "" agent = cherrypy.request.json.get("agent", None) sid = slycat.web.server.remote.create_session(hostname, username, password, agent) - ''' + """ save sid to user session the session will be stored as follows in the users session {sessions:[{{"sid": sid,"hostname": hostname, "username": username}},...]} - ''' + """ try: database = slycat.web.server.database.couchdb.connect() session = database.get("session", cherrypy.request.cookie["slycatauth"].value) for i in range(len(session["sessions"])): if session["sessions"][i]["hostname"] == hostname: - if("sid" in session["sessions"][i] and session["sessions"][i]["sid"] is not None): - slycat.web.server.remote.delete_session(session["sessions"][i]["sid"]) + if ( + "sid" in session["sessions"][i] + and session["sessions"][i]["sid"] is not None + ): + slycat.web.server.remote.delete_session( + session["sessions"][i]["sid"] + ) del session["sessions"][i] - session["sessions"].append({"sid": sid, "hostname": hostname, "username": username, "session_type": "ssh"}) + session["sessions"].append( + { + "sid": sid, + "hostname": hostname, + "username": username, + "session_type": "ssh", + } + ) database.save(session) except Exception as e: cherrypy.log.error("login could not save session for remotes %s" % e) msg = "login could not save session for remote host" return {"sid": sid, "status": True, "msg": msg} + @cherrypy.tools.json_in(on=True) @cherrypy.tools.json_out(on=True) def post_remotes_smb(): """ - Given username, hostname, password as a json payload - establishes a session with the remote host and attaches - it to the users session - :return: {"sid":sid, "status":boolean, msg:""} - user_name password + Given username, hostname, password as a json payload + establishes a session with the remote host and attaches + it to the users session + :return: {"sid":sid, "status":boolean, msg:""} + user_name password - encode with in js + encode with in js - b64EncodeUnicode(str) { - return btoa(encodeURIComponent(str).replace(/%([0-9A-F]{2})/g, function(match, p1) { - return String.fromCharCode('0x' + p1); - })); - } + b64EncodeUnicode(str) { + return btoa(encodeURIComponent(str).replace(/%([0-9A-F]{2})/g, function(match, p1) { + return String.fromCharCode('0x' + p1); + })); + } """ # try and decode the username and password username, password = slycat.web.server.decode_username_and_password() server = cherrypy.request.json["server"] share = cherrypy.request.json["share"] - cherrypy.log.error("username:%s server:%s share:%s" % (username, server , share)) + cherrypy.log.error("username:%s server:%s share:%s" % (username, server, share)) if username == None: username = cherrypy.request.login msg = "" - sid = slycat.web.server.smb.create_session(username,password,server,share) - ''' + sid = slycat.web.server.smb.create_session(username, password, server, share) + """ save sid to user session the session will be stored as follows in the users session {sessions:[{{"sid": sid,"hostname": hostname, "username": username}},...]} - ''' + """ try: database = slycat.web.server.database.couchdb.connect() session = database.get("session", cherrypy.request.cookie["slycatauth"].value) for i in range(len(session["sessions"])): if session["sessions"][i]["hostname"] == server: - if("sid" in session["sessions"][i] and session["sessions"][i]["sid"] is not None): - slycat.web.server.remote.delete_session(session["sessions"][i]["sid"]) + if ( + "sid" in session["sessions"][i] + and session["sessions"][i]["sid"] is not None + ): + slycat.web.server.remote.delete_session( + session["sessions"][i]["sid"] + ) del session["sessions"][i] - session["sessions"].append({"sid": sid, "hostname": server, "username": username, "session_type": "smb"}) + session["sessions"].append( + { + "sid": sid, + "hostname": server, + "username": username, + "session_type": "smb", + } + ) database.save(session) except Exception as e: cherrypy.log.error("login could not save session for remotes %s" % e) msg = "login could not save session for remote host" return {"sid": sid, "status": True, "msg": msg} + @cherrypy.tools.json_in(on=True) @cherrypy.tools.json_out(on=True) def post_smb_browse(hostname, path): @@ -2760,65 +3336,81 @@ def post_smb_browse(hostname, path): with slycat.web.server.smb.get_session(sid) as session: return session.browse(path=path) + def post_hdf5_table(path, pid, mid): """ - Takes a user selected path inside an HDF5 file, and stores the - table at that path as either inputs or outputs for the model. + Takes a user selected path inside an HDF5 file, and stores the + table at that path as either inputs or outputs for the model. - path {string} -- path to table inside of HDF5 file + path {string} -- path to table inside of HDF5 file """ # Need to find the HDF5 stored on Slycat server, so we can query it for the path. - path = path.replace('-', '/') + path = path.replace("-", "/") database = slycat.web.server.database.couchdb.connect() model = database.get("model", mid) project = database.get("project", pid) - did = model['project_data'][0] + did = model["project_data"][0] project_data = database.get("project_data", did) - file_name = project_data['hdf5_name'] - hdf5_path = cherrypy.request.app.config["slycat-web-server"]["data-store"] + "/" + file_name - h5 = h5py.File(hdf5_path, 'r') + file_name = project_data["hdf5_name"] + hdf5_path = ( + cherrypy.request.app.config["slycat-web-server"]["data-store"] + "/" + file_name + ) + h5 = h5py.File(hdf5_path, "r") table = list(h5[path]) column_headers = list(h5[path].dims[1][0]) headers = [] attributes = [] dimensions = [{"name": "row", "type": "int64", "begin": 0, "end": len(table[0])}] - if 'hdf5-inputs' not in model: - model['hdf5-inputs'] = path + if "hdf5-inputs" not in model: + model["hdf5-inputs"] = path else: - model['hdf5-outputs'] = path + model["hdf5-outputs"] = path slycat.web.server.authentication.require_project_writer(project) database.save(model) cherrypy.response.status = "200 Project updated." + def post_combine_hdf5_tables(mid): """ - Combines user selected input/output tables inside of an HDF5 file into one table. - Uses the hdf5-input and hdf5-output paths in the model to find the tables in the HDF5 file. + Combines user selected input/output tables inside of an HDF5 file into one table. + Uses the hdf5-input and hdf5-output paths in the model to find the tables in the HDF5 file. """ database = slycat.web.server.database.couchdb.connect() model = database.get("model", mid) - pid = model['project'] + pid = model["project"] project = database.get("project", pid) slycat.web.server.authentication.require_project_writer(project) - input_path = '/' + model['hdf5-inputs'] - output_path = '/' + model['hdf5-outputs'] - did = model['project_data'][0] + input_path = "/" + model["hdf5-inputs"] + output_path = "/" + model["hdf5-outputs"] + did = model["project_data"][0] project_data = database.get("project_data", did) - file_name = project_data['hdf5_name'] - hdf5_path = cherrypy.request.app.config["slycat-web-server"]["data-store"] + "/" + file_name - h5 = h5py.File(hdf5_path, 'r') + file_name = project_data["hdf5_name"] + hdf5_path = ( + cherrypy.request.app.config["slycat-web-server"]["data-store"] + "/" + file_name + ) + h5 = h5py.File(hdf5_path, "r") # Getting indices for input/output columns unformatted_input = list(h5[input_path]) input_column_headers_indices = [i for i in range(0, len(unformatted_input[0]))] unformatted_output = list(h5[output_path]) - output_column_headers_indices = [i for i in range(len(unformatted_input[0]), (len(unformatted_input[0]) + len(unformatted_output[0])))] - - slycat.web.server.put_model_parameter(database, model, 'input-columns', input_column_headers_indices, True) - slycat.web.server.put_model_parameter(database, model, 'output-columns', output_column_headers_indices, True) + output_column_headers_indices = [ + i + for i in range( + len(unformatted_input[0]), + (len(unformatted_input[0]) + len(unformatted_output[0])), + ) + ] + + slycat.web.server.put_model_parameter( + database, model, "input-columns", input_column_headers_indices, True + ) + slycat.web.server.put_model_parameter( + database, model, "output-columns", output_column_headers_indices, True + ) combined_dataset = [] output_headers = [] @@ -2828,13 +3420,27 @@ def post_combine_hdf5_tables(mid): column_headers_input = list(h5[input_path].dims[1][0]) column_headers_output = list(h5[output_path].dims[1][0]) - # Once we have column headers, this is how we can get/store them. + # Once we have column headers, this is how we can get/store them. for i, column in enumerate(column_headers_input): - input_headers.append(str(column.decode('utf-8'))) - attributes.append({"name": str(column.decode('utf-8')), "type": str(type(unformatted_input[0][i])).split('numpy.')[1].split("'>")[0]}) + input_headers.append(str(column.decode("utf-8"))) + attributes.append( + { + "name": str(column.decode("utf-8")), + "type": str(type(unformatted_input[0][i])) + .split("numpy.")[1] + .split("'>")[0], + } + ) for j, column in enumerate(column_headers_output): - output_headers.append(str(column.decode('utf-8'))) - attributes.append({"name": str(column.decode('utf-8')), "type": str(type(unformatted_output[0][j])).split('numpy.')[1].split("'>")[0]}) + output_headers.append(str(column.decode("utf-8"))) + attributes.append( + { + "name": str(column.decode("utf-8")), + "type": str(type(unformatted_output[0][j])) + .split("numpy.")[1] + .split("'>")[0], + } + ) combined_headers = numpy.concatenate((input_headers, output_headers), axis=0) combined_headers = combined_headers.tolist() @@ -2846,58 +3452,70 @@ def post_combine_hdf5_tables(mid): for row in combined_data: combined_dataset.append(numpy.asarray(row)) - dimensions = [{"name": "row", "type": "int64", "begin": 0, "end": len(combined_dataset[0])}] + dimensions = [ + {"name": "row", "type": "int64", "begin": 0, "end": len(combined_dataset[0])} + ] array_index = 0 with slycat.web.server.database.couchdb.db_lock: - slycat.web.server.put_model_arrayset(database, model, 'data-table', input) - slycat.web.server.put_model_array(database, model, 'data-table', 0, attributes, dimensions) - slycat.web.server.put_model_arrayset_data(database, model, 'data-table', "%s/.../..." % array_index, combined_data) + slycat.web.server.put_model_arrayset(database, model, "data-table", input) + slycat.web.server.put_model_array( + database, model, "data-table", 0, attributes, dimensions + ) + slycat.web.server.put_model_arrayset_data( + database, model, "data-table", "%s/.../..." % array_index, combined_data + ) cherrypy.response.status = "200 Project updated." + def post_browse_hdf5(path, pid, mid): """ - Given a path inside of an HDF5 file, builds out the current tree at that path. - Formats the tree structure to conform with remote/hdf5 browser requirements. + Given a path inside of an HDF5 file, builds out the current tree at that path. + Formats the tree structure to conform with remote/hdf5 browser requirements. """ + def allkeys_single_level(obj, tree_structure): - path = obj.name # This is current top level path + path = obj.name # This is current top level path # Need to include all these fields because we are repurposing the remote file browser, which expects all these - tree_structure['path'] = path - tree_structure['name'] = [] - tree_structure['sizes'] = [] - tree_structure['types'] = [] - tree_structure['mtimes'] = [] - tree_structure['mime-types'] = [] + tree_structure["path"] = path + tree_structure["name"] = [] + tree_structure["sizes"] = [] + tree_structure["types"] = [] + tree_structure["mtimes"] = [] + tree_structure["mime-types"] = [] all_items = obj.items() for key, value in all_items: # key will be all the sub groups and datasets in the current path - tree_structure['name'].append(key) - tree_structure['sizes'].append(0) - tree_structure['mtimes'].append('2024') + tree_structure["name"].append(key) + tree_structure["sizes"].append(0) + tree_structure["mtimes"].append("2024") if isinstance(value, h5py.Group): - tree_structure['mime-types'].append('application/x-directory') - tree_structure['types'].append('d') + tree_structure["mime-types"].append("application/x-directory") + tree_structure["types"].append("d") else: - tree_structure['mime-types'].append('file') - tree_structure['types'].append('f') + tree_structure["mime-types"].append("file") + tree_structure["types"].append("f") return tree_structure + # Need to find the HDF5 stored on Slycat server, so we can query it for the path. - path = path.replace('-', '/') + path = path.replace("-", "/") database = slycat.web.server.database.couchdb.connect() model = database.get("model", mid) - did = model['project_data'][0] + did = model["project_data"][0] project_data = database.get("project_data", did) - file_name = project_data['hdf5_name'] - hdf5_path = cherrypy.request.app.config["slycat-web-server"]["data-store"] + "/" + file_name - h5 = h5py.File(hdf5_path, 'r') + file_name = project_data["hdf5_name"] + hdf5_path = ( + cherrypy.request.app.config["slycat-web-server"]["data-store"] + "/" + file_name + ) + h5 = h5py.File(hdf5_path, "r") tree_structure = {} current_level = allkeys_single_level(h5[path], tree_structure) json_payload = json.dumps(current_level) return json_payload + @cherrypy.tools.json_out(on=True) def get_remotes(hostname): """ @@ -2914,16 +3532,24 @@ def get_remotes(hostname): share = "" for h_session in session["sessions"]: if h_session["hostname"] == hostname: - if h_session["session_type"] == "ssh" and slycat.web.server.remote.check_session(h_session["sid"]): + if h_session[ + "session_type" + ] == "ssh" and slycat.web.server.remote.check_session(h_session["sid"]): status = True msg = "hostname session was found" - elif h_session["session_type"] == "smb" and slycat.web.server.smb.check_session(h_session["sid"]): + elif h_session[ + "session_type" + ] == "smb" and slycat.web.server.smb.check_session(h_session["sid"]): status = True msg = "hostname session was found" with slycat.web.server.smb.get_session(h_session["sid"]) as session: share = session.getShare() else: - session["sessions"][:] = [tup for tup in session["sessions"] if tup["hostname"] != hostname] + session["sessions"][:] = [ + tup + for tup in session["sessions"] + if tup["hostname"] != hostname + ] database.save(session) except Exception as e: @@ -2940,10 +3566,18 @@ def get_remote_show_user_password(): ssh = False slycat_pass = False msg = "unknown" - if cherrypy.request.app.config["slycat-web-server"]["remote-authentication"]["method"] == "password": + if ( + cherrypy.request.app.config["slycat-web-server"]["remote-authentication"][ + "method" + ] + == "password" + ): ssh = True msg = "password auth required for remotes" - if cherrypy.request.app.config["slycat-web-server"]["authentication"]["plugin"] == "slycat-password-authentication": + if ( + cherrypy.request.app.config["slycat-web-server"]["authentication"]["plugin"] + == "slycat-password-authentication" + ): slycat_pass = True return {"ssh": ssh, "slycat": slycat_pass, "msg": msg} @@ -2953,7 +3587,7 @@ def delete_remote(sid): TODO: this function needs review for deprecation as we no longer send the sid over Deletes a remote session created with POST /api/remotes - + Arguments: sid {string} -- unique session id """ @@ -3024,15 +3658,15 @@ def job_time(nodes, tasks, size): :return: json time in seconds as an integer {'time-seconds': 1800} """ if int(nodes) < 0 or int(tasks) < 0 or int(size) < 0: - time = None + time = None else: - time = int(1.8 * float(size)) / (float(nodes) * float(tasks)) + 900 + time = int(1.8 * float(size)) / (float(nodes) * float(tasks)) + 900 return { - 'time-seconds': time, - 'nodes': nodes, - 'tasks': tasks, - 'size': size - } # return an approximation based on size, nodes, and tasks for now + "time-seconds": time, + "nodes": nodes, + "tasks": tasks, + "size": size, + } # return an approximation based on size, nodes, and tasks for now @cherrypy.tools.json_in(on=True) @@ -3057,7 +3691,7 @@ def post_remote_command(hostname): hostname: str name of the hpc host Returns - ------- + ------- json message: string a message that is supplied by the agent @@ -3098,70 +3732,85 @@ def get_remote_job_status(hostname, jid): @cherrypy.tools.json_out(on=True) def post_remote_browse(hostname, path): sid, session_type = get_sid(hostname) - file_reject = re.compile( - cherrypy.request.json.get("file-reject")) if "file-reject" in cherrypy.request.json else None - file_allow = re.compile(cherrypy.request.json.get("file-allow")) if "file-allow" in cherrypy.request.json else None - directory_reject = re.compile( - cherrypy.request.json.get("directory-reject")) if "directory-reject" in cherrypy.request.json else None - directory_allow = re.compile( - cherrypy.request.json.get("directory-allow")) if "directory-allow" in cherrypy.request.json else None + file_reject = ( + re.compile(cherrypy.request.json.get("file-reject")) + if "file-reject" in cherrypy.request.json + else None + ) + file_allow = ( + re.compile(cherrypy.request.json.get("file-allow")) + if "file-allow" in cherrypy.request.json + else None + ) + directory_reject = ( + re.compile(cherrypy.request.json.get("directory-reject")) + if "directory-reject" in cherrypy.request.json + else None + ) + directory_allow = ( + re.compile(cherrypy.request.json.get("directory-allow")) + if "directory-allow" in cherrypy.request.json + else None + ) with slycat.web.server.remote.get_session(sid) as session: - return session.browse(path, file_reject, file_allow, directory_reject, directory_allow) + return session.browse( + path, file_reject, file_allow, directory_reject, directory_allow + ) def get_remote_file(hostname, path, **kwargs): """ - Uses an existing remote session to retrieve a remote file. The remote - session must have been created using :http:post:`/api/remotes`. Use - :http:post:`/api/remotes/(hostname)/browse(path)` to lookup remote file paths. - The returned file may be optionally cached on the server and retrieved - using :http:get:`/api/projects/(pid)/cache/(key)`. - - Parameters - ---------- - hostname: string - connection host name - path: string - path to file - kwargs: json - cache: – - Optional cache identifier. - Set to project to store the retrieved file in a project cache. - project: uuid - Project identifier. Required when cache is set to project. - key: uuid - Cached object key. Must be specified when cache is set to project. - Returns - ------- - file: binary - the file that was asked for - - Raises - ------ - 200 OK - The requested file is returned in the body of the response. - 404 Not Found - The session doesn’t exist or has timed-out. - 400 Bad Request - “Can’t read directory” The remote path is a directory instead of a file. - 400 Bad Request - “File not found” The remote path doesn’t exist. - 400 Bad Request - “Access denied” The session user doesn’t have permissions to access the file. + Uses an existing remote session to retrieve a remote file. The remote + session must have been created using :http:post:`/api/remotes`. Use + :http:post:`/api/remotes/(hostname)/browse(path)` to lookup remote file paths. + The returned file may be optionally cached on the server and retrieved + using :http:get:`/api/projects/(pid)/cache/(key)`. + + Parameters + ---------- + hostname: string + connection host name + path: string + path to file + kwargs: json + cache: – + Optional cache identifier. + Set to project to store the retrieved file in a project cache. + project: uuid + Project identifier. Required when cache is set to project. + key: uuid + Cached object key. Must be specified when cache is set to project. + Returns + ------- + file: binary + the file that was asked for + + Raises + ------ + 200 OK + The requested file is returned in the body of the response. + 404 Not Found + The session doesn’t exist or has timed-out. + 400 Bad Request + “Can’t read directory” The remote path is a directory instead of a file. + 400 Bad Request + “File not found” The remote path doesn’t exist. + 400 Bad Request + “Access denied” The session user doesn’t have permissions to access the file. """ sid, session_type = get_sid(hostname) - if session_type == 'smb': + if session_type == "smb": with slycat.web.server.smb.get_session(sid) as session: - split_list = path.split('/') + split_list = path.split("/") del split_list[1] del split_list[0] content_type, encoding = slycat.mime_type.guess_type(path) if content_type is None: content_type = "application/octet-stream" cherrypy.response.headers["content-type"] = content_type - return session.get_file(path='/{0}'.format('/'.join(split_list))) + return session.get_file(path="/{0}".format("/".join(split_list))) else: with slycat.web.server.remote.get_session(sid) as session: return session.get_file(path, **kwargs) @@ -3180,7 +3829,8 @@ def get_remote_image(hostname, path, **kwargs): with slycat.web.server.remote.get_session(sid) as session: return session.get_image(path, **kwargs) -#TODO: clean up everything + +# TODO: clean up everything @cherrypy.tools.json_out(on=True) def get_time_series_names(hostname, path, **kwargs): """ @@ -3193,11 +3843,12 @@ def get_time_series_names(hostname, path, **kwargs): sid, session_type = get_sid(hostname) with slycat.web.server.remote.get_session(sid) as session: csv_file = str(session.get_file(path, **kwargs).decode("utf-8")) - csv_file = csv_file.replace("\\r\\n","\r\n") + csv_file = csv_file.replace("\\r\\n", "\r\n") rows = [row.split(",") for row in csv_file.splitlines()] column_names = [name.strip() for name in rows[0]] + def _isNumeric(j): - """" + """ " Check if the input object is a numerical value, i.e. a float :param j: input object to check :return: boolean @@ -3226,7 +3877,9 @@ def _isNumeric(j): if len(response_time_series_names) >= 1: return response_time_series_names else: - raise cherrypy.HTTPError("400 could not detect timeseries names. There could be hidden characters in your csv") + raise cherrypy.HTTPError( + "400 could not detect timeseries names. There could be hidden characters in your csv" + ) def get_remote_video(hostname, vsid): @@ -3249,42 +3902,55 @@ def post_events(event): @cherrypy.tools.json_out(on=True) def get_configuration_markings(): - return [dict(list(marking.items()) + [("type", key)]) for key, marking in - list(slycat.web.server.plugin.manager.markings.items()) if - key in cherrypy.request.app.config["slycat-web-server"]["allowed-markings"]] + return [ + dict(list(marking.items()) + [("type", key)]) + for key, marking in list(slycat.web.server.plugin.manager.markings.items()) + if key in cherrypy.request.app.config["slycat-web-server"]["allowed-markings"] + ] + @cherrypy.tools.json_out(on=True) def get_selectable_configuration_markings(): - return [dict(list(marking.items()) + [("type", key)]) for key, marking in - list(slycat.web.server.plugin.manager.markings.items()) if - key in cherrypy.request.app.config["slycat-web-server"]["selectable-markings"]] + return [ + dict(list(marking.items()) + [("type", key)]) + for key, marking in list(slycat.web.server.plugin.manager.markings.items()) + if key + in cherrypy.request.app.config["slycat-web-server"]["selectable-markings"] + ] + @cherrypy.tools.json_out(on=True) def get_configuration_parsers(): - return [{"type": key, "label": parser["label"], "categories": parser["categories"]} for key, parser in - list(slycat.web.server.plugin.manager.parsers.items())] + return [ + {"type": key, "label": parser["label"], "categories": parser["categories"]} + for key, parser in list(slycat.web.server.plugin.manager.parsers.items()) + ] @cherrypy.tools.json_out(on=True) def get_configuration_remote_hosts(): remote_hosts = [] - for hostname, remote in list(cherrypy.request.app.config["slycat-web-server"]["remote-hosts"].items()): + for hostname, remote in list( + cherrypy.request.app.config["slycat-web-server"]["remote-hosts"].items() + ): agent = True if remote.get("agent", False) else False remote_hosts.append({"hostname": hostname, "agent": agent}) return remote_hosts + @cherrypy.tools.json_out(on=True) def get_configuration_smb_remote_hosts(): if "smb-remote-hosts" in cherrypy.request.app.config["slycat-web-server"]: hostnames = cherrypy.request.app.config["slycat-web-server"]["smb-remote-hosts"] json_hostnames = {"hostnames": hostnames} return json_hostnames - return {"hostnames":[]} + return {"hostnames": []} + @cherrypy.tools.json_out(on=True) def get_configuration_smb_domains(): """ - Returns the SMB domains from the slycat web server + Returns the SMB domains from the slycat web server configuration file :return: domain names """ @@ -3292,12 +3958,14 @@ def get_configuration_smb_domains(): domains = cherrypy.request.app.config["slycat-web-server"]["smb-domains"] json_domains = {"domains": domains} return json_domains - return {"domains":[]} + return {"domains": []} + @cherrypy.tools.json_out(on=True) def get_configuration_support_email(): return cherrypy.request.app.config["slycat-web-server"]["support-email"] + @cherrypy.tools.json_out(on=True) def get_configuration_ga_tracking_id(): return cherrypy.request.app.config["slycat-web-server"].get("ga-tracking-id", "") @@ -3309,12 +3977,21 @@ def get_configuration_version(): if not get_configuration_version.initialized: get_configuration_version.initialized = True try: - get_configuration_version.commit = \ - subprocess.Popen(["git", "rev-parse", "HEAD"], cwd=os.path.dirname(__file__), - stdout=subprocess.PIPE).communicate()[0].strip() + get_configuration_version.commit = ( + subprocess.Popen( + ["git", "rev-parse", "HEAD"], + cwd=os.path.dirname(__file__), + stdout=subprocess.PIPE, + ) + .communicate()[0] + .strip() + ) except: pass - return {"version": slycat.__version__, "commit": get_configuration_version.commit.decode('utf-8')} + return { + "version": slycat.__version__, + "commit": get_configuration_version.commit.decode("utf-8"), + } get_configuration_version.lock = threading.Lock() @@ -3324,8 +4001,10 @@ def get_configuration_version(): @cherrypy.tools.json_out(on=True) def get_configuration_wizards(): - return [dict([("type", type)] + list(wizard.items())) for type, wizard in - list(slycat.web.server.plugin.manager.wizards.items())] + return [ + dict([("type", type)] + list(wizard.items())) + for type, wizard in list(slycat.web.server.plugin.manager.wizards.items()) + ] def tests_request(*arguments, **keywords):