diff --git a/upsonic_on_prem/api/operations/user.py b/upsonic_on_prem/api/operations/user.py index 4dc08f50..3edb7c3b 100644 --- a/upsonic_on_prem/api/operations/user.py +++ b/upsonic_on_prem/api/operations/user.py @@ -218,12 +218,8 @@ def get_version_code_of_scope(): documentation_tasks = [] -@app.route(create_document_of_scope_url, methods=["POST"]) -def create_document_of_scope(): - global documentation_tasks - scope = request.form.get("scope") - version = request.form.get("version") +def create_document_of_scope_(scope, version): task_name = scope if version != None: task_name = scope+":"+version @@ -245,43 +241,198 @@ def create_document_of_scope(): while task_name in documentation_tasks: time.sleep(1) work = the_scope.documentation - return jsonify({"status": True, "result": work}) -@app.route(create_time_complexity_of_scope_url, methods=["POST"]) -def create_time_complexity_of_scope(): - scope = request.form.get("scope") - version = request.form.get("version") + print("Complated doc task: ", scope) + return work + + +time_complexity_tasks = [] + + +def create_time_complexity_of_scope_(scope, version): + task_name = scope if version != None: + task_name = scope+":"+version the_scope = Scope.get_version(scope+":"+version) else: the_scope = Scope(scope) - return jsonify({"status": True, "result": the_scope.create_time_complexity()}) + if not task_name in time_complexity_tasks: + time_complexity_tasks.append(task_name) + try: + work = the_scope.create_time_complexity() + except: + pass + try: + time_complexity_tasks.remove(task_name) + except: + pass + else: + while task_name in time_complexity_tasks: + time.sleep(1) + work = the_scope.time_complexity + + print("Complated time_complexity task: ", scope) + return work -@app.route(create_mistakes_of_scope_url, methods=["POST"]) -def create_mistakes_of_scope(): - scope = request.form.get("scope") - version = request.form.get("version") +mistakes_tasks = [] + + +def create_mistakes_of_scope_(scope, version): + task_name = scope if version != None: + task_name = scope+":"+version the_scope = Scope.get_version(scope+":"+version) else: the_scope = Scope(scope) - return jsonify({"status": True, "result": the_scope.create_mistakes()}) + if not task_name in mistakes_tasks: + mistakes_tasks.append(task_name) + try: + work = the_scope.create_mistakes() + except: + pass + try: + mistakes_tasks.remove(task_name) + except: + pass + else: + while task_name in mistakes_tasks: + time.sleep(1) + work = the_scope.mistakes -@app.route(create_required_test_types_of_scope_url, methods=["POST"]) -def create_required_test_types_of_scope(): - scope = request.form.get("scope") - version = request.form.get("version") + print("Complated mistakes_tasks task: ", scope) + return work + + + +required_test_types_tasks = [] + + +def create_required_test_types_of_scope_(scope, version): + task_name = scope if version != None: + task_name = scope+":"+version the_scope = Scope.get_version(scope+":"+version) else: the_scope = Scope(scope) - return jsonify({"status": True, "result": the_scope.create_required_test_types()}) + if not task_name in required_test_types_tasks: + required_test_types_tasks.append(task_name) + try: + work = the_scope.create_required_test_types() + except: + pass + try: + required_test_types_tasks.remove(task_name) + except: + pass + else: + while task_name in required_test_types_tasks: + time.sleep(1) + work = the_scope.required_test_types + + print("Complated required_test_types_tasks task: ", scope) + return work + + +tags_tasks = [] + + +def create_tags_of_scope_(scope, version): + task_name = scope + if version != None: + task_name = scope+":"+version + the_scope = Scope.get_version(scope+":"+version) + else: + the_scope = Scope(scope) + + if not task_name in tags_tasks: + tags_tasks.append(task_name) + try: + work = the_scope.create_tags() + except: + pass + try: + tags_tasks.remove(task_name) + except: + pass + else: + while task_name in tags_tasks: + time.sleep(1) + work = the_scope.tags + + print("Complated tags_tasks task: ", scope) + return work + + +security_analyses_tasks = [] + + +def create_security_analyses_of_scope_(scope, version): + task_name = scope + if version != None: + task_name = scope+":"+version + the_scope = Scope.get_version(scope+":"+version) + else: + the_scope = Scope(scope) + + if not task_name in security_analyses_tasks: + security_analyses_tasks.append(task_name) + try: + work = the_scope.create_security_analysis() + except: + pass + try: + security_analyses_tasks.remove(task_name) + except: + pass + else: + while task_name in security_analyses_tasks: + time.sleep(1) + work = the_scope.security_analysis + + print("Complated security_analyses_tasks task: ", scope) + return work + + +@app.route(create_document_of_scope_url, methods=["POST"]) +def create_document_of_scope(): + global documentation_tasks + + scope = request.form.get("scope") + version = request.form.get("version") + + return jsonify({"status": True, "result": create_document_of_scope_(scope, version)}) + +@app.route(create_time_complexity_of_scope_url, methods=["POST"]) +def create_time_complexity_of_scope(): + scope = request.form.get("scope") + version = request.form.get("version") + + return jsonify({"status": True, "result": create_time_complexity_of_scope_(scope, version)}) + + + + +@app.route(create_mistakes_of_scope_url, methods=["POST"]) +def create_mistakes_of_scope(): + scope = request.form.get("scope") + version = request.form.get("version") + + + return jsonify({"status": True, "result": create_mistakes_of_scope_(scope, version)}) + +@app.route(create_required_test_types_of_scope_url, methods=["POST"]) +def create_required_test_types_of_scope(): + scope = request.form.get("scope") + version = request.form.get("version") + + + return jsonify({"status": True, "result": create_required_test_types_of_scope_(scope, version)}) @app.route(create_tags_of_scope_url, methods=["POST"]) @@ -292,7 +443,7 @@ def create_tags_of_scope(): the_scope = Scope.get_version(scope+":"+version) else: the_scope = Scope(scope) - return jsonify({"status": True, "result": the_scope.create_tags()}) + return jsonify({"status": True, "result": create_tags_of_scope_(scope, version)}) @app.route(create_security_analysis_of_scope_url, methods=["POST"]) @@ -304,7 +455,7 @@ def create_security_analysis_of_scope(): else: the_scope = Scope(scope) - return jsonify({"status": True, "result": the_scope.create_security_analysis()}) + return jsonify({"status": True, "result": create_security_analyses_of_scope_(scope, version)}) @@ -514,17 +665,13 @@ def get_default_ai_model(): return jsonify({"status": True, "result": AI.default_model}) -@app.route(create_readme_url, methods=["POST"]) -def create_readme(): +def create_readme_(top_library, version, request=None): global documentation_tasks - top_library = request.form.get("top_library") - version = request.form.get("version") - print("CREATE README TASK for: ", top_library) print() - all_scopes_response = Scope.get_all_scopes_name_prefix(AccessKey(request.authorization.password), top_library) + all_scopes_response = Scope.get_all_scopes_name_prefix(AccessKey(request.authorization.password), top_library) if request != None else Scope.get_all_scopes_name_prefix(prefix = top_library) all_scopes = [] for each_scope in all_scopes_response: if version != None: @@ -582,7 +729,13 @@ def create_readme(): storage_4.set(sha256, result) - if len(AccessKey(request.authorization.password).scopes_read) == ["*"] or AccessKey(request.authorization.password).is_admin == True: + + make_sync = False + if request != None: + make_sync = len(AccessKey(request.authorization.password).scopes_read) == ["*"] or AccessKey(request.authorization.password).is_admin == True + else: + make_sync = True + if make_sync: path = top_library.replace(".", "/") if "." in top_library else top_library path += f'/README.md' @@ -617,9 +770,17 @@ def create_readme(): storage_4.set(sha256+"github_sha", sha_of_readme) get_sha = storage_4.get(sha256+"github_sha") - print("RETRIVE_HASH", get_sha) + print("RETRIVE_HASH", get_sha) - return jsonify({"status": True, "result": result}) + return result + + +@app.route(create_readme_url, methods=["POST"]) +def create_readme(): + top_library = request.form.get("top_library") + version = request.form.get("version") + + return jsonify({"status": True, "result": create_readme_(top_library, version, request)}) @app.route(get_readme_github_sync_url, methods=["POST"]) diff --git a/upsonic_on_prem/utils/scope.py b/upsonic_on_prem/utils/scope.py index 84bbce81..3ed1e8db 100644 --- a/upsonic_on_prem/utils/scope.py +++ b/upsonic_on_prem/utils/scope.py @@ -11,6 +11,7 @@ from upsonic_on_prem.utils.github_sync import github + import cloudpickle import dill @@ -21,6 +22,41 @@ import textwrap import time +import threading + + + + +background_tasks = [] + +def background_worker(group_name, func, *args, **kwargs): + #Write a function and if there is another task with same group_name wait for it to finish and run the new task after that + #If there is no task with same group_name start the task + + print("Automatic Background Worker Started for") + print(group_name) + + global background_tasks + + for i in background_tasks: + if i["group_name"] == group_name: + i["thread"].join() + + the_thread = threading.Thread(target=func, args=args, kwargs=kwargs) + the_thread.start() + background_tasks.append({"group_name":group_name, "thread":the_thread}) + + +def split_dotted_string(s): + results = [] + while '.' in s: + results.append(s) + s = s.rsplit('.', 1)[0] + results.append(s) + results = results[1:] + return results + + class Scope: def __init__(self, key, specific=False): @@ -77,13 +113,7 @@ def delete(self, user=None): github.delete_file(scope=self, message=f"Deleted {path} by {user.name}") - def split_dotted_string(s): - results = [] - while '.' in s: - results.append(s) - s = s.rsplit('.', 1)[0] - results.append(s) # to include the first part - return results + #Checking for top libray if len(user.scopes_read) == ["*"] or user.is_admin == True: @@ -455,6 +485,22 @@ def code(self): return source def set_code(self, code): + from upsonic_on_prem.api.operations.user import create_document_of_scope_, create_time_complexity_of_scope_, create_mistakes_of_scope_, create_required_test_types_of_scope_, create_tags_of_scope_, create_security_analyses_of_scope_, create_readme_ + currently_code = self.code + if currently_code != code: + task_id = "create_documentation"+self.key + background_worker(task_id, create_document_of_scope_, scope=self.key, version=None) + background_worker("create_time_complexity_"+self.key, create_time_complexity_of_scope_, scope=self.key, version=None) + background_worker("create_mistakes_"+self.key, create_mistakes_of_scope_, scope=self.key, version=None) + background_worker("create_required_test_types_"+self.key, create_required_test_types_of_scope_, scope=self.key, version=None) + background_worker("create_tags_"+self.key, create_tags_of_scope_, scope=self.key, version=None) + background_worker("create_security_analyses_"+self.key, create_security_analyses_of_scope_, scope=self.key, version=None) + readmes = split_dotted_string(self.key) + print("Triggered readmes", readmes) + for i in readmes: + task_id = "create_readme_"+i + background_worker(task_id, create_readme_, top_library=i, version=None) + return self.the_storage.set(self.key + ":code", code) @property @@ -584,9 +630,9 @@ def get_all_scopes_name(user: AccessKey): return result @staticmethod - def get_all_scopes_name_prefix(user, prefix): + def get_all_scopes_name_prefix(user=None, prefix=None): prefix = prefix + "." - all_scopes = Scope.get_all_scopes_name(user) + all_scopes = Scope.get_all_scopes_name(user) if user != None else Scope.get_all_scopes() result = [] for i in all_scopes: if i.startswith(prefix):