CSBID Cron Script

genome_browser/backend/core/jira/CSBID.py → csbid_cron_script/CSBID.py

@@ -1,12 +1,14 @@
 import requests 
-import json
-from django.conf import settings
-import base64 
-import re
+import json 
+import base64  
 import os
+from jira_import_util import JIRAImportUtil, JIRAObjectID
+from urllib.parse import quote
+import logging
+logger = logging.getLogger("Logger") 
 
 
-class CSBIDTask:    
+class CSBIDTask(JIRAImportUtil):    
 
     ANL_TOKEN = os.environ.get("ANL_API_TOKEN")
     JIRA_API_TOKEN_YONG = os.environ.get("JIRA_API_TOKEN_YONG")
@@ -27,8 +29,9 @@ class CSBIDTask:
 
 
     def __init__(self):
+        logger.debug("Start CSBID task...")
         self.workspace_id = self.get_workspace_id()
-        self.base_url = self.get_base_url(self.workspace_id)
+        self.base_url = self.get_base_url(self.workspace_id) 
 
 
     def get_workspace_id(self):
@@ -168,9 +171,6 @@ def run(self):
         print("Total allcrystalsummary from api:",len(allcrystalsummary_json))
         print("Total crystals from api:",len(crystals_json)) 
 
-        targets_json = self.get_data();
-        return targets_json
-        create_record_response = self.create_new_jira_record(targets_json["targets"]);
         return create_record_response
 
 
@@ -182,6 +182,7 @@ def get_data(self):
         # the below is a token for Yeongshnn (generated by Gyorgy)
         api_key = self.ANL_TOKEN
         response = requests.get(login_endpoint, headers={'Accept': 'application/json', 'authorization': 'Bearer ' + api_key})
+        #print("ANL api login: ", response.text)
         response_json = response.json()
 
         sessionid = ''
@@ -203,30 +204,26 @@ def get_data(self):
         response = requests.post(braveapi_endpoint, data = json_to_get_targets)
         #print(response.text)
         targets_json = response.json()
-        return targets_json
+        #return targets_json
 
         print("Get Constructs...")
         json_to_get_constructs = '{"data" : [{"apiaction" : "constructsummary", "metadata" : "possible apiactions: [\'targetsummary\', \'constructsummary\', \'purifiedproteinsummary\', \'allpurifiedproteinsummary\', \'allcrystalsummary\' , \'crystalsummary\']"}],"submissionid" : "' + sessionid +'"}'
         response = requests.post(braveapi_endpoint, data = json_to_get_constructs)
-        print(response.text)
         constructs_json = response.json()
-
+        
         print("Get Purified Proteins...")
         json_to_get_purifiedproteins = '{"data" : [{"apiaction" : "purifiedproteinsummary", "metadata" :"possible apiactions: [\'targetsummary\', \'constructsummary\', \'purifiedproteinsummary\', \'allpurifiedproteinsummary\', \'allcrystalsummary\' , \'crystalsummary\']"}],"submissionid" : "' + sessionid +'"}'
         response = requests.post(braveapi_endpoint, data = json_to_get_purifiedproteins)
-        print(response.text)
         purifiedproteins_json = response.json()
-
+ 
         print("Get All Purified Proteins...")
-        json_to_get_allpurifiedproteins = '{"data" : [{"apiaction" : "allpurifiedproteinsummary", "metadata" : "possible apiactions: [\'targetsummary\', \'constructsummary\', \'purifiedproteinsummary\', \'allpurifiedproteinsummary\', \'allcrystalsummary\' , \'crystalsummary\']"}],"submissionid" : "' + sessionid +'"}'
-        response = requests.post(braveapi_endpoint, data = json_to_get_allpurifiedproteins)
-        print(response.text)
-        allpurifiedproteins_json = response.json()
+        json_to_get_all_purifiedproteins = '{"data" : [{"apiaction" : "allpurifiedproteinsummary", "metadata" :"possible apiactions: [\'targetsummary\', \'constructsummary\', \'purifiedproteinsummary\', \'allpurifiedproteinsummary\', \'allcrystalsummary\' , \'crystalsummary\']"}],"submissionid" : "' + sessionid +'"}'
+        response = requests.post(braveapi_endpoint, data = json_to_get_all_purifiedproteins)
+        all_purifiedproteins_json = response.json()
 
         print("Get All Crystal Summary...")
         json_to_get_allcrystalsummary = '{"data" : [{"apiaction" : "allcrystalsummary", "metadata" : "possible apiactions: [\'targetsummary\', \'constructsummary\', \'purifiedproteinsummary\', \'allpurifiedproteinsummary\', \'allcrystalsummary\' , \'crystalsummary\']"}],"submissionid" : "' + sessionid +'"}'
         response = requests.post(braveapi_endpoint, data = json_to_get_allcrystalsummary)
-        print(response.text)
         allcrystalsummary_json = response.json()
         print("All Crystal Summary source size:", len(allcrystalsummary_json["allcrystals"]))
 
@@ -258,55 +255,75 @@ def get_data(self):
                 "structuresummary_json_list":structuresummary_json_list}
 
 
-    def create_new_jira_record(self, data):
-
-        for data_item in data:
-            found = self.check_jira_record_exit(data_item)
-            if not found:
-                print("not found, create new...")
-                self.push_new_record(data_item)
 
-        return None
-
-
-    def push_new_record(self, data):
-        print("Create new jira record...") 
-
-        print("target_id: "+data["targets"][0]["targetid"])
-        attributes_data = [
+    def get_target_attributes_data(self, data):
+        return [
             {
                     "objectTypeAttributeId": "96",  # Attribute ID for Targetid
                     "objectAttributeValues": [
                         {
-                            "value": data["targets"][0]["targetid"] # Setting the value for Targetid to UNIQUEID123
+                            "value": data["targetid"].strip()  #Targetid    
+                        }
+                    ]
+            },
+            {
+                    "objectTypeAttributeId": "97", #Originaltargetid
+                    "objectAttributeValues": [
+                        {
+                            "value": data["originaltargetid"].strip()  
+                        }
+                    ]
+            },
+            {
+                    "objectTypeAttributeId": "491", #Parent Name
+                    "objectAttributeValues": [
+                        {
+                            "value": self.getVirusObjectID(data["speciesname"].strip()) 
+                        }
+                    ]
+            }, 
+            {
+                    "objectTypeAttributeId": "99", #Taxonid
+                    "objectAttributeValues": [
+                        {
+                            "value": data["taxonid"].strip() 
+                        }
+                    ]
+            },
+            {
+                    "objectTypeAttributeId": "100", #Targetannotation , unique key
+                    "objectAttributeValues": [
+                        {
+                            "value": data["targetannotation"].strip() 
+                        }
+                    ]
+            },
+            {
+                    "objectTypeAttributeId": "101", #Parentproteinseguid
+                    "objectAttributeValues": [
+                        {
+                            "value": data["parentproteinseguid"].strip()  
+                        }
+                    ]
+            },
+            {
+                    "objectTypeAttributeId": "102", #Parentproteinseq
+                    "objectAttributeValues": [
+                        {
+                            "value": data["parentproteinseq"].strip()  
+                        }
+                    ]
+            },
+            {
+                    "objectTypeAttributeId": "103", #Parentdnaseq
+                    "objectAttributeValues": [
+                        {
+                            "value": data["parentdnaseq"].strip()  
                         }
                     ]
             }
         ]
 
-        create_data = {
-            "objectTypeId": "9",  # Create a "Target" object
-            "attributes": attributes_data,
-            "hasAvatar": False  # Optional avatar
-        }
-
-        base_url = f'{self.base_url}/object/create'
-        headers = {
-                    'Authorization': f'Basic {self.encoded_credentials}',
-                    'Content-Type': 'application/json'
-        }
-
-        # Perform the POST request to create a new asset record
-        create_response = requests.post(base_url, headers=headers, data=json.dumps(create_data))
-
-        # Check the response and handle accordingly
-        if create_response.status_code in [200, 201]:
-            create_results = create_response.json()
-            print("Asset creation successful:", create_results)
-            return create_results
-        else:
-            print("Failed to create asset:", create_response.status_code, create_response.text)
-            return None
 
     def get_construct_attributes_data(self, data):
         return [
@@ -671,16 +688,6 @@ def get_purifiedproteins_attributes_data(self, data):
             } 
         ]
 
-        headers = {
-            'Authorization': f'Basic {self.encoded_credentials}',
-            'Content-Type': 'application/json'
-        }
-        # IQL search query
-        iql_query = f'Targetid == "{data_item["targetid"]}" '
-        search_url = f'{self.base_url}/object/aql'
-        search_data = {
-            "qlQuery": iql_query
-        }
 
     def get_proteininventory_attributes_data(self, data):
         return [ 
@@ -1798,32 +1805,94 @@ def delete_jira_record(self, record_id):
             return False
 
 
-
+    def getVirusObjectID(self, speciesname):
+        return self.virus_map[speciesname.lower().strip()]
+
+
+    def getTargetObjectID(self, target_id):
+        return  self.target_id_map[target_id]
+
 
-    '''
-    print('CSBID run task...') 
-    jira_servicedeskapi_url = 'https://taskforce5.atlassian.net/rest/servicedeskapi/assets/workspace'  
-    response = requests.get(jira_servicedeskapi_url, 
-                            headers={'Content-Type': 'application/json', 
-                                    'authorization': 'Basic '+ self.encoded_credentials})
-    workspaces_response_json = response.json()
-    print(workspaces_response_json["values"])
-    print(workspaces_response_json)
+    def setTargetIDMap(self, response_list): 
+        for item in response_list:  
+            attributes = item["attributes"]
+            for attr in attributes:
+                if attr["objectTypeAttributeId"] == "96":  #Target ID attribute
+                    key = attr["objectAttributeValues"][0]["value"].strip() 
+                    if key in self.target_id_map:
+                        print("Something wrong!! Duplicated Target ID in target_id_map ",key)
+                    else:    
+                        self.target_id_map[key] = item["id"] 
+
 
+    def getPurProteinObjectID(self, purbatchcproprcid): 
+        try:
+            if (len(self.all_pur_protein_id_map)!=0) and (purbatchcproprcid in self.all_pur_protein_id_map):
+                return self.all_pur_protein_id_map[purbatchcproprcid]
+            #elif (len(self.pur_protein_id_map)!=0) and (purbatchcproprcid in self.pur_protein_id_map):
+            #    return self.pur_protein_id_map[purbatchcproprcid]
+            else:
+                return None
+        except:
+            return None
+        '''
+        if (len(self.pur_protein_id_map)==0) or (purbatchcproprcid not in self.pur_protein_id_map):
+            return None
+        try:
+            return self.pur_protein_id_map[purbatchcproprcid]
+        except:
+            return None
+        '''
+
 
-    workspace_id = workspaces_response_json["values"][0]["workspaceId"]
-    jira_servicedeskapi_url = "https://api.atlassian.com/jsm/assets/workspace/" + workspace_id + "/v1/objectschema/list"
-    response = requests.get(jira_servicedeskapi_url, 
-                            headers={'Content-Type': 'application/json', 
-                                    'authorization': 'Basic '+ self.encoded_credentials})
+    def setPurProteinIDMap(self, response_list, attribute_id):
+        try:
+            for item in response_list:  
+                attributes = item["attributes"] 
+                for attr in attributes:
+                    if attr["objectTypeAttributeId"] == attribute_id:  #Pur Protein Purbatchcproprcid attribute 326,374
+                        key = attr["objectAttributeValues"][0]["value"].strip() 
+                        if key in self.pur_protein_id_map:
+                            print("Something wrong!! Duplicated Pur protein Purbatchcproprcid in pur_protein_id_map",key)
+                        else:    
+                            self.pur_protein_id_map[key] = item["id"] 
+        except:
+            print("Erroe:setPurProteinIDMap response_list:",response_list)
+
 
-    print(response.json())
+    def setAllPurProteinIDMap(self, response_list, attribute_id):
+        try:
+            for item in response_list:  
+                attributes = item["attributes"] 
+                for attr in attributes:
+                    if attr["objectTypeAttributeId"] == attribute_id:  #Pur Protein Purbatchcproprcid attribute 326,374
+                        key = attr["objectAttributeValues"][0]["value"].strip() 
+                        if key in self.all_pur_protein_id_map:
+                            print("Something wrong!! Duplicated Pur protein Purbatchcproprcid in all_pur_protein_id_map",key)
+                        else:    
+                            self.all_pur_protein_id_map[key] = item["id"] 
+        except:
+            print("Error:setAllPurProteinIDMap response_list:",response_list)
+
 
-    #update assets
-    base_url = f'https://api.atlassian.com/jsm/assets/workspace/{workspace_id}/v1'
+    def remove_purifiedproteins_from_all_purifiedproteins(self, purifiedproteins_json, all_purifiedproteins_json):
+        out=[]
+        for apitem in all_purifiedproteins_json:
+            found = False
+            for pitem in purifiedproteins_json: 
+                if apitem["purbatchcproprcid"] == pitem["purbatchcproprcid"]:
+                    found = True
+                    break
+                    #print("Found:",pitem["cloneidf"],apitem["cloneidf"])
+            if not found:
+                out.append(apitem)
+
+        return out
 
-    return update_response.json()
-    '''
+
+    os.umask(0)
+    def opener(path, flags):
+        return os.open(path, flags, 0o777)
 
     def upload_attachment(self, object_id, structure_json):
         file_path = os.path.dirname(os.path.abspath(__file__)) + "/temp_files/"  #current path: /srv/app/core/jira
@@ -1836,7 +1905,9 @@ def upload_attachment(self, object_id, structure_json):
                 if mediatypes['commonname'] == 'coordinates':
                     #print ('found coordinates')
                     coordinates = base64.b64decode(mediatypes['filedata'])
-                    with open(file_path+"structure.pdb", "w") as f:
+
+                    #file_path+"structure.pdb"
+                    with open("/temp_data/"+"structure.pdb", "w") as f:
                         print(coordinates, file=f)
                         file_generated = True
 
@@ -1911,3 +1982,6 @@ def upload_attachment(self, object_id, structure_json):
             print("Failed to attach file:", attach_response.status_code, attach_response.text)
 
 
+
+
+

csbid_cron_script/Dockerfile

@@ -0,0 +1,14 @@
+
+FROM python:3
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY .env .env
+COPY . .
+
+#CMD [ "python", "./my.py" ]
+ENTRYPOINT ["sh", "start.sh"]
+#ENTRYPOINT ["tail", "-f", "/dev/null"] 

csbid_cron_script/README.md

@@ -0,0 +1,26 @@
+# CSBID Cron Script
+
+
+A Python script that automates data retrieval from CSBID, mailin_SAXS, and simplescattering API. The script then pushes this data to the Atlassian taskforce5 Asset. It's currently running as a Spin CronJob.
+
+
+## Installation
+
+1. Get API Keys from .env file at [https://drive.google.com/file/d/1H4Mxbo6sc3ZDC4OthUwUdDcW9LN-w3Qz/view?usp=drive_link]
+2. build with tag:
+    ```sh
+    docker buildx build --platform linux/amd64 --no-cache -t registry.nersc.gov/m4521/brave-jira:1 .
+    ```
+    or
+
+    mac chip: --platform linux/rm64
+3. run locally:
+    ```sh
+    docker run -it registry.nersc.gov/m4521/brave-jira:1
+    ```
+
+4. push image to Harbor:
+    ```sh
+    docker login registry.nersc.gov
+    docker push registry.nersc.gov/m4521/brave-jira:1
+    ```