From 68a22451a64f7bc5a9a7e85ece466e880b636671 Mon Sep 17 00:00:00 2001 From: MarianG Date: Tue, 26 May 2020 08:13:14 +0200 Subject: [PATCH 1/2] connector - more error handling api_actions - included method to listHosts and childtags, and included more error handling for listReports - similar to launchReports --- qualysapi/api_actions.py | 101 ++++++++++++++++++++++++++++++++++++++- qualysapi/connector.py | 50 ++++++++++++++++++- 2 files changed, 149 insertions(+), 2 deletions(-) diff --git a/qualysapi/api_actions.py b/qualysapi/api_actions.py index cd94eb3..c16ad94 100644 --- a/qualysapi/api_actions.py +++ b/qualysapi/api_actions.py @@ -23,6 +23,56 @@ def getHost(self, host): hostData.find("TRACKING_METHOD"), ) + + def listHosts( + self, + ips=None, + tags=None, + os_pattern=None, + tag_set_exclude=None, + id_min=None, + detailed=False, + echo_request=None, + limit=100): + + call = "/api/2.0/fo/asset/host/" + parameters = {"action": "list", "truncation_limit": str(limit)} + if detailed: + parameters["details"] ="All/AGs" + if tag_set_exclude: + parameters["tag_set_exclude"]=tags + if id_min: + parameters["id_min"] =str(id_min) + if ips: + parameters["ips"]=str(ips) + if tags: + parameters["use_tags"] ="1" + parameters["tag_set_by"]="name" + parameters["tag_set_include"] = tags + parameters["show_tags"] = "1" + if os_pattern: + parameters["os_pattern"] = os_pattern + if echo_request: + parameters["echo_request"] = echo_request + + hostData = objectify.fromstring(self.request(call, parameters).encode('utf-8')) + hostArray = [] + for host in hostData.RESPONSE.HOST_LIST.HOST: + hostArray.append( + Host( + host.find('DNS'), + host.find('ID'), + host.find('IP'), + host.find('LAST_VULN_SCAN_DATETIME'), + host.find('NETBIOS'), + host.find('OS'), + host.find('TRACKING_METHOD') + ) + ) + + return hostArray + + def getHostRange(self, start, end): call = "/api/2.0/fo/asset/host/" parameters = {"action": "list", "ips": f"{start}-{end}"} @@ -142,7 +192,7 @@ def listReportTemplates(self): def listReports(self, id=0): call = "/api/2.0/fo/report" - + max_retries = 10 if id == 0: parameters = {"action": "list"} @@ -150,6 +200,16 @@ def listReports(self, id=0): self.request(call, parameters).encode("utf-8") ).RESPONSE reportsArray = [] + while repData.find("REPORT_LIST") is None and max_retries > 0: + max_retries = max_retries - 1 + time.sleep(30) + qualys_resp = self.request(call, parameters).encode('utf-8') + logging.info("QUALYS_REPONSE "+ str(qualys_resp)) + repData = objectify.fromstring(qualys_resp).RESPONSE + + if max_retries <= 0: + logging.info("Report Listing not successful") + return None for report in repData.REPORT_LIST.REPORT: reportsArray.append( @@ -170,6 +230,20 @@ def listReports(self, id=0): else: parameters = {"action": "list", "id": id} + qualys_resp = self.request(call, parameters).encode('utf-8') + + repData_debug = objectify.fromstring(qualys_resp).RESPONSE + while repData_debug.find("REPORT_LIST") is None and max_retries > 0: + max_retries = max_retries - 1 + time.sleep(30) + qualys_resp = self.request(call, parameters).encode('utf-8') + logging.info("QUALYS_REPONSE "+ str(qualys_resp)) + repData_debug = objectify.fromstring(qualys_resp).RESPONSE + + + if max_retries <= 0: + logging.info("Report Listing not successful") + return None repData = objectify.fromstring( self.request(call, parameters).encode("utf-8") ).RESPONSE.REPORT_LIST.REPORT @@ -366,6 +440,31 @@ def listScans(self, launched_after="", state="", target="", type="", user_login= return scanArray + def listChildTags(self, tag_name=None, tag_id=None, filename=None): + if tag_id: + files =""" + +"""+tag_id+""" + +""" + elif filename: + files = open(filename,"rb").read() + elif tag_name: + files =(""" + +"""+tag_name+""" + +""").encode('ascii', 'ignore') + + call = "/qps/rest/2.0/search/am/tag" + parameters = files + response = objectify.fromstring(self.request(call, parameters, api_version=2, http_method="post").encode("utf-8")) + childs = list() + for child in response.getchildren()[3][0].Tag.children.list.getchildren(): + childs.append(child.getchildren()) + + return childs + def launchScan(self, title, option_title, iscanner_name, asset_groups="", ip=""): # TODO: Add ability to scan by tag. call = "/api/2.0/fo/scan/" diff --git a/qualysapi/connector.py b/qualysapi/connector.py index c3f3b52..71f86a2 100644 --- a/qualysapi/connector.py +++ b/qualysapi/connector.py @@ -407,7 +407,55 @@ def request( api_call, self.rate_limit_remaining[api_call], ) - if self.rate_limit_remaining[api_call] > rate_warn_threshold: + response = request.text + if ('1960' in response and 'This API cannot be run again until' in response): + max_retries = 10 + retry_count = 0 + while ('1960' in response and 'This API cannot be run again until' in response): + retry_count += 1 + time_to_wait = int(30) + logger.info('Concurrency Limit Exceeded waiting %d seconds. %d retries remaining' %(time_to_wait, max_retries-retry_count)) + time.sleep(time_to_wait) + + logger.info(str(url), str(data))# self.auth, headers, self.proxies) + if http_method == 'get': + # GET + logger.debug('GET request.') + request = self.session.get(url, params=data, auth=self.auth, headers=headers, proxies=self.proxies) + else: + # POST + logger.debug('POST request.') + # Make POST request. + request = self.session.post(url, data=data, auth=self.auth, headers=headers, proxies=self.proxies) + logger.debug('response headers =\n%s' % (str(request.headers))) + response = request.text + if retry_count >= max_retries: + break + elif ('1965' in response and 'This API cannot be run again for another' in response): + max_retries = 10 + retry_count = 0 + while ('1965' in response and 'This API cannot be run again for another' in response): + retry_count += 1 + time_to_wait = int(request.headers['x-ratelimit-towait-sec']) + logger.info('API Limit Exceeded waiting %d seconds. %d retries remaining' %(time_to_wait, max_retries-retry_count)) + time.sleep(time_to_wait) + + logger.info(str(url), str(data))# self.auth, headers, self.proxies) + if http_method == 'get': + # GET + logger.debug('GET request.') + request = self.session.get(url, params=data, auth=self.auth, headers=headers, proxies=self.proxies) + else: + # POST + logger.debug('POST request.') + # Make POST request. + request = self.session.post(url, data=data, auth=self.auth, headers=headers, proxies=self.proxies) + logger.debug('response headers =\n%s' % (str(request.headers))) + response = request.text + if retry_count >= max_retries: + break + + elif self.rate_limit_remaining[api_call] > rate_warn_threshold: logger.debug( "rate limit for api_call, %s = %s", api_call, From 09996858f0de97f3cc1aa4a12cda39d3c9ff8c1a Mon Sep 17 00:00:00 2001 From: MarianG Date: Tue, 26 May 2020 09:17:32 +0200 Subject: [PATCH 2/2] linting --- qualysapi/api_actions.py | 98 ++++++++++++++++++++++------------------ qualysapi/connector.py | 84 +++++++++++++++++++++++++--------- 2 files changed, 116 insertions(+), 66 deletions(-) diff --git a/qualysapi/api_actions.py b/qualysapi/api_actions.py index c16ad94..22ab7c4 100644 --- a/qualysapi/api_actions.py +++ b/qualysapi/api_actions.py @@ -23,31 +23,31 @@ def getHost(self, host): hostData.find("TRACKING_METHOD"), ) - def listHosts( self, - ips=None, - tags=None, - os_pattern=None, - tag_set_exclude=None, - id_min=None, - detailed=False, - echo_request=None, - limit=100): + ips=None, + tags=None, + os_pattern=None, + tag_set_exclude=None, + id_min=None, + detailed=False, + echo_request=None, + limit=100, + ): call = "/api/2.0/fo/asset/host/" parameters = {"action": "list", "truncation_limit": str(limit)} if detailed: - parameters["details"] ="All/AGs" + parameters["details"] = "All/AGs" if tag_set_exclude: - parameters["tag_set_exclude"]=tags + parameters["tag_set_exclude"] = tags if id_min: - parameters["id_min"] =str(id_min) + parameters["id_min"] = str(id_min) if ips: - parameters["ips"]=str(ips) + parameters["ips"] = str(ips) if tags: - parameters["use_tags"] ="1" - parameters["tag_set_by"]="name" + parameters["use_tags"] = "1" + parameters["tag_set_by"] = "name" parameters["tag_set_include"] = tags parameters["show_tags"] = "1" if os_pattern: @@ -55,24 +55,23 @@ def listHosts( if echo_request: parameters["echo_request"] = echo_request - hostData = objectify.fromstring(self.request(call, parameters).encode('utf-8')) + hostData = objectify.fromstring(self.request(call, parameters).encode("utf-8")) hostArray = [] for host in hostData.RESPONSE.HOST_LIST.HOST: hostArray.append( Host( - host.find('DNS'), - host.find('ID'), - host.find('IP'), - host.find('LAST_VULN_SCAN_DATETIME'), - host.find('NETBIOS'), - host.find('OS'), - host.find('TRACKING_METHOD') + host.find("DNS"), + host.find("ID"), + host.find("IP"), + host.find("LAST_VULN_SCAN_DATETIME"), + host.find("NETBIOS"), + host.find("OS"), + host.find("TRACKING_METHOD"), ) ) return hostArray - def getHostRange(self, start, end): call = "/api/2.0/fo/asset/host/" parameters = {"action": "list", "ips": f"{start}-{end}"} @@ -200,13 +199,13 @@ def listReports(self, id=0): self.request(call, parameters).encode("utf-8") ).RESPONSE reportsArray = [] - while repData.find("REPORT_LIST") is None and max_retries > 0: + while repData.find("REPORT_LIST") is None and max_retries > 0: max_retries = max_retries - 1 time.sleep(30) - qualys_resp = self.request(call, parameters).encode('utf-8') - logging.info("QUALYS_REPONSE "+ str(qualys_resp)) + qualys_resp = self.request(call, parameters).encode("utf-8") + logging.info("QUALYS_REPONSE " + str(qualys_resp)) repData = objectify.fromstring(qualys_resp).RESPONSE - + if max_retries <= 0: logging.info("Report Listing not successful") return None @@ -230,17 +229,16 @@ def listReports(self, id=0): else: parameters = {"action": "list", "id": id} - qualys_resp = self.request(call, parameters).encode('utf-8') - + qualys_resp = self.request(call, parameters).encode("utf-8") + repData_debug = objectify.fromstring(qualys_resp).RESPONSE while repData_debug.find("REPORT_LIST") is None and max_retries > 0: max_retries = max_retries - 1 time.sleep(30) - qualys_resp = self.request(call, parameters).encode('utf-8') - logging.info("QUALYS_REPONSE "+ str(qualys_resp)) + qualys_resp = self.request(call, parameters).encode("utf-8") + logging.info("QUALYS_REPONSE " + str(qualys_resp)) repData_debug = objectify.fromstring(qualys_resp).RESPONSE - - + if max_retries <= 0: logging.info("Report Listing not successful") return None @@ -442,27 +440,37 @@ def listScans(self, launched_after="", state="", target="", type="", user_login= def listChildTags(self, tag_name=None, tag_id=None, filename=None): if tag_id: - files =""" + files = ( + """ -"""+tag_id+""" +""" + + tag_id + + """ """ + ) elif filename: - files = open(filename,"rb").read() + files = open(filename, "rb").read() elif tag_name: - files =(""" + files = ( + """ -"""+tag_name+""" +""" + + tag_name + + """ -""").encode('ascii', 'ignore') +""" + ).encode("ascii", "ignore") call = "/qps/rest/2.0/search/am/tag" - parameters = files - response = objectify.fromstring(self.request(call, parameters, api_version=2, http_method="post").encode("utf-8")) - childs = list() + parameters = files + response = objectify.fromstring( + self.request(call, parameters, api_version=2, http_method="post").encode("utf-8") + ) + childs = list() for child in response.getchildren()[3][0].Tag.children.list.getchildren(): - childs.append(child.getchildren()) - + childs.append(child.getchildren()) + return childs def launchScan(self, title, option_title, iscanner_name, asset_groups="", ip=""): diff --git a/qualysapi/connector.py b/qualysapi/connector.py index 71f86a2..ef49cf0 100644 --- a/qualysapi/connector.py +++ b/qualysapi/connector.py @@ -408,49 +408,91 @@ def request( self.rate_limit_remaining[api_call], ) response = request.text - if ('1960' in response and 'This API cannot be run again until' in response): + if ( + "1960" in response + and "This API cannot be run again until" in response + ): max_retries = 10 retry_count = 0 - while ('1960' in response and 'This API cannot be run again until' in response): + while ( + "1960" in response + and "This API cannot be run again until" in response + ): retry_count += 1 time_to_wait = int(30) - logger.info('Concurrency Limit Exceeded waiting %d seconds. %d retries remaining' %(time_to_wait, max_retries-retry_count)) + logger.info( + "Concurrency Limit Exceeded waiting %d seconds. %d retries remaining" + % (time_to_wait, max_retries - retry_count) + ) time.sleep(time_to_wait) - logger.info(str(url), str(data))# self.auth, headers, self.proxies) - if http_method == 'get': + logger.info(str(url), str(data)) # self.auth, headers, self.proxies) + if http_method == "get": # GET - logger.debug('GET request.') - request = self.session.get(url, params=data, auth=self.auth, headers=headers, proxies=self.proxies) + logger.debug("GET request.") + request = self.session.get( + url, + params=data, + auth=self.auth, + headers=headers, + proxies=self.proxies, + ) else: # POST - logger.debug('POST request.') + logger.debug("POST request.") # Make POST request. - request = self.session.post(url, data=data, auth=self.auth, headers=headers, proxies=self.proxies) - logger.debug('response headers =\n%s' % (str(request.headers))) + request = self.session.post( + url, + data=data, + auth=self.auth, + headers=headers, + proxies=self.proxies, + ) + logger.debug("response headers =\n%s" % (str(request.headers))) response = request.text if retry_count >= max_retries: break - elif ('1965' in response and 'This API cannot be run again for another' in response): + elif ( + "1965" in response + and "This API cannot be run again for another" in response + ): max_retries = 10 retry_count = 0 - while ('1965' in response and 'This API cannot be run again for another' in response): + while ( + "1965" in response + and "This API cannot be run again for another" in response + ): retry_count += 1 - time_to_wait = int(request.headers['x-ratelimit-towait-sec']) - logger.info('API Limit Exceeded waiting %d seconds. %d retries remaining' %(time_to_wait, max_retries-retry_count)) + time_to_wait = int(request.headers["x-ratelimit-towait-sec"]) + logger.info( + "API Limit Exceeded waiting %d seconds. %d retries remaining" + % (time_to_wait, max_retries - retry_count) + ) time.sleep(time_to_wait) - logger.info(str(url), str(data))# self.auth, headers, self.proxies) - if http_method == 'get': + logger.info(str(url), str(data)) # self.auth, headers, self.proxies) + if http_method == "get": # GET - logger.debug('GET request.') - request = self.session.get(url, params=data, auth=self.auth, headers=headers, proxies=self.proxies) + logger.debug("GET request.") + request = self.session.get( + url, + params=data, + auth=self.auth, + headers=headers, + proxies=self.proxies, + ) else: # POST - logger.debug('POST request.') + logger.debug("POST request.") # Make POST request. - request = self.session.post(url, data=data, auth=self.auth, headers=headers, proxies=self.proxies) - logger.debug('response headers =\n%s' % (str(request.headers))) + request = self.session.post( + url, + data=data, + auth=self.auth, + headers=headers, + proxies=self.proxies, + ) + logger.debug("response headers =\n%s" % (str(request.headers))) response = request.text if retry_count >= max_retries: break