Version 2.0.1 (#70)

* Add 404, 403 error handling and warning to CTR_ENTITIES_LIMIT variable

* Add Jenkinsfile

* Fix autotest

* Updated test data into test_judgement.py and test_verdict.py files.

* Alpine & Python version update

* Update tips

Author: ytsek

Dockerfile

@@ -1,8 +1,10 @@
-FROM alpine:3.13
+FROM alpine:3.14
 LABEL maintainer="Ian Redden <[email protected]>"
 
 # install packages we need
-RUN apk update && apk add --no-cache musl-dev openssl-dev gcc python3 py3-configobj python3-dev supervisor git libffi-dev uwsgi-python3 uwsgi-http jq nano syslog-ng uwsgi-syslog py3-pip
+RUN apk update && apk add --no-cache musl-dev openssl-dev gcc py3-configobj \
+supervisor git libffi-dev uwsgi-python3 uwsgi-http jq syslog-ng uwsgi-syslog \
+py3-pip python3-dev
 
 # do the Python dependencies
 ADD code /app

Jenkinsfile

@@ -0,0 +1,3 @@
+@Library('softserve-jenkins-library@main') _
+
+startPipeline()

README.md

@@ -72,6 +72,8 @@ curl http://localhost:9090
 
 ## Implementation Details
 
+This application was developed and tested under Python version 3.9.
+
 ### Implemented Relay Endpoints
 
 - `POST /health`

code/api/utils.py

@@ -1,10 +1,17 @@
-import requests
-import jwt
 import json
-from jwt import InvalidSignatureError, DecodeError, InvalidAudienceError
-from flask import request, current_app, jsonify, g
-from requests.exceptions import SSLError, ConnectionError, InvalidURL
 from http import HTTPStatus
+from json.decoder import JSONDecodeError
+
+import jwt
+import requests
+from flask import request, current_app, jsonify, g
+from jwt import InvalidSignatureError, DecodeError, InvalidAudienceError
+from requests.exceptions import (
+    SSLError,
+    ConnectionError,
+    InvalidURL,
+    HTTPError
+)
 
 from api.errors import (
     AuthorizationError,
@@ -14,6 +21,20 @@
     UnexpectedPulsediveError
 )
 
+NO_AUTH_HEADER = 'Authorization header is missing'
+WRONG_AUTH_TYPE = 'Wrong authorization type'
+WRONG_PAYLOAD_STRUCTURE = 'Wrong JWT payload structure'
+WRONG_JWT_STRUCTURE = 'Wrong JWT structure'
+WRONG_AUDIENCE = 'Wrong configuration-token-audience'
+KID_NOT_FOUND = 'kid from JWT header not found in API response'
+WRONG_KEY = ('Failed to decode JWT with provided key. '
+             'Make sure domain in custom_jwks_host '
+             'corresponds to your SecureX instance region.')
+JWKS_HOST_MISSING = ('jwks_host is missing in JWT payload. Make sure '
+                     'custom_jwks_host field is present in module_type')
+WRONG_JWKS_HOST = ('Wrong jwks_host in JWT payload. Make sure domain follows '
+                   'the visibility.<region>.cisco.com structure')
+
 
 def set_ctr_entities_limit(payload):
     try:
@@ -25,16 +46,15 @@ def set_ctr_entities_limit(payload):
 
 
 def get_public_key(jwks_host, token):
-    expected_errors = {
-        ConnectionError: 'Wrong jwks_host in JWT payload. '
-                         'Make sure domain follows the '
-                         'visibility.<region>.cisco.com structure',
-        InvalidURL: 'Wrong jwks_host in JWT payload. '
-                    'Make sure domain follows the '
-                    'visibility.<region>.cisco.com structure',
-    }
+    expected_errors = (
+        ConnectionError,
+        InvalidURL,
+        JSONDecodeError,
+        HTTPError,
+    )
     try:
         response = requests.get(f"https://{jwks_host}/.well-known/jwks")
+        response.raise_for_status()
         jwks = response.json()
 
         public_keys = {}
@@ -45,9 +65,8 @@ def get_public_key(jwks_host, token):
             )
         kid = jwt.get_unverified_header(token)['kid']
         return public_keys.get(kid)
-    except tuple(expected_errors) as error:
-        message = expected_errors[error.__class__]
-        raise AuthorizationError(message)
+    except expected_errors:
+        raise AuthorizationError(WRONG_JWKS_HOST)
 
 
 def get_jwt():
@@ -57,24 +76,19 @@ def get_jwt():
     """
 
     expected_errors = {
-        KeyError: 'Wrong JWT payload structure',
-        AssertionError: 'jwks_host is missing in JWT payload. '
-                        'Make sure custom_jwks_host field '
-                        'is present in module_type',
-        InvalidSignatureError: 'Failed to decode JWT with provided key. '
-                               'Make sure domain in custom_jwks_host '
-                               'corresponds to your SecureX instance region.',
-        DecodeError: 'Wrong JWT structure',
-        InvalidAudienceError: 'Wrong configuration-token-audience',
-        TypeError: 'kid from JWT header not found in API response'
+        KeyError: WRONG_PAYLOAD_STRUCTURE,
+        AssertionError: JWKS_HOST_MISSING,
+        InvalidSignatureError: WRONG_KEY,
+        DecodeError: WRONG_JWT_STRUCTURE,
+        InvalidAudienceError: WRONG_AUDIENCE,
+        TypeError: KID_NOT_FOUND,
     }
 
     token = get_auth_token()
     try:
-        jwks_host = jwt.decode(
-            token, options={'verify_signature': False}
-        ).get('jwks_host')
-        assert jwks_host
+        jwks_payload = jwt.decode(token, options={'verify_signature': False})
+        assert 'jwks_host' in jwks_payload
+        jwks_host = jwks_payload.get('jwks_host')
         key = get_public_key(jwks_host, token)
         aud = request.url_root
         payload = jwt.decode(
@@ -93,8 +107,8 @@ def get_auth_token():
     """
 
     expected_errors = {
-        KeyError: 'Authorization header is missing',
-        AssertionError: 'Wrong authorization type'
+        KeyError: NO_AUTH_HEADER,
+        AssertionError: WRONG_AUTH_TYPE
     }
 
     try:

code/container_settings.json

@@ -1,4 +1,4 @@
 {
-  "VERSION": "2.0.0",
+  "VERSION": "2.0.1",
   "NAME": "Pulsedive"
 }

code/requirements.txt

@@ -1,8 +1,8 @@
-Flask==1.1.2
-marshmallow==3.11.1
+Flask==2.0.1
+marshmallow==3.12.1
 requests==2.25.1
 cryptography==3.3.2
-pyjwt[crypto]==2.0.1
-flake8==3.9.0
-coverage==5.2.1
-pytest==6.2.2
+pyjwt[crypto]==2.1.0
+flake8==3.9.2
+coverage==5.5
+pytest==6.2.4

code/tests/functional/tests/test_judgement.py

@@ -10,8 +10,8 @@
 
 @pytest.mark.parametrize(
     'observable,observable_type,disposition_name,disposition',
-    (('50.63.202.50', 'ip', 'Clean', 1),
-     ('1.0.3.4', 'ip', 'Suspicious', 3),
+    (('110.174.93.63', 'ip', 'Clean', 1),
+     ('5.9.56.12', 'ip', 'Suspicious', 3),
      ('0-100-disc.foxypool.cf', 'domain', 'Malicious', 2),
      ('https://www.youtube.com/', 'url', 'Unknown', 5))
 )

code/tests/functional/tests/test_sightings.py

@@ -14,7 +14,7 @@
     (
      ('url', 'http://51jianli.cn/images'),
      ('ip', '1.1.1.1'),
-     ('domain', 'xcj10.me'),
+     ('domain', 'baidu.com'),
      )
 )
 def test_positive_sighting(module_headers, observable, observable_type):
@@ -65,7 +65,7 @@ def test_positive_sighting(module_headers, observable, observable_type):
         assert sighting['observables'] == observables
         if sighting['description'] == 'Active DNS':
             for relation in sighting['relations']:
-                assert relation['origin'] == f'{MODULE_NAME} Enrichment Module'
+                assert relation['origin'] == f'{SOURCE} Enrichment Module'
                 assert relation['relation'] == 'Resolved_To'
 
                 if observable_type == 'domain':

code/tests/functional/tests/test_verdict.py

@@ -6,9 +6,9 @@
 
 @pytest.mark.parametrize(
     'observable,observable_type,disposition_name,disposition',
-    (('50.63.202.50', 'ip', 'Clean', 1),
+    (('110.174.93.63', 'ip', 'Clean', 1),
      ('0-100-disc.foxypool.cf', 'domain', 'Malicious', 2),
-     ('1.0.3.4', 'ip', 'Suspicious', 3),
+     ('5.9.56.12', 'ip', 'Suspicious', 3),
      ('98.159.110.20', 'ip', 'Unknown', 5))
 )
 def test_positive_verdict(module_headers, observable, observable_type,

module_type.json.sample

@@ -3,7 +3,7 @@
     "default_name": "Pulsedive",
     "short_description": "Pulsedive threat intelligence enriches any domain, URL, or IP. Scan new indicators, pivot to search on any data point, and investigate threats.",
     "description": "Pulsedive threat intelligence offers a community platform that scans, enriches, and scores millions of indicators of compromise, setting the foundation for powerful threat investigation and research capabilities. Register for a free account and API key to access the intelligence sourced and contextualized from dozens of feeds and submitted by users all over the world.\n\nAdditionally, leverage Pulsedive to run passive or active scans on any indicator, investigate shared threat properties and attributes, and pull related threat news/summaries from the web.\n## Snapshot of Pulsedive's API\n\n### Retrieve Indicator Data\n- Risk scores and risk factors\n- Registration timeline\n- Source feeds and comments\n- Associated threats\n- Ports and protocols\n- Web technologies\n- WHOIS registration\n- Location data\n- DNS records\n- Query strings\n- HTTP headers\n- SSL certificate metadata\n- Cookies\n- Meta tags\n- Mail servers\n- Redirects\n- Related domains and urls\n\n### Retrieve Threat Data\n- Related news\n- Comments\n- Risky properties\n- Source feeds\n- Indicators\n\n### Retrieve Feed Data\n- Name and organization\n- Threats\n- Indicators\n\n### Explore (Search) Our Database\n- Create queries using almost any data point(s) to pivot across indicators, threats, or feeds.\n\n### Scan Indicators\n- Perform passive or active scans of hosts to retrieve live data on-demand.",
-    "tips": "When configuring this integration, you must first gather some information from your Pulsedive account.\n\n1. Log into Pulsedive, click **ACCOUNT**\n2. Copy the **API KEY** into a file, or leave the tab open\n\n3. Complete the **Add New Pulsedive Integration Module** form:\n    - **Module Name** - Leave the default name or enter a name that is meaningful to you\n    - **API Key** - Enter the Pulsedive API Key\n    - **Entities Limit** - Enter the limit that restricts the maximum number of CTIM entities of each type returned in a single response per each requested observable. Must be a positive integer. Defaults to 100 (if unset or incorrect)\n4. Click **Save** to complete the Pulsedive module configuration.",
+    "tips": "When configuring Pulsedive integration, you must obtain the API key from your Pulsedive account and then add the Pulsedive integration module in SecureX.\n\n1. Log in to Pulsedive and click **ACCOUNT**.\n2. Copy the **API KEY** into a file or leave the tab open.\n3. In SecureX, complete the **Add New Pulsedive Integration Module** form:\n    - **Integration Module Name** - Leave the default name or enter a name that is meaningful to you.\n    - **API Key** - Paste the copied API key from Pulsedive into this field.\n    - **Entities Limit** - Specify the maximum number of indicators and sightings in a single response, per requested observable (must be a positive value). We recommend that you enter a limit in the range of 50 to 1000. The default is 100 entities.\n\n4. Click **Save** to complete the Pulsedive integration module configuration.",
     "external_references": [
         {
             "label": "About",
@@ -26,7 +26,7 @@
             "key": "custom_CTR_ENTITIES_LIMIT",
             "type": "integer",
             "label": "Entities Limit",
-            "tooltip": "Restricts the maximum number of `Indicators` and `Sightings`",
+            "tooltip": "Restricts the maximum number of `Indicators` and `Sightings`. Please note that the number over 100 might lead to data inconsistency.",
             "required": false
         }
     ],