[OpenKAT v1.6] Feature/multipart download

boefjes/katalogus/local_repository.py

@@ -151,7 +151,6 @@ def _normalizer_to_plugin(normalizer: NormalizerResource) -> Normalizer:
         def_file = normalizer.path / "normalizer.json"
         def_obj = json.loads(def_file.read_text())
         def_obj["repository_id"] = LocalPluginRepository.RESERVED_ID
-        def_obj["produces"] = []
 
         return Normalizer.parse_obj(def_obj)
 

boefjes/local.py

@@ -26,7 +26,7 @@ class LocalBoefjeJobRunner(BoefjeJobRunner):
     def __init__(self, local_repository: LocalPluginRepository):
         self.local_repository = local_repository
 
-    def run(self, boefje_meta: BoefjeMeta, environment: Dict[str, str]) -> Tuple[BoefjeMeta, Union[str, bytes]]:
+    def run(self, boefje_meta: BoefjeMeta, environment: Dict[str, str]) -> List[Tuple[set, Union[bytes, str]]]:
         logger.info("Running local boefje plugin")
 
         boefjes = self.local_repository.resolve_boefjes()

boefjes/plugins/kat_snyk/requirements.txt

@@ -1,2 +1,2 @@
-requests
-beautifulsoup4
+requests==2.28.1
+beautifulsoup4==4.11.1

boefjes/plugins/kat_webpage_analysis/check_images/normalize.py

@@ -0,0 +1,59 @@
+from typing import Union, Iterator
+from octopoes.models import OOI, Reference
+
+from boefjes.job_models import NormalizerMeta
+from octopoes.models.ooi.findings import KATFindingType, Finding
+from octopoes.models.ooi.web import ImageMetadata
+
+from io import BytesIO
+
+from PIL import Image, UnidentifiedImageError
+from PIL.ExifTags import TAGS
+
+
+def run(normalizer_meta: NormalizerMeta, raw: Union[bytes, str]) -> Iterator[OOI]:
+    # fetch a reference to the original resource where these headers where downloaded from
+    resource = Reference.from_str(normalizer_meta.raw_data.boefje_meta.input_ooi)
+    image = Image.open(BytesIO(raw))
+    image.MAX_IMAGE_PIXELS = 7680 * 4320  # 8K pixels for now
+
+    try:
+        image_info = {
+            "size": image.size,
+            "height": image.height,
+            "width": image.width,
+            "format": image.format,
+            "mode": image.mode,
+            "is_animated": getattr(image, "is_animated", False),
+            "frames": getattr(image, "n_frames", 1),
+        }
+        exif_data = image.getexif()
+
+        for tag_id in exif_data:
+            # human readable tag name
+            tag = TAGS.get(tag_id, tag_id)
+            tag_data = exif_data.get(tag_id)
+
+            if isinstance(tag_data, bytes):
+                tag_data = tag_data.decode()
+
+            image_info[tag] = tag_data
+
+        yield ImageMetadata(resource=resource, image_info=image_info)
+    except UnidentifiedImageError:
+        kat_number = "BrokenImage"
+        kat_ooi = KATFindingType(id=kat_number)
+        yield Finding(
+            finding_type=kat_ooi.reference,
+            ooi=resource,
+            description="Image is not recognized, possibly served with broken mime-type.",
+        )
+
+    except Image.DecompressionBombWarning:
+        kat_number = "DecompressionBomb"
+        kat_ooi = KATFindingType(id=kat_number)
+        yield Finding(
+            finding_type=kat_ooi.reference,
+            ooi=resource,
+            description="Image ended up bigger than %d Pixels, possible decompression Bomb" % image.MAX_IMAGE_PIXELS,
+        )

boefjes/plugins/kat_webpage_analysis/check_images/normalizer.json

@@ -0,0 +1,9 @@
+{
+    "id": "kat_check_images",
+    "consumes": [
+        "image/jpeg", "image/jpg", "image/gif", "image/png", "image/bpm", "image/ico"
+    ],
+    "produces": [
+        "ImageMetadata"
+    ]
+}

boefjes/plugins/kat_webpage_analysis/check_images/requirements.txt

@@ -0,0 +1 @@
+Pillow==9.4.0

boefjes/plugins/kat_webpage_analysis/find_images_in_html/normalize.py

@@ -0,0 +1,32 @@
+from typing import Union, Iterator
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup
+
+import validators
+
+from octopoes.models import OOI
+from octopoes.models.ooi.network import Network
+from octopoes.models.ooi.web import URL
+
+from boefjes.job_models import NormalizerMeta
+
+
+def run(normalizer_meta: NormalizerMeta, raw: Union[bytes, str]) -> Iterator[OOI]:
+    soup = BeautifulSoup(raw, "html.parser")
+    images = set([img["src"] for img in soup.find_all("img") if hasattr(img, "src")])
+
+    network_name = normalizer_meta.raw_data.boefje_meta.arguments["input"]["website"]["hostname"]["network"]["name"]
+    host = normalizer_meta.raw_data.boefje_meta.arguments["input"]["website"]["hostname"]["name"]
+    service = normalizer_meta.raw_data.boefje_meta.arguments["input"]["website"]["ip_service"]["service"]["name"]
+
+    url = f"{service}://{host}/"
+
+    for img in images:
+        if not validators.url(img):
+            img = urljoin(url, img)
+
+        yield URL(
+            network=Network(name=network_name).reference,
+            raw=img,
+        )

boefjes/plugins/kat_webpage_analysis/find_images_in_html/normalizer.json

@@ -0,0 +1,9 @@
+{
+    "id": "kat_find_images_in_html",
+    "consumes": [
+        "text/html"
+    ],
+    "produces": [
+        "HTTPResource"
+    ]
+}

boefjes/plugins/kat_webpage_analysis/find_images_in_html/requirements.txt

@@ -0,0 +1,2 @@
+beautifulsoup4==4.11.1
+validators==0.20.0

boefjes/plugins/kat_webpage_analysis/headers/normalize.py

@@ -0,0 +1,18 @@
+import json
+from typing import Union, Iterator
+from octopoes.models import OOI, Reference
+from octopoes.models.ooi.web import HTTPHeader
+
+from boefjes.job_models import NormalizerMeta
+
+
+def run(normalizer_meta: NormalizerMeta, raw: Union[bytes, str]) -> Iterator[OOI]:
+    # fetch a reference to the original resource where these headers where downloaded from
+    resource = Reference.from_str(normalizer_meta.raw_data.boefje_meta.input_ooi)
+
+    for key, value in json.loads(raw).items():
+        yield HTTPHeader(
+            resource=resource,
+            key=key,
+            value=value,
+        )

boefjes/plugins/kat_webpage_analysis/normalizer.json → boefjes/plugins/kat_webpage_analysis/headers/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_webpage_analysis_headers_normalize",
     "consumes": [
-        "webpage-analysis"
+        "openkat-http/headers"
     ],
     "produces": [
         "HTTPHeader"

boefjes/plugins/kat_webpage_analysis/main.py

@@ -2,26 +2,22 @@
 from typing import Tuple, Union, List
 from boefjes.job_models import BoefjeMeta
 
+from os import getenv
 import requests
+from requests import Session
+
 from urllib.parse import urlparse, urlunsplit
 from forcediphttpsadapter.adapters import ForcedIPHTTPSAdapter
 
 
 def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
     input_ = boefje_meta.arguments["input"]
+    useragent = getenv("useragent", default="OpenKAT")
 
-    port = f":{input_['web_url']['port']}"
-    netloc = (
-        input_["web_url"]["netloc"]["address"]
-        if "address" in input_["web_url"]["netloc"]
-        else input_["web_url"]["netloc"]["name"]
-    )
-
-    url = f"{input_['web_url']['scheme']}://{netloc}{port}{input_['web_url']['path']}"
+    uri = get_uri(input_)
     ip = input_["website"]["ip_service"]["ip_port"]["address"]["address"]
 
     # Code from https://github.com/Roadmaster/forcediphttpsadapter/blob/master/example.py
-    uri = url
     url_parts = urlparse(uri)
     hostname = url_parts.netloc
     session = requests.Session()
@@ -31,8 +27,7 @@ def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
         base_url = urlunsplit((url_parts.scheme, url_parts.netloc, "", "", ""))
         session.mount(base_url, ForcedIPHTTPSAdapter(dest_ip=ip))
     else:
-        # Fall back to old hack-ip-into-url behavior, for either
-        # https with no adapter, or http.
+        # Fall back to old hack-ip-into-url behavior, for either https with no adapter, or http.
         if ip:
             url_parts = url_parts._replace(netloc=ip)
             uri = urlunsplit(
@@ -45,21 +40,45 @@ def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
                 ]
             )
 
+    body_mimetypes = {"openkat-http/body"}
     try:
-        response = session.get(
-            uri,
-            headers={"Host": hostname, "Accept": "application/json"},
-            verify=False,
-            allow_redirects=False,
-        )
-        result = {
-            # "content": response.content,
-            "cookies": response.cookies.get_dict(),
-            "headers": dict(response.headers),
-            "code": response.status_code,
-            "text": response.text,
-        }
-    except requests.exceptions.ConnectionError:
-        result = {}
-
-    return [(set(), json.dumps(result))]
+        response = do_request(hostname, session, uri, useragent)
+    except requests.exceptions.RequestException as request_error:
+        return [({"openkat-http/error"}, str(request_error))]
+
+    if "content-type" in response.headers:
+        content_type = response.headers.get("content-type")
+        body_mimetypes.add(content_type)
+
+        # Pick up the content type for the body from the server and split away encodings to make normalization easier
+        content_type = content_type.split(";")
+        body_mimetypes.add(content_type[0])
+
+    return [
+        ({"openkat-http/full"}, f"{response.headers}\n\n{response.content}"),
+        ({"openkat-http/headers"}, json.dumps(dict(response.headers))),
+        (body_mimetypes, response.content),
+    ]
+
+
+def do_request(hostname: str, session: Session, uri: str, useragent: str):
+    response = session.get(
+        uri,
+        headers={"Host": hostname, "User-Agent": useragent},
+        verify=False,
+        allow_redirects=False,
+    )
+
+    return response
+
+
+def get_uri(input_: dict) -> str:
+    port = f":{input_['web_url']['port']}"
+    netloc = (
+        input_["web_url"]["netloc"]["address"]
+        if "address" in input_["web_url"]["netloc"]
+        else input_["web_url"]["netloc"]["name"]
+    )
+    uri = f"{input_['web_url']['scheme']}://{netloc}{port}{input_['web_url']['path']}"
+
+    return uri

tests/__init__.py

@@ -0,0 +1,4 @@
+import os
+from pathlib import Path
+
+os.chdir(Path(__file__).parent.parent)

tests/examples/body-normalize.json

@@ -0,0 +1,65 @@
+{
+  "id": "dns-lookup-job",
+  "raw_data": {
+    "id": "webpage-analysis-job",
+    "boefje_meta": {
+      "id": "webpage-analysis-job",
+      "boefje": {
+        "id": "webpage-analysis"
+      },
+      "organization": "_dev",
+      "input_ooi": "HTTPResource|internet|134.209.85.72|tcp|443|https|internet|mispo.es|https|internet|mispo.es|443|/",
+      "arguments": {
+        "input": {
+          "object_type": "HTTPResource",
+          "scan_profile": "reference=Reference('HTTPResource|internet|134.209.85.72|tcp|443|https|internet|mispo.es|https|internet|mispo.es|443|/') level=4 scan_profile_type='inherited'",
+          "primary_key": "HTTPResource|internet|134.209.85.72|tcp|443|https|internet|mispo.es|https|internet|mispo.es|443|/",
+          "website": {
+            "ip_service": {
+              "ip_port": {
+                "address": {
+                  "network": {
+                    "name": "internet"
+                  },
+                  "address": "134.209.85.72"
+                },
+                "protocol": "tcp",
+                "port": "443"
+              },
+              "service": {
+                "name": "https"
+              }
+            },
+            "hostname": {
+              "network": {
+                "name": "internet"
+              },
+              "name": "mispo.es"
+            }
+          },
+          "web_url": {
+            "scheme": "https",
+            "netloc": {
+              "network": {
+                "name": "internet"
+              },
+              "name": "mispo.es"
+            },
+            "port": "443",
+            "path": "/"
+          },
+          "redirects_to": "None"
+        }
+      }
+    },
+    "mime_types": [
+      {
+        "value": "boefje/dns-records"
+      }
+    ]
+  },
+  "normalizer": {
+    "id": "kat_find_images_in_html",
+    "version": null
+  }
+}