Merge branch 'feature/multipart-download' of github.com:minvws/nl-kat…

…-boefjes into feature/multipart-download

.github/workflows/tests.yml

@@ -8,11 +8,16 @@ on:
   push:
     branches:
       - 'develop'
-      - 'master'
+      - 'main'
 
 jobs:
   Tests:
 
+    strategy:
+      fail-fast: false
+      matrix:
+        version: [ '3.8', '3.9', '3.10', '3.11' ]
+
     runs-on: ubuntu-20.04
     env:
       COMPOSE_FILE: .ci/docker-compose.yml
@@ -25,11 +30,11 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: ${{ matrix.version }}
           cache: 'pip' # caching pip dependencies
 
       - name: Install pip
-        run: python3.8 -m pip install --upgrade pip
+        run: python3 -m pip install --upgrade pip
 
       - name: Install dev requirements
         run: pip install -r requirements-dev.txt
@@ -38,4 +43,4 @@ jobs:
         run: find . -name requirements.txt | xargs -L 1 pip install -r
 
       - name: Run pytests
-        run: python3.8 -m pytest
+        run: python3 -m pytest

boefjes/plugin_repository/requirements.txt

@@ -1,4 +1,4 @@
-pydantic==1.8.2
+pydantic==1.10.4
 requests==2.28.1
 fastapi==0.86.0
 uvicorn==0.19.0

boefjes/plugins/kat_censys/main.py

@@ -1,16 +1,17 @@
 import json
-from typing import Tuple, Union
+from typing import Tuple, Union, List
 
 from censys.search import CensysHosts
 
 from boefjes.job_models import BoefjeMeta
 
 
-def run(boefje_meta: BoefjeMeta) -> Tuple[BoefjeMeta, Union[bytes, str]]:
+def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
 
     h = CensysHosts()
     input_ = boefje_meta.arguments["input"]
     ip = input_["address"]
     host = h.view(ip)
 
-    return boefje_meta, json.dumps(host)
+    # return boefje_meta, json.dumps(host)
+    return [(set(), json.dumps(host))]

boefjes/plugins/kat_censys/normalize.py

@@ -25,7 +25,7 @@
 
 def run(normalizer_meta: NormalizerMeta, raw: Union[bytes, str]) -> Iterator[OOI]:
     results = json.loads(raw)
-    ip_ooi_reference = Reference.from_str(normalizer_meta.boefje_meta.input_ooi)
+    ip_ooi_reference = Reference.from_str(normalizer_meta.raw_data.boefje_meta.input_ooi)
 
     network_reference = Network(name=ip_ooi_reference.tokenized.network.name).reference
     ip = results["ip"]
@@ -73,8 +73,7 @@ def run(normalizer_meta: NormalizerMeta, raw: Union[bytes, str]) -> Iterator[OOI
                 valid_until=0,
                 pk_algorithm=certificate["leaf_data"]["pubkey_algorithm"],
                 pk_size=certificate["leaf_data"]["pubkey_bit_size"],
-                pk_number=certificate["leaf_data"]["fingerprint"],
-                website="",  # todo: should be fixed in the Certificate model
+                serial_number=certificate["leaf_data"]["fingerprint"],
                 signed_by=None,
             )
 

boefjes/plugins/kat_crt_sh/main.py

@@ -48,7 +48,7 @@ def request_certs(search_string, search_type="Identity", match="=", deduplicate=
     if deduplicate:
         query["deduplicate"] = "Y"
 
-    response = requests.get(CRT_SH_API, query)
+    response = requests.get(CRT_SH_API, params=query)
     if response.status_code != 200:
         response.raise_for_status()
     if json_output:
@@ -59,7 +59,7 @@ def request_certs(search_string, search_type="Identity", match="=", deduplicate=
 def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
     input_ = boefje_meta.arguments["input"]
     fqdn = input_["hostname"]["name"]
-    domain = fqdn if not fqdn.endswith(".") else fqdn[:-1]
+    domain = fqdn.rstrip(".")
     results = request_certs(domain)
 
     return [(set(), results)]

boefjes/plugins/kat_crt_sh/normalize.py

@@ -1,6 +1,7 @@
+import datetime
 import json
 from typing import Iterator, Union
-
+from dateutil.parser import parse
 from octopoes.models import OOI
 from octopoes.models.ooi.certificate import Certificate
 from octopoes.models.ooi.dns.zone import Hostname
@@ -13,42 +14,39 @@ def run(normalizer_meta: NormalizerMeta, raw: Union[bytes, str]) -> Iterator[OOI
     results = json.loads(raw)
     input_ = normalizer_meta.raw_data.boefje_meta.arguments["input"]
     fqdn = input_["hostname"]["name"]
-    current = f".{fqdn.lower()}"
-    if current.endswith("."):
-        current = current[:-1]
+    current = fqdn.lstrip(".").rstrip(".")
 
-    internet = Network(name="internet")
-    yield internet
+    network = Network(name="internet")
+    yield network
+    network_reference = network.reference
 
-    unique_domains = {}
+    unique_domains = set()
     for certificate in results:
-        common_name = certificate["common_name"].lower()
+        common_name = certificate["common_name"].lower().lstrip(".*")
 
         # walk over all name_value parts (possibly just one, possibly more)
         names = certificate["name_value"].lower().splitlines()
         for name in names:
             if not name.endswith(current):
                 # todo: do we want to hint other unrelated hostnames using the same certificate / and this possibly
                 #  the same private keys for tls?
-                break
+                pass
             if name not in unique_domains:
-                unique_domains[name] = True
-                yield Hostname(name=name, network=internet.reference)
+                yield Hostname(name=name, network=network_reference)
+                unique_domains.add(name)
 
         # todo: yield only current certs?
         yield Certificate(
             subject=common_name,
             issuer=certificate["issuer_name"],
             valid_from=certificate["not_before"],
             valid_until=certificate["not_after"],
-            pk_algorithm="",
-            pk_size=0,  # todo: fix
-            pk_number=certificate["serial_number"].upper(),
-            website=None,  # This should be a hostname, not website.
-            signed_by=None,
+            serial_number=certificate["serial_number"].upper(),
+            expires_in=parse(certificate["not_after"]).astimezone(datetime.timezone.utc)
+            - datetime.datetime.now(datetime.timezone.utc),
         )
         # walk over the common_name. which might be unrelated to the requested domain, or it might be a parent domain
         # which our dns Boefje should also have picked up.
         # wilcards also trigger here, and wont be visible from a DNS query
-        if common_name.endswith(current) and common_name not in unique_domains:
-            yield Hostname(name=common_name, network=internet.reference)
+        if common_name.endswith(current) or common_name not in unique_domains:
+            yield Hostname(name=common_name, network=network_reference)

boefjes/plugins/kat_leakix/main.py

@@ -31,7 +31,7 @@ def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
                 headers={"Accept": "application/json", "api-key": getenv("LEAKIX_API")},
             )
             page_counter += 1
-            if not response:
+            if not response or not response.content:
                 break
             response_json = response.json()
             if not response_json:

boefjes/plugins/kat_nmap/boefje.json

@@ -1,7 +1,7 @@
     {
-        "id": "nmap-tcp-top250",
-        "name": "Nmap TCP (Top 250)",
-        "description": "Scan top 250 TCP ports. Including service detection",
+        "id": "nmap",
+        "name": "Nmap",
+        "description": "Defaults to top 250 TCP ports. Includes service detection.",
         "consumes": [
             "IPAddressV4",
             "IPAddressV6"
@@ -13,6 +13,9 @@
             "IPAddressV4",
             "IPService"
         ],
-        "environment_keys": [],
+        "environment_keys": [
+            "TOP_PORTS",
+            "PROTOCOL"
+        ],
         "scan_level": 2
     }

boefjes/plugins/kat_nmap/description.md

@@ -2,16 +2,22 @@
 
 Nmap is a network scanning tool that uses IP packets to identify all the devices connected to a network and to provide
 information on the services and operating systems they are running. In KAT, a Python wrapper around Nmap is used to find
-open ports with their services of an IpAddress.
+open ports with their services of an IpAddress. Nmap itself runs in a temporary Docker container.
+
 ### Options
 
 This Nmap has the following hardcoded options:
 
-For TCP ports:
-`"-T4", "-Pn", "-r", "-v10", "-sV", "-sS", "-p-"`
+| Option | Function |
+| ----------- | ----------- |
+| `T4` | assume a fast and reliable network |
+| `Pn` | skips host discovery, treats hosts as online |
+|`-r` | scan ports in order |
+|`-v10` |use verbosity level 10 |
+|`-sV` |probe open ports to determine version info |
+|`-oX` |Output in XML |
 
-For UDP ports:
-`"-T4", "-Pn", "-r", "-v10", "-sV", "-sU"`
+A TCP scan uses `-sS` (TCP SYN), a UDP scan uses `-sU`. TOP_PORTS defaults to `250`.
 
 ### Input OOIs
 
@@ -28,6 +34,7 @@ Nmap outputs the following OOIs:
 |IpService|IpService that couples a service to an open port|
 |Finding|Finding if ports are open that should not be open (TEMP!)|
 |KatFindingType|FindingType if ports are open that should not be open (TEMP!)|
+
 ### Running Boefje
 
 ```json
@@ -51,8 +58,7 @@ Nmap outputs the following OOIs:
 ```
 boefjes/tools/kat_nmap
 ├── normalize.py
-├── scan.py
-└── requirements.txt
+├── main.py
 ```
 
-**Cat name**: Elsje
+**Cat name**: Elsje

boefjes/plugins/kat_nmap/main.py

@@ -1,3 +1,4 @@
+from os import getenv
 from enum import Enum
 from ipaddress import ip_address, IPv6Address
 from typing import List, Tuple, Union, Optional
@@ -7,9 +8,13 @@
 from boefjes.job_models import BoefjeMeta
 
 NMAP_IMAGE = "instrumentisto/nmap:latest"
+TOP_PORTS_MAX = 65535
+TOP_PORTS_DEFAULT = 250
+TOP_PORTS_MIN = 1
 
 
 def run_nmap(args: List[str]) -> str:
+    """Run Nmap in Docker."""
     client = docker.from_env()
     return client.containers.run(NMAP_IMAGE, args, remove=True).decode()
 
@@ -19,19 +24,22 @@ class Protocol(Enum):
     UDP = "udp"
 
 
-def build_nmap_arguments(host: str, protocol: Protocol, top_ports: Optional[int]):
+def build_nmap_arguments(host: str, protocol: Protocol, top_ports: Optional[int]) -> List[str]:
+    """Returns Nmap arguments to use based on protocol and top_ports for host."""
     ip = ip_address(host)
-    args = ["nmap"]
-
-    if protocol == Protocol.TCP:
-        args.extend(["-T4", "-Pn", "-r", "-v10", "-sV", "-sS"])
+    args = [
+        "nmap",
+        "-T4",
+        "-Pn",
+        "-r",
+        "-v10",
+        "-sV",
+        "-sS" if protocol == Protocol.TCP else "-sU",
+    ]
+    if top_ports is None:
+        args.append("-p-")
     else:
-        args.extend(["-T4", "-Pn", "-r", "-v10", "-sV", "-sU"])
-
-    if top_ports is not None:
         args.extend(["--top-ports", str(top_ports)])
-    elif protocol == Protocol.TCP:
-        args.append("-p-")
 
     if isinstance(ip, IPv6Address):
         args.append("-6")
@@ -42,13 +50,22 @@ def build_nmap_arguments(host: str, protocol: Protocol, top_ports: Optional[int]
 
 
 def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
-    input_ = boefje_meta.arguments["input"]
-    host = input_["address"]
-
-    # protocol = boefje_meta.arguments["protocol"]
-    # top_ports = boefje_meta.arguments.get("top_ports", None)
-    top_ports = 250
-    protocol = "tcp"
-    args = build_nmap_arguments(host, Protocol(protocol), top_ports)
-
-    return [(set(), run_nmap(args))]
+    """Build Nmap arguments and return results to normalizer."""
+    top_ports = int(getenv("TOP_PORTS", TOP_PORTS_DEFAULT))
+    assert (
+        TOP_PORTS_MIN <= top_ports <= TOP_PORTS_MAX
+    ), f'{TOP_PORTS_MIN} <= {top_ports} <= {TOP_PORTS_MAX} fails. Check "TOP_PORTS" argument.'
+
+    protocol = getenv("PROTOCOL", "tcp").lower()
+    assert protocol in ["tcp", "udp"], f'Invalid protocol "{protocol}".'
+
+    return [
+        (
+            set(),
+            run_nmap(
+                args=build_nmap_arguments(
+                    host=boefje_meta.arguments["input"]["address"], protocol=Protocol(protocol), top_ports=top_ports
+                )
+            ),
+        )
+    ]