Merge branch 'main' into feature/multipart-download

boefjes/plugins/kat_nmap/normalize.py

@@ -1,6 +1,6 @@
+import logging
 from typing import Iterator, Union
-
-from libnmap.objects import NmapHost, NmapService, NmapReport
+from libnmap.objects import NmapHost, NmapService
 from libnmap.parser import NmapParser
 from octopoes.models import OOI
 from octopoes.models.ooi.network import (
@@ -16,48 +16,55 @@
 from boefjes.job_models import NormalizerMeta
 
 
-def get_ports_and_service(host: NmapHost) -> Iterator[OOI]:
-    internet = Network(name="internet")
-    yield internet
-
-    ip = (
-        IPAddressV4(network=internet.reference, address=host.address)
-        if host.ipv4
-        else IPAddressV6(network=internet.reference, address=host.address)
-    )
-    yield ip
+def get_ip_ports_and_service(host: NmapHost, network: Network) -> Iterator[OOI]:
+    """Yields IPs, open ports and services if any ports are open on this host."""
+    open_ports = host.get_open_ports()
+    if open_ports:
+        ip = (
+            IPAddressV4(network=network.reference, address=host.address)
+            if host.ipv4
+            else IPAddressV6(network=network.reference, address=host.address)
+        )
+        yield ip
 
-    for port, protocol in host.get_open_ports():
-        service: NmapService = host.get_service(port, protocol)
+        for port, protocol in open_ports:
+            service: NmapService = host.get_service(port, protocol)
 
-        # If service is tcpwrapped we should consider the port closed
-        if service.service == "tcpwrapped":
-            continue
+            # If service is tcpwrapped we should consider the port closed
+            if service.service == "tcpwrapped":
+                continue
 
-        ip_port = IPPort(
-            address=ip.reference,
-            protocol=Protocol(protocol),
-            port=port,
-            state=PortState(service.state),
-        )
-        yield ip_port
+            ip_port = IPPort(
+                address=ip.reference,
+                protocol=Protocol(protocol),
+                port=port,
+                state=PortState(service.state),
+            )
+            yield ip_port
 
-        service_name = service.service
-        if port == 80:
-            service_name = "http"
-        if port == 443:
-            service_name = "https"
+            service_name = service.service
+            if port == 80:
+                service_name = "http"
+            if port == 443:
+                service_name = "https"
 
-        port_service = Service(name=service_name)
-        yield port_service
+            port_service = Service(name=service_name)
+            yield port_service
 
-        ip_service = IPService(ip_port=ip_port.reference, service=port_service.reference)
-        yield ip_service
+            ip_service = IPService(ip_port=ip_port.reference, service=port_service.reference)
+            yield ip_service
 
 
 def run(normalizer_meta: NormalizerMeta, raw: Union[bytes, str]) -> Iterator[OOI]:
+    """Decouple and parse Nmap XMLs and yield relevant network."""
+    # Multiple XMLs are concatenated through "\n\n". XMLs end with "\n"; we split on "\n\n\n".
+    raw = raw.decode().split("\n\n\n")
 
-    parsed: NmapReport = NmapParser.parse_fromstring(raw.decode())
+    # Relevant network object is received from the normalizer_meta.
+    network = Network(name=normalizer_meta.raw_data.boefje_meta.arguments["input"]["network"]["name"])
+    yield network
 
-    for host in parsed.hosts:
-        yield from get_ports_and_service(host)
+    logging.info("Parsing %d Nmap-xml(s) for %s.", len(raw), network)
+    for r in raw:
+        for host in NmapParser.parse_fromstring(r).hosts:
+            yield from get_ip_ports_and_service(host=host, network=network)

boefjes/plugins/kat_nmap/normalizer.json

@@ -2,7 +2,8 @@
     "id": "kat_nmap_normalize",
     "consumes": [
         "nmap",
-        "nmap-ports"
+        "nmap-ports",
+        "nmap-ip-range"
     ],
     "produces": [
         "IPAddressV6",

boefjes/plugins/kat_nmap_ip_range/boefje.json

@@ -0,0 +1,20 @@
+{
+    "id": "nmap-ip-range",
+    "name": "Nmap IP range",
+    "description": "Scan an IP range and store found IPs.",
+    "consumes": [
+        "NetBlock"
+    ],
+    "produces": [
+        "IPAddressV6",
+        "Service",
+        "IPPort",
+        "IPAddressV4",
+        "IPService"
+    ],
+    "environment_keys": [
+        "TOP_PORTS_TCP",
+        "TOP_PORTS_UDP"
+    ],
+    "scan_level": 2
+}

boefjes/plugins/kat_nmap_ip_range/description.md

@@ -0,0 +1,37 @@
+# Nmap IP-range
+
+This boefje checks an IP range/NetBlock and stores any IP addresses that seem to be active.
+
+### Options
+
+This Nmap boefje has the following hardcoded options:
+
+| Option | Function |
+| ----------- | ----------- |
+| `T4` | assume a fast and reliable network |
+| `Pn` | skips host discovery, treats hosts as online |
+|`-r` | scan ports in order |
+|`-v10` |use verbosity level 10 |
+|`-oX` |Output in XML |
+
+For TCP `-sS` is used, for UDP `-sU` is used. Both have their own TOP_PORTS argument.
+
+### Input OOIs
+
+Nmap expects an NetBlock as input.
+
+### Output OOIs
+
+Nmap outputs the following OOIs:
+
+|OOI type|Description|
+|---|---|
+|IPAddressV4 | IPv4 |
+|IPAddressV6 | IPv6 |
+|IpPort|Open ports of IpAddress|
+|Service|Services that are found|
+|IpService|IpService that couples a service to an open port|
+
+The boefje uses the same normalizer and structure as the generic `kat_nmap` boefje.
+
+**Cat name**: Elsje (inverted, mirrored)

boefjes/plugins/kat_nmap_ip_range/main.py

@@ -0,0 +1,49 @@
+from os import getenv
+from ipaddress import ip_network, IPv6Network
+from typing import List, Tuple, Union
+
+import docker
+from boefjes.job_models import BoefjeMeta
+
+NMAP_IMAGE = "instrumentisto/nmap:latest"
+TOP_PORTS_MAX = 65535
+TOP_PORTS_DEFAULT = 250
+TOP_PORTS_MIN = 1
+
+
+def run_nmap(args: List[str]) -> str:
+    """Run Nmap in Docker."""
+    client = docker.from_env()
+    return client.containers.run(NMAP_IMAGE, args, remove=True).decode()
+
+
+def build_nmap_arguments(ip_range: str, top_ports: int, protocol_str: str) -> List[str]:
+    """Build nmap arguments from the hosts IP with the required ports."""
+    if protocol_str not in ["S", "U"]:
+        raise ValueError('Protocol should be "S" or "U"')
+    if not TOP_PORTS_MIN <= top_ports <= TOP_PORTS_MAX:
+        raise ValueError(f"{TOP_PORTS_MIN} <= TOP_PORTS: {top_ports} <= {TOP_PORTS_MAX} is invalid.")
+
+    args = ["nmap", "-T4", "-Pn", "-r", "-v10", f"-s{protocol_str}", "--top-ports", str(top_ports)]
+    if isinstance(ip_network(ip_range), IPv6Network):
+        args.append("-6")
+    args.extend(["-oX", "-", ip_range])
+
+    return args
+
+
+def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
+    """Build Nmap arguments and return results to normalizer."""
+    ip_range = f"{boefje_meta.arguments['input']['start_ip']['address']}/{str(boefje_meta.arguments['input']['mask'])}"
+    top_ports_tcp = int(getenv("TOP_PORTS_TCP", 250))
+    top_ports_udp = int(getenv("TOP_PORTS_UDP", 10))
+    if not top_ports_tcp and not top_ports_udp:
+        raise ValueError("At least one TOP_PORTS argument should be non-zero")
+
+    results = []
+    if top_ports_tcp:
+        results.append(run_nmap(build_nmap_arguments(ip_range=ip_range, top_ports=top_ports_tcp, protocol_str="S")))
+    if top_ports_udp:
+        results.append(run_nmap(build_nmap_arguments(ip_range=ip_range, top_ports=top_ports_udp, protocol_str="U")))
+
+    return [(set(), "\n\n".join(results))]

boefjes/plugins/kat_nmap_ip_range/schema.json

@@ -0,0 +1,17 @@
+{
+    "title": "Arguments",
+    "type": "object",
+    "properties": {
+        "TOP_PORTS_TCP": {
+            "title": "TOP_PORTS_TCP",
+            "type": "integer",
+            "description": "Scan TOP_PORTS most common TCP ports. Defaults to 250."
+        },
+        "TOP_PORTS_UDP": {
+            "title": "TOP_PORTS_UDP",
+            "type": "integer",
+            "description": "Scan TOP_PORTS most common UDP ports. Defaults to 10."
+        }
+    },
+    "required": []
+}