diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml new file mode 100644 index 0000000..b4815fc --- /dev/null +++ b/.github/workflows/python-app.yml @@ -0,0 +1,52 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python application + +on: + push: + branches: [ "master", "pilot" ] + pull_request: + branches: [ "master", "pilot" ] + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + make venv + - name: Run the lint checker + run: | + make lint + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + make venv + - name: Run the tests + run: | + make test + + docker-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run the hadolint tool on the Dockerfile files + run: | + make hadolint diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3d555a0..5dea5ef 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -149,3 +149,35 @@ Zeek Enhancements (#177) Author: Nikhileswar Reddy +3.4.0 (2024-11-12) +################## + +Version 3.4.x is available initially on the pilot branch, +in a sort of pre-release mode. + +* Use pyproject.toml (#184) (#189) +* Use ruff format to format the code (#183) (#190) +* Use ruff check --fix to make style changes (#183) (#192) +* Add github actions CI (#191) (#193) +* Be able to run unit tests on dalton and flowsynth (#182) (#194) +* Update nginx from 1.19 to 1.27 (#200) (#202) +* Update redis from 3.2 to 7.4 (#201) +* Add unit tests for flowsynth (#204) +* Use ruff to sort and format imports (#207) +* Use ruff to detect flake8 bugbears (B) (#209) +* Use pre-built zeek images (#181) +* Use bump-my-version to update the version and tag (#197) + * Also, use bump-my-version to update the dalton-agent version + * Also, show the dalton controller version on the About page + +3.4.1 (2024-11-14) +################## + +* Fixed bug with zeek processing. (#213) (#214) (#216) +* Added some unit tests. (#203) (#215) + +3.4.2 (2024-11-15) +################## + +* Updated flask dependencies (#180) (#222) + * Configure flask maximum content length diff --git a/Dockerfile-dalton b/Dockerfile-dalton index f466baa..ae0d0be 100644 --- a/Dockerfile-dalton +++ b/Dockerfile-dalton @@ -1,24 +1,32 @@ -FROM python:3.9.0 -MAINTAINER David Wharton +FROM python:3.10.15 # wireshark needed for mergecap; statically compiled # mergecap would be smaller but doing this for now -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install wireshark-common \ - p7zip-full + +# hadolint ignore=DL3008 +RUN apt-get update -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + wireshark-common \ + p7zip-full \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* # for development; not needed by the app #RUN apt-get install -y less nano net-tools WORKDIR /opt/dalton -COPY requirements.txt /opt/dalton/requirements.txt -RUN pip install -r requirements.txt +COPY pyproject.toml /opt/dalton COPY app /opt/dalton/app +RUN pip install --no-cache-dir -e . COPY run.py /opt/dalton/run.py COPY dalton.conf /opt/dalton/dalton.conf COPY rulesets /opt/dalton/rulesets COPY engine-configs /opt/dalton/engine-configs -CMD python /opt/dalton/run.py -c /opt/dalton/dalton.conf +STOPSIGNAL SIGINT +EXPOSE 8080 + +# Note: if changing the next line, also look to change the command in docker-compose.yml +CMD ["flask", "--app", "app", "run", "--port=8080", "--host=0.0.0.0"] diff --git a/Dockerfile-nginx b/Dockerfile-nginx index 3d5e3b7..0dbc3c2 100644 --- a/Dockerfile-nginx +++ b/Dockerfile-nginx @@ -1,9 +1,8 @@ # spin up nginx with custom conf -FROM nginx:1.19.4 -MAINTAINER David Wharton +FROM nginx:1.27.2 -ARG DALTON_EXTERNAL_PORT -ARG DALTON_EXTERNAL_PORT_SSL +ARG DALTON_EXTERNAL_PORT=80 +ARG DALTON_EXTERNAL_PORT_SSL=443 RUN rm /etc/nginx/nginx.conf && rm -rf /etc/nginx/conf.d COPY nginx-conf/nginx.conf /etc/nginx/nginx.conf @@ -13,6 +12,5 @@ COPY nginx-conf/tls /etc/nginx/tls # adjust nginx config so redirects point to external port(s). # order of sed operations matters since one replaced string is a subset of the other. RUN sed -i 's/REPLACE_AT_DOCKER_BUILD_SSL/'"${DALTON_EXTERNAL_PORT_SSL}"'/' /etc/nginx/conf.d/dalton.conf +# hadolint ignore=DL3059 RUN sed -i 's/REPLACE_AT_DOCKER_BUILD/'"${DALTON_EXTERNAL_PORT}"'/' /etc/nginx/conf.d/dalton.conf - -CMD nginx diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1d6757f --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ + +VENV := $(or ${VENV},${VENV},$(CURDIR)/.venv) +PIP=$(VENV)/bin/pip +PYTHON=$(VENV)/bin/python +PYTEST=$(VENV)/bin/pytest +COVERAGE=$(VENV)/bin/coverage +RUFF=$(VENV)/bin/ruff +ACTIVATE=$(VENV)/bin/activate +BUMPVERSION=$(VENV)/bin/bump-my-version +BUMPPART ?= patch + +venv $(VENV): + python3 -m venv $(VENV) + $(PIP) install --upgrade pip wheel + $(PIP) install -e . -e ".[testing]" -e ".[devtools]" + +test: $(VENV) + . $(ACTIVATE) && $(PYTEST) tests + +coverage: $(VENV) + . $(ACTIVATE) && $(COVERAGE) run -m pytest tests + $(COVERAGE) report + +lint: $(VENV) + $(RUFF) format --check + $(RUFF) check + +fix: $(VENV) + $(RUFF) format + $(RUFF) check --fix + +hadolint: Dockerfile-dalton Dockerfile-nginx dalton-agent/Dockerfiles/Dockerfile_* + docker run -t --rm -v `pwd`:/app -w /app hadolint/hadolint /bin/hadolint $^ + +bumpversion: $(VENV) pyproject.toml + $(BUMPVERSION) bump $(BUMPPART) + +bumpagent: $(VENV) pyproject.toml + $(BUMPVERSION) bump --config-file dalton-agent/.bumpversion.toml $(BUMPPART) diff --git a/api/dalton.py b/api/dalton.py index c1b9f4c..42b254c 100644 --- a/api/dalton.py +++ b/api/dalton.py @@ -1,10 +1,10 @@ """Dalton API client.""" -import requests + import time +import requests from requests.exceptions import HTTPError - RETRIES = 3 SLEEP_TIME = 60 @@ -142,6 +142,7 @@ def get_current_sensors(self) -> dict: 'tech': 'suricata/5.0.7', 'agent_version': '3.1.1'}} """ - response = self._dalton_get("dalton/controller_api/get-current-sensors-json-full") + response = self._dalton_get( + "dalton/controller_api/get-current-sensors-json-full" + ) return response.json() - \ No newline at end of file diff --git a/api/examples/job_submission.py b/api/examples/job_submission.py index 9d15074..ec9c448 100644 --- a/api/examples/job_submission.py +++ b/api/examples/job_submission.py @@ -1,4 +1,5 @@ """Example on how to submit a job using the Dalton Client API. Mock data is in mocks directory.""" + import os from api.dalton import DaltonAPI diff --git a/app/__init__.py b/app/__init__.py index e69de29..923c684 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -0,0 +1,50 @@ +import logging +import os + +from flask import Flask + +from app.dalton import dalton_blueprint, ensure_rulesets_exist, setup_dalton_logging +from app.flowsynth import flowsynth_blueprint, setup_flowsynth_logging + +__version__ = "3.4.2" + + +def create_app(test_config=None): + """Create the flask app.""" + curdir = os.path.dirname(os.path.abspath(__file__)) + static_folder = os.path.join(curdir, "static") + daltonfs = Flask("app", static_folder=static_folder) + if test_config: + # load the test config if passed in + daltonfs.config.from_mapping(test_config) + + if not daltonfs.testing: + setup_dalton_logging() + setup_flowsynth_logging() + ensure_rulesets_exist() + + # register modules + # + # dalton + daltonfs.register_blueprint(dalton_blueprint) + + # flowsynth + daltonfs.register_blueprint(flowsynth_blueprint, url_prefix="/flowsynth") + + daltonfs.debug = True + + # Apparently the werkzeug default logger logs every HTTP request + # which bubbles up to the root logger and gets output to the + # console which ends up in the docker logs. Since each agent + # checks in every second (by default), this can be voluminous + # and is superfluous for my current needs. + try: + logging.getLogger("werkzeug").setLevel(logging.ERROR) + except Exception: + pass + + # Allow the user or the agent to upload large files + daltonfs.config["MAX_CONTENT_LENGTH"] = 1024 * 1024 * 1024 + daltonfs.config["MAX_FORM_MEMORY_SIZE"] = None + + return daltonfs diff --git a/app/certsynth.py b/app/certsynth.py index eb6c65c..44b6200 100644 --- a/app/certsynth.py +++ b/app/certsynth.py @@ -1,8 +1,8 @@ -import struct import binascii import re +import struct -SYNTH_START = ''' +SYNTH_START = """ ## Client Hello default > ( content:"\\x80\\x80"; #Length @@ -16,15 +16,15 @@ ## Server Hello and Certificate default < ( ### Server Hello -content:"\\x16\\x03\\x01\\x00Q\\x02\\x00\\x00M\\x03\\x01U\\xe8\\x83\\x0f\\xa1\\xe8\\xcd\\xc6^k\\xae`\\xf4\\xbe\\x0er\\xab~w,\\xce\\xf6L\\x89#\\xc9\\xbaU\\x8b\\xae\\x0ef\\x20\\xcf\\xb4\\xc1\\xb9\\xb0-\\x1e\\xe6Zl\\x0f\\xff\\xfd\\xe3)\\x97\\x89cy\\xa9\\xdbNV\\x83\\xa5\\x97:\\x12Td\\x09\\xac\\x009\\x00\\x00\\x05\\xff\\x01\\x00\\x01\\x00";''' +content:"\\x16\\x03\\x01\\x00Q\\x02\\x00\\x00M\\x03\\x01U\\xe8\\x83\\x0f\\xa1\\xe8\\xcd\\xc6^k\\xae`\\xf4\\xbe\\x0er\\xab~w,\\xce\\xf6L\\x89#\\xc9\\xbaU\\x8b\\xae\\x0ef\\x20\\xcf\\xb4\\xc1\\xb9\\xb0-\\x1e\\xe6Zl\\x0f\\xff\\xfd\\xe3)\\x97\\x89cy\\xa9\\xdbNV\\x83\\xa5\\x97:\\x12Td\\x09\\xac\\x009\\x00\\x00\\x05\\xff\\x01\\x00\\x01\\x00";""" -SYNTH_END = ''' +SYNTH_END = """ ### Server Key Exchange content:"\\x16\\x03\\x01\\x01\\x8d\\x0c\\x00\\x01\\x89\\x00\\x80\\xbb\\xbc-\\xca\\xd8Ft\\x90|C\\xfc\\xf5\\x80\\xe9\\xcf\\xdb\\xd9X\\xa3\\xf5h\\xb4-K\\x08\\xee\\xd4\\xeb\\x0f\\xb3PLl\\x03\\x02v\\xe7\\x10\\x80\\x0c\\\\xcb\\xba\\xa8\\x92&\\x14\\xc5\\xbe\\xec\\xa5e\\xa5\\xfd\\xf1\\xd2\\x87\\xa2\\xbc\\x04\\x9b\\xe6w\\x80`\\xe9\\x1a\\x92\\xa7W\\xe3\\x04\\x8fh\\xb0v\\xf7\\xd3l\\xc8\\xf2\\x9b\\xa5\\xdf\\x81\\xdc,\\xa7%\\xec\\xe6bp\\xcc\\x9aP5\\xd8\\xce\\xce\\xef\\x9e\\xa0'Jc\\xab\\x1eX\\xfa\\xfdI\\x88\\xd0\\xf6]\\x14gW\\xda\\x07\\x1d\\xf0E\\xcf\\xe1k\\x9b\\x00\\x01\\x02\\x00\\x80\\xa9\\xc5y)\\U\\x00\\x1f\\xa30\\x9b\\x8e\\xd6.\\xed\\x01\\xe9VY0\\x9e\\x03\\x95\\x1b\\x88[q\\xdd\\xfd\\x16\\x0e\\x1a\\xc3\\xbd\\xd3\\x1c\\xbc\\x92\\xa1o\\xed\\xa5T\\xea\\xaa\\xf7\\xdd\\xcd\\xd7\\xb8\\x20E\\x9b\\x1a\\xd4H}[\\xf46\\x98dL\\x0d\\xb6\\xfc\\xb2\\x0d\\xf7\\x94\\xecv\\xd5\\x1f\\xf2\\x85;\\xa6\\xf6\\xf2U\\xcb\\x16\\xc4z\\xa1/\\xdeq\\xf3\\xb0\\x20\\x19\\xef\\xc8\\xc9\\xa5\\x15\\xae\\x9f\\xe9\\x07:\\x0d\\x10\\xbe\\xc8\\xb3\\x98Zh\\xe6k\\x7f5\\x1d\\x8f\\x00\\x80\\x19\\xbb\\x17\\xd6e\\x00\\xc8Y\\x95L\\xde\\xdb\\x9b\\xc7I\\x20F\\x96Po\\xf8\\xedV\\x92\\x85\\xe5V\\xf2zC\\x06\\xcb\\xcc\\xfe(\\x82\\x1c\\x11\\x9d\\xb8\\xd3wT\\x9c\\x08\\xe6\\x0aA\\x06\\xbax\\xb8\\x85\\x94p+\\x88/\\xb4\\x20%\\x1bhx\\xc462\\xa4;\\x9e\\xe7\\x98`\\x01]H', len(cert_bytes) + 10) + handshake_len_bytes = struct.pack(">H", len(cert_bytes) + 10) handshake_len_synth_bytes = to_synth_bytes(handshake_len_bytes) - cert_handshake_len_bytes = struct.pack('>I', len(cert_bytes) + 6)[1:] + cert_handshake_len_bytes = struct.pack(">I", len(cert_bytes) + 6)[1:] cert_handshake_len_synth_bytes = to_synth_bytes(cert_handshake_len_bytes) - certs_len_bytes = struct.pack('>I', len(cert_bytes) + 3)[1:] + certs_len_bytes = struct.pack(">I", len(cert_bytes) + 3)[1:] certs_len_synth_bytes = to_synth_bytes(certs_len_bytes) - cert_len_bytes = struct.pack('>I', len(cert_bytes))[1:] + cert_len_bytes = struct.pack(">I", len(cert_bytes))[1:] cert_len_synth_bytes = to_synth_bytes(cert_len_bytes) - return "".join([SYNTH_START, - SYNTH_CERTIFICATES.format( - handshake_len_synth_bytes, - cert_handshake_len_synth_bytes, - certs_len_synth_bytes, - cert_len_synth_bytes, - to_synth_bytes(cert_bytes)), - SYNTH_END]) + return "".join( + [ + SYNTH_START, + SYNTH_CERTIFICATES.format( + handshake_len_synth_bytes, + cert_handshake_len_synth_bytes, + certs_len_synth_bytes, + cert_len_synth_bytes, + to_synth_bytes(cert_bytes), + ), + SYNTH_END, + ] + ) diff --git a/app/dalton.py b/app/dalton.py index 90dd385..2519ee8 100644 --- a/app/dalton.py +++ b/app/dalton.py @@ -17,76 +17,93 @@ # limitations under the License. # app imports -from flask import Blueprint, render_template, request, Response, redirect, url_for -#from flask_login import current_user +import base64 +import bz2 +import configparser +import copy +import datetime +import glob +import gzip + +# from flask_login import current_user import hashlib +import json +import logging import os -import glob +import random import re -import redis -import datetime +import shutil +import subprocess +import tarfile +import tempfile import time -import json +import traceback import zipfile -import tarfile -import gzip -import bz2 -import sys -import shutil from distutils.version import LooseVersion -import configparser -import logging +from functools import lru_cache from logging.handlers import RotatingFileHandler -import subprocess -from ruamel import yaml -import base64 -import traceback -import subprocess -import random from threading import Thread -import tempfile -import copy + +from flask import Blueprint, Response, redirect, render_template, request, url_for +from redis import Redis +from ruamel import yaml # setup the dalton blueprint -dalton_blueprint = Blueprint('dalton_blueprint', __name__, template_folder='templates/dalton/') +dalton_blueprint = Blueprint( + "dalton_blueprint", __name__, template_folder="templates/dalton/" +) -# logging -file_handler = RotatingFileHandler('/var/log/dalton.log', 'a', 1 * 1024 * 1024, 10) -file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) logger = logging.getLogger("dalton") -logger.addHandler(file_handler) -logger.setLevel(logging.INFO) -logger.info("Logging started") +ONLY_RUN_ONCE = lru_cache(maxsize=None) + + +def setup_dalton_logging(): + """Set up logging.""" + file_handler = RotatingFileHandler("/var/log/dalton.log", "a", 1 * 1024 * 1024, 10) + file_handler.setFormatter( + logging.Formatter("%(asctime)s %(levelname)s: %(message)s") + ) + logger.addHandler(file_handler) + logger.setLevel(logging.INFO) + + logger.info("Logging started") + try: - dalton_config_filename = 'dalton.conf' - dalton_config = configparser.SafeConfigParser() + dalton_config_filename = "dalton.conf" + dalton_config = configparser.ConfigParser() dalton_config.read(dalton_config_filename) - TEMP_STORAGE_PATH = dalton_config.get('dalton', 'temp_path') - RULESET_STORAGE_PATH = dalton_config.get('dalton', 'ruleset_path') - SCRIPT_STORAGE_PATH = dalton_config.get('dalton', 'script_path') - JOB_STORAGE_PATH = dalton_config.get('dalton', 'job_path') - CONF_STORAGE_PATH = dalton_config.get('dalton', 'engine_conf_path') - REDIS_EXPIRE = (dalton_config.getint('dalton', 'redis_expire') * 60) - SHARE_EXPIRE = (dalton_config.getint('dalton', 'share_expire') * 60) - TEAPOT_REDIS_EXPIRE = (dalton_config.getint('dalton', 'teapot_redis_expire') * 60) - JOB_RUN_TIMEOUT = dalton_config.getint('dalton', 'job_run_timeout') - AGENT_PURGE_TIME = dalton_config.getint('dalton', 'agent_purge_time') - REDIS_HOST = dalton_config.get('dalton', 'redis_host') - API_KEYS = dalton_config.get('dalton', 'api_keys') - MERGECAP_BINARY = dalton_config.get('dalton', 'mergecap_binary') - U2_ANALYZER = dalton_config.get('dalton', 'u2_analyzer') - RULECAT_SCRIPT = dalton_config.get('dalton', 'rulecat_script') - MAX_PCAP_FILES = dalton_config.getint('dalton', 'max_pcap_files') - DEBUG = dalton_config.getboolean('dalton', 'debug') - - #options for flowsynth - FS_BIN_PATH = dalton_config.get('flowsynth-web', 'bin_path') #Path to the flowsynth application - FS_PCAP_PATH = dalton_config.get('flowsynth-web', 'pcap_path') #Path to temporarily store PCAPs + TEMP_STORAGE_PATH = dalton_config.get("dalton", "temp_path") + RULESET_STORAGE_PATH = dalton_config.get("dalton", "ruleset_path") + SCRIPT_STORAGE_PATH = dalton_config.get("dalton", "script_path") + JOB_STORAGE_PATH = dalton_config.get("dalton", "job_path") + CONF_STORAGE_PATH = dalton_config.get("dalton", "engine_conf_path") + REDIS_EXPIRE = dalton_config.getint("dalton", "redis_expire") * 60 + SHARE_EXPIRE = dalton_config.getint("dalton", "share_expire") * 60 + TEAPOT_REDIS_EXPIRE = dalton_config.getint("dalton", "teapot_redis_expire") * 60 + JOB_RUN_TIMEOUT = dalton_config.getint("dalton", "job_run_timeout") + AGENT_PURGE_TIME = dalton_config.getint("dalton", "agent_purge_time") + REDIS_HOST = dalton_config.get("dalton", "redis_host") + API_KEYS = dalton_config.get("dalton", "api_keys") + MERGECAP_BINARY = dalton_config.get("dalton", "mergecap_binary") + U2_ANALYZER = dalton_config.get("dalton", "u2_analyzer") + RULECAT_SCRIPT = dalton_config.get("dalton", "rulecat_script") + MAX_PCAP_FILES = dalton_config.getint("dalton", "max_pcap_files") + DEBUG = dalton_config.getboolean("dalton", "debug") + + # options for flowsynth + FS_BIN_PATH = dalton_config.get( + "flowsynth-web", "bin_path" + ) # Path to the flowsynth application + FS_PCAP_PATH = dalton_config.get( + "flowsynth-web", "pcap_path" + ) # Path to temporarily store PCAPs except Exception as e: - logger.critical("Problem parsing config file '%s': %s" % (dalton_config_filename, e)) + logger.critical( + "Problem parsing config file '%s': %s" % (dalton_config_filename, e) + ) if DEBUG or ("CONTROLLER_DEBUG" in os.environ and int(os.getenv("CONTROLLER_DEBUG"))): logger.setLevel(logging.DEBUG) @@ -94,59 +111,91 @@ logger.debug("DEBUG logging enabled") if not MERGECAP_BINARY or not os.path.exists(MERGECAP_BINARY): - logger.error("mergecap binary '%s' not found. Suricata jobs cannot contain more than one pcap." % MERGECAP_BINARY) + logger.error( + "mergecap binary '%s' not found. Suricata jobs cannot contain more than one pcap." + % MERGECAP_BINARY + ) MERGECAP_BINARY = None -#connect to the datastore -try: - # redis values are returned as byte objects by default. Automatically - # decode them to utf-8. - r = redis.Redis(REDIS_HOST, charset="utf-8", decode_responses=True) -except Exception as e: - logger.critical("Problem connecting to Redis host '%s': %s" % (REDIS_HOST, e)) # if there are no rules, use idstools rulecat to download a set for Suri and Snort # if rulecat fails (eaten by proxy), empty rules file(s) may be created # TODO: change this to use suricata-update? -for engine in ['suricata', 'snort']: - ruleset_dir = os.path.join(RULESET_STORAGE_PATH, engine) - rules = [f for f in os.listdir(ruleset_dir) if (os.path.isfile(os.path.join(ruleset_dir, f)) and f.endswith(".rules"))] - if len(rules) == 0: - filename = "ET-%s-all-%s.rules" % (datetime.datetime.utcnow().strftime("%Y%m%d"), engine) - logger.info("No rulesets for %s found. Downloading the latest ET set as '%s'" % (engine, filename)) - if engine == "suricata": - url = "https://rules.emergingthreats.net/open/suricata-5.0/emerging.rules.tar.gz" - if engine == "snort": - url = "https://rules.emergingthreats.net/open/snort-2.9.0/emerging.rules.tar.gz" - command = "%s --url %s --merged %s" % (RULECAT_SCRIPT, url, os.path.join(ruleset_dir, filename)) - try: - subprocess.call(command, stdin=None, stdout=None, stderr=None, shell=True) - except Exception as e: - logger.info("Unable to download ruleset for %s" % engine) - logger.debug("Exception: %s" % e) +def ensure_rulesets_exist(): + for engine in ["suricata", "snort"]: + ruleset_dir = os.path.join(RULESET_STORAGE_PATH, engine) + rules = [ + f + for f in os.listdir(ruleset_dir) + if (os.path.isfile(os.path.join(ruleset_dir, f)) and f.endswith(".rules")) + ] + if len(rules) == 0: + filename = "ET-%s-all-%s.rules" % ( + datetime.datetime.utcnow().strftime("%Y%m%d"), + engine, + ) + logger.info( + "No rulesets for %s found. Downloading the latest ET set as '%s'" + % (engine, filename) + ) + if engine == "suricata": + url = "https://rules.emergingthreats.net/open/suricata-5.0/emerging.rules.tar.gz" + if engine == "snort": + url = "https://rules.emergingthreats.net/open/snort-2.9.0/emerging.rules.tar.gz" + command = "%s --url %s --merged %s" % ( + RULECAT_SCRIPT, + url, + os.path.join(ruleset_dir, filename), + ) + try: + subprocess.call( + command, stdin=None, stdout=None, stderr=None, shell=True + ) + except Exception as e: + logger.info("Unable to download ruleset for %s" % engine) + logger.debug("Exception: %s" % e) + # check for sane timeout values if REDIS_EXPIRE <= 0: - logger.critical("redis_expire value of %d minutes is invalid. Expect problems." % dalton_config.getint('dalton', 'redis_expire')) + logger.critical( + "redis_expire value of %d minutes is invalid. Expect problems." + % dalton_config.getint("dalton", "redis_expire") + ) if TEAPOT_REDIS_EXPIRE <= 0: - logger.critical("teapot_redis_expire value of %d minutes is invalid. Expect problems." % dalton_config.getint('dalton', 'teapot_redis_expire')) + logger.critical( + "teapot_redis_expire value of %d minutes is invalid. Expect problems." + % dalton_config.getint("dalton", "teapot_redis_expire") + ) if AGENT_PURGE_TIME <= 1: - logger.critical("agent_purge_time value of %d seconds is invalid. Expect problems." % AGENT_PURGE_TIME) + logger.critical( + "agent_purge_time value of %d seconds is invalid. Expect problems." + % AGENT_PURGE_TIME + ) if JOB_RUN_TIMEOUT <= 4: - logger.critical("job_run_time value of %d seconds is invalid. Expect problems." % JOB_RUN_TIMEOUT) + logger.critical( + "job_run_time value of %d seconds is invalid. Expect problems." + % JOB_RUN_TIMEOUT + ) if TEAPOT_REDIS_EXPIRE > REDIS_EXPIRE: - logger.warn("teapot_redis_expire value %d greater than redis_expire value %d. This is not recommended and may result in teapot jobs being deleted from disk before they expire in Redis." % (TEAPOT_REDIS_EXPIRE, REDIS_EXPIRE)) + logger.warning( + "teapot_redis_expire value %d greater than redis_expire value %d. This is not recommended and may result in teapot jobs being deleted from disk before they expire in Redis." + % (TEAPOT_REDIS_EXPIRE, REDIS_EXPIRE) + ) # other checks if MAX_PCAP_FILES < 1: default_max = 8 - logger.warn("max_pcap_files value of '%d' invalid. Using '%d'" % (MAX_PCAP_FILES, default_max)) + logger.warning( + "max_pcap_files value of '%d' invalid. Using '%d'" + % (MAX_PCAP_FILES, default_max) + ) MAX_PCAP_FILES = default_max -#global values used by Flask +# global values used by Flask TRAP_BAD_REQUEST_KEY_ERRORS = True -#status codes +# status codes STAT_CODE_INVALID = -1 STAT_CODE_QUEUED = 0 STAT_CODE_RUNNING = 1 @@ -155,27 +204,30 @@ STAT_CODE_TIMEOUT = 4 # engine technologies supported; used for validation (sometimes) -supported_engines = ['suricata', 'snort', 'zeek'] +supported_engines = ["suricata", "snort", "zeek"] logger.info("Dalton Started.") -""" returns normalized path; used to help prevent directory traversal """ + def clean_path(mypath): - return os.path.normpath('/' + mypath).lstrip('/') + """returns normalized path; used to help prevent directory traversal""" + return os.path.normpath("/" + mypath).lstrip("/") -def prefix_strip(mystring, prefixes=["rust_"]): - """ strip passed in prefixes from the beginning of passed in string and return it - """ +def prefix_strip(mystring, prefixes=None): + """strip passed in prefixes from the beginning of passed in string and return it""" + if not prefixes: + prefixes = ["rust_"] if not isinstance(prefixes, list): prefixes = [prefixes] for prefix in prefixes: - if mystring.startswith(prefix): - return mystring[len(prefix):] + if prefix and mystring.startswith(prefix): + return mystring[len(prefix) :] return mystring + def get_engine_and_version(sensor_tech): - """ returns list with engine ("suricata" or "snort") as first element, and + """returns list with engine ("suricata" or "snort") as first element, and version (e.g. "5.0.1", "2.9.9.0" as second element. Strips out prefix (e.g. "rust_") and ignores custom config (if present). Example passed in 'sensor_tech' values: suricata/5.0.1 @@ -185,16 +237,29 @@ def get_engine_and_version(sensor_tech): snort/2.9.9.0 """ try: - engine = sensor_tech.split('/')[0] - version = prefix_strip(sensor_tech.split('/')[1]) + engine = sensor_tech.split("/")[0] + version = prefix_strip(sensor_tech.split("/")[1]) return (engine, version) except Exception as e: - logger.error(f"Unable to process value '{sensor_tech}' in get_engine_and_version(): {e}") + logger.error( + f"Unable to process value '{sensor_tech}' in get_engine_and_version(): {e}" + ) return (None, None) + +@ONLY_RUN_ONCE +def get_redis(): + """Connect to the datastore.""" + try: + # redis values are returned as byte objects by default. Automatically + # decode them to utf-8. + return Redis(REDIS_HOST, charset="utf-8", decode_responses=True) + except Exception as e: + logger.critical("Problem connecting to Redis host '%s': %s" % (REDIS_HOST, e)) + + def delete_temp_files(job_id): - """ deletes temp files for given job ID""" - global TEMP_STORAGE_PATH + """deletes temp files for given job ID""" if os.path.exists(TEMP_STORAGE_PATH): for file in glob.glob(os.path.join(TEMP_STORAGE_PATH, "%s*" % job_id)): if os.path.isfile(file): @@ -202,21 +267,36 @@ def delete_temp_files(job_id): if os.path.exists(os.path.join(TEMP_STORAGE_PATH, job_id)): shutil.rmtree(os.path.join(TEMP_STORAGE_PATH, job_id)) + def verify_temp_storage_path(): """verify and create if necessary the temp location where we will store files (PCAPs, configs, etc.) - when build a job zip file + when build a job zip file """ - global TEMP_STORAGE_PATH if not os.path.exists(TEMP_STORAGE_PATH): os.makedirs(TEMP_STORAGE_PATH) return True -@dalton_blueprint.route('/dalton/controller_api/get-prod-rulesets/', methods=['GET']) + +def create_hash(values): + """Create an MD5 hash of the values and return the digest.""" + md5 = hashlib.md5() + for val in values: + md5.update(val.encode()) + return md5.hexdigest() + + +@dalton_blueprint.route( + "/dalton/controller_api/get-prod-rulesets/", methods=["GET"] +) def api_get_prod_rulesets(engine): - global supported_engines - if engine is None or engine == '' or engine not in supported_engines: - return Response("Invalid 'engine' supplied. Must be one of %s.\nExample URI:\n\n/dalton/controller_api/get-prod-rulesets/suricata" % supported_engines, - status=400, mimetype='text/plain', headers = {'X-Dalton-Webapp':'OK'}) + if engine is None or engine == "" or engine not in supported_engines: + return Response( + "Invalid 'engine' supplied. Must be one of %s.\nExample URI:\n\n/dalton/controller_api/get-prod-rulesets/suricata" + % supported_engines, + status=400, + mimetype="text/plain", + headers={"X-Dalton-Webapp": "OK"}, + ) # return json ruleset_list = [] # this is a 2D array with filename and full path for each rules file @@ -226,12 +306,17 @@ def api_get_prod_rulesets(engine): if len(ruleset) > 1: ruleset_list.append(ruleset[1]) - json_response = {'prod-rulesets': ruleset_list} - return Response(json.dumps(json_response), status=200, mimetype='application/json', headers = {'X-Dalton-Webapp':'OK'}) + json_response = {"prod-rulesets": ruleset_list} + return Response( + json.dumps(json_response), + status=200, + mimetype="application/json", + headers={"X-Dalton-Webapp": "OK"}, + ) -def get_rulesets(engine=''): - """ return a list of locally stored ruleset for jobs to use """ - global RULESET_STORAGE_PATH + +def get_rulesets(engine=""): + """return a list of locally stored ruleset for jobs to use""" ruleset_list = [] logger.debug("in get_rulesets(engine=%s)" % engine) # engine var should already be validated but just in case @@ -247,99 +332,127 @@ def get_rulesets(engine=''): for file in file_list: if not os.path.isfile(os.path.join(ruleset_dir, file)): continue - if os.path.splitext(file)[1] == '.rules': + if os.path.splitext(file)[1] == ".rules": # just add file (base) for now so we can sort; build 2D list on return ruleset_list.append(os.path.basename(file)) - #sort + # sort ruleset_list.sort(reverse=True) # return 2D array with base and full path return [[file, os.path.join(ruleset_dir, file)] for file in ruleset_list] -def set_job_status_msg(jobid, msg): - """set a job's status message """ - global r - r.set("%s-status" % jobid, msg) + +def set_job_status_msg(redis, jobid, msg): + """set a job's status message""" + redis.set("%s-status" % jobid, msg) # status keys do not expire if/when they are queued if msg != "Queued": - if r.get("%s-teapotjob" % jobid): - r.expire("%s-status" % jobid, TEAPOT_REDIS_EXPIRE) + if redis.get("%s-teapotjob" % jobid): + redis.expire("%s-status" % jobid, TEAPOT_REDIS_EXPIRE) else: - r.expire("%s-status" % jobid, REDIS_EXPIRE) + redis.expire("%s-status" % jobid, REDIS_EXPIRE) -def get_job_status_msg(jobid): + +def get_job_status_msg(redis, jobid): """returns a job's status message""" - return r.get("%s-status" % jobid) + return redis.get("%s-status" % jobid) -def set_job_status(jobid, status): +def set_job_status(redis, jobid, status): """set's a job status code""" - global r - r.set("%s-statcode" % jobid, status) + redis.set("%s-statcode" % jobid, status) # statcode keys do not expire if/when they are queued if status != STAT_CODE_QUEUED: - if r.get("%s-teapotjob" % jobid): - r.expire("%s-statcode" % jobid, TEAPOT_REDIS_EXPIRE) + if redis.get("%s-teapotjob" % jobid): + redis.expire("%s-statcode" % jobid, TEAPOT_REDIS_EXPIRE) else: - r.expire("%s-statcode" % jobid, REDIS_EXPIRE) + redis.expire("%s-statcode" % jobid, REDIS_EXPIRE) + -def get_job_status(jobid): +def get_job_status(redis, jobid): """return a job's status code""" - return r.get("%s-statcode" % jobid) + return redis.get("%s-statcode" % jobid) + -def get_alert_count(jobid): - if r.exists(f"{jobid}-alert"): - return r.get(f"{jobid}-alert").count('[**]') // 2 +def get_alert_count(redis, jobid): + if redis.exists(f"{jobid}-alert"): + return redis.get(f"{jobid}-alert").count("[**]") // 2 else: return None -def set_keys_timeout(jobid): + +def set_keys_timeout(redis, jobid): """set timeout of REDIS_EXPIRE seconds on keys that (should) be set when job results are posted""" EXPIRE_VALUE = REDIS_EXPIRE - if r.get("%s-teapotjob" % jobid): + if redis.get("%s-teapotjob" % jobid): EXPIRE_VALUE = TEAPOT_REDIS_EXPIRE try: - r.expire("%s-ids" % jobid, EXPIRE_VALUE) - r.expire("%s-perf" % jobid, EXPIRE_VALUE) - r.expire("%s-alert" % jobid, EXPIRE_VALUE) - r.expire("%s-error" % jobid, EXPIRE_VALUE) - r.expire("%s-debug" % jobid, EXPIRE_VALUE) - r.expire("%s-time" % jobid, EXPIRE_VALUE) - r.expire("%s-alert_detailed" % jobid, EXPIRE_VALUE) - r.expire("%s-other_logs" % jobid, EXPIRE_VALUE) - r.expire("%s-eve" % jobid, EXPIRE_VALUE) - r.expire("%s-teapotjob" % jobid, EXPIRE_VALUE) - r.expire("%s-zeek_json" % jobid, EXPIRE_VALUE) - except: + redis.expire("%s-ids" % jobid, EXPIRE_VALUE) + redis.expire("%s-perf" % jobid, EXPIRE_VALUE) + redis.expire("%s-alert" % jobid, EXPIRE_VALUE) + redis.expire("%s-error" % jobid, EXPIRE_VALUE) + redis.expire("%s-debug" % jobid, EXPIRE_VALUE) + redis.expire("%s-time" % jobid, EXPIRE_VALUE) + redis.expire("%s-alert_detailed" % jobid, EXPIRE_VALUE) + redis.expire("%s-other_logs" % jobid, EXPIRE_VALUE) + redis.expire("%s-eve" % jobid, EXPIRE_VALUE) + redis.expire("%s-teapotjob" % jobid, EXPIRE_VALUE) + redis.expire("%s-zeek_json" % jobid, EXPIRE_VALUE) + except Exception: pass -def expire_all_keys(jid): + +def expire_all_keys(redis, jid): """expires (deletes) all keys for a give job ID""" # using the redis keys function ('r.keys("%s-*" % jid)') searches thru all keys which is not # efficient for large key sets so we are deleting each one individually - global r logger.debug("Dalton calling expire_all_keys() on job %s" % jid) - keys_to_delete = ["ids", "perf", "alert", "alert_detailed", "other_logs", "eve", "error", "debug", "time", "statcode", "status", "start_time", "user", "tech", "submission_time", "teapotjob", "zeek_json"] + keys_to_delete = [ + "ids", + "perf", + "alert", + "alert_detailed", + "other_logs", + "eve", + "error", + "debug", + "time", + "statcode", + "status", + "start_time", + "user", + "tech", + "submission_time", + "teapotjob", + "zeek_json", + ] try: for cur_key in keys_to_delete: - r.delete("%s-%s" % (jid, cur_key)) - except: + redis.delete("%s-%s" % (jid, cur_key)) + except Exception: pass -def check_for_timeout(jobid): + +def check_for_timeout(redis, jobid): """checks to see if a job has been running more than JOB_RUN_TIMEOUT seconds and sets it to STAT_CODE_TIMEOUT and sets keys to expire""" - global r try: - start_time = int(r.get("%s-start_time" % jobid)) - except: + start_time = int(redis.get("%s-start_time" % jobid)) + except Exception: start_time = int(time.time()) - (JOB_RUN_TIMEOUT + 1) - #logger.debug("Dalton in check_for_timeout(): job %s start time: %d" % (jobid, start_time)) + # logger.debug("Dalton in check_for_timeout(): job %s start time: %d" % (jobid, start_time)) if not start_time or ((int(time.time()) - start_time) > JOB_RUN_TIMEOUT): - if int(get_job_status(jobid)) == STAT_CODE_RUNNING: - logger.info("Dalton in check_for_timeout(): job %s timed out. Start time: %d, now: %d" % (jobid, start_time, int(time.time()))) - set_job_status(jobid, STAT_CODE_TIMEOUT) - set_job_status_msg(jobid, "Job %s has timed out, please try submitting the job again." % jobid) - set_keys_timeout(jobid) + if int(get_job_status(redis, jobid)) == STAT_CODE_RUNNING: + logger.info( + "Dalton in check_for_timeout(): job %s timed out. Start time: %d, now: %d" + % (jobid, start_time, int(time.time())) + ) + set_job_status(redis, jobid, STAT_CODE_TIMEOUT) + set_job_status_msg( + redis, + jobid, + "Job %s has timed out, please try submitting the job again." % jobid, + ) + set_keys_timeout(redis, jobid) return True else: return False @@ -347,24 +460,28 @@ def check_for_timeout(jobid): return False -@dalton_blueprint.route('/dalton/controller_api/delete-old-job-files', methods=['GET']) +@dalton_blueprint.route("/dalton/controller_api/delete-old-job-files", methods=["GET"]) def delete_old_job_files(): """Deletes job files on disk if modification time exceeds expire time(s)""" - global REDIS_EXPIRE, TEAPOT_REDIS_EXPIRE, JOB_STORAGE_PATH, logger total_deleted = 0 - # this coded but not enabled since there isn't any authentication and I don't think + # this coded but not enabled since there isn't any authentication and I don't think # anyone should be able to delete jobs older than any arbitrary number of minutes if request: - mmin = request.args.get('mmin') - teapot_mmin = request.args.get('teapot_mmin') + mmin = request.args.get("mmin") + teapot_mmin = request.args.get("teapot_mmin") if mmin is not None: - logger.warn("Passing a mmin value to delete_old_job_files() is currently not enabled. Using %d seconds for regular jobs." % REDIS_EXPIRE) + logger.warning( + "Passing a mmin value to delete_old_job_files() is currently not enabled. Using %d seconds for regular jobs." + % REDIS_EXPIRE + ) if teapot_mmin is not None: - logger.warn("Passing a teapot_mmin value to delete_old_job_files() is currently not enabled. Using %d seconds for teapot jobs." % TEAPOT_REDIS_EXPIRE) + logger.warning( + "Passing a teapot_mmin value to delete_old_job_files() is currently not enabled. Using %d seconds for teapot jobs." + % TEAPOT_REDIS_EXPIRE + ) # these values represent number of minutes - job_mmin = REDIS_EXPIRE teapot_mmin = TEAPOT_REDIS_EXPIRE if os.path.exists(JOB_STORAGE_PATH): @@ -373,15 +490,33 @@ def delete_old_job_files(): for file in glob.glob(os.path.join(JOB_STORAGE_PATH, "*.zip")): if os.path.isfile(file): mtime = os.path.getmtime(file) - if (now-mtime) > REDIS_EXPIRE: - logger.debug("Deleting job file '%s'. mtime %s; now %s; diff %d seconds; expire threshold %d seconds" % (os.path.basename(file), now, mtime, (now-mtime), REDIS_EXPIRE)) + if (now - mtime) > REDIS_EXPIRE: + logger.debug( + "Deleting job file '%s'. mtime %s; now %s; diff %d seconds; expire threshold %d seconds" + % ( + os.path.basename(file), + now, + mtime, + (now - mtime), + REDIS_EXPIRE, + ) + ) os.unlink(file) total_deleted += 1 for file in glob.glob(os.path.join(JOB_STORAGE_PATH, "teapot_*.zip")): if os.path.isfile(file): mtime = os.path.getmtime(file) - if (now-mtime) > TEAPOT_REDIS_EXPIRE: - logger.debug("Deleting teapot job file '%s'. mtime %s; now %s; diff %d seconds; expire threshold %d seconds" % (os.path.basename(file), now, mtime, (now-mtime), TEAPOT_REDIS_EXPIRE)) + if (now - mtime) > TEAPOT_REDIS_EXPIRE: + logger.debug( + "Deleting teapot job file '%s'. mtime %s; now %s; diff %d seconds; expire threshold %d seconds" + % ( + os.path.basename(file), + now, + mtime, + (now - mtime), + TEAPOT_REDIS_EXPIRE, + ) + ) os.unlink(file) total_deleted += 1 if total_deleted > 0: @@ -390,58 +525,69 @@ def delete_old_job_files(): # return value need to cast it back to int if they wish to use it as an int return str(total_deleted) -@dalton_blueprint.route('/') + +@dalton_blueprint.route("/") def index(): logger.debug("ENVIRON:\n%s" % request.environ) # make sure redirect is set to use http or https as appropriate - rurl = url_for('dalton_blueprint.page_index', _external=True) - if rurl.startswith('http'): + rurl = url_for("dalton_blueprint.page_index", _external=True) + if rurl.startswith("http"): if "HTTP_X_FORWARDED_PROTO" in request.environ: # if original request was https, make sure redirect uses https - rurl = rurl.replace('http', request.environ['HTTP_X_FORWARDED_PROTO']) + rurl = rurl.replace("http", request.environ["HTTP_X_FORWARDED_PROTO"]) else: - logger.warn("Could not find request.environ['HTTP_X_FORWARDED_PROTO']. Make sure the web server (proxy) is configured to send it.") + logger.warning( + "Could not find request.environ['HTTP_X_FORWARDED_PROTO']. Make sure the web server (proxy) is configured to send it." + ) else: # this shouldn't be the case with '_external=True' passed to url_for() - logger.warn("URL does not start with 'http': %s" % rurl) + logger.warning("URL does not start with 'http': %s" % rurl) return redirect(rurl) -@dalton_blueprint.route('/dalton') -@dalton_blueprint.route('/dalton/') -#@login_required() + +@dalton_blueprint.route("/dalton") +@dalton_blueprint.route("/dalton/") +# @login_required() def page_index(): """the default homepage for Dalton""" - return render_template('/dalton/index.html', page='') + return render_template("/dalton/index.html", page="") # 'sensor' value includes forward slashes so this isn't a RESTful endpoint # and 'sensor' value must be passed as a GET parameter -@dalton_blueprint.route('/dalton/controller_api/request_engine_conf', methods=['GET']) -#@auth_required() +@dalton_blueprint.route("/dalton/controller_api/request_engine_conf", methods=["GET"]) +# @auth_required() def api_get_engine_conf_file(): - global supported_engines try: - sensor = request.args['sensor'] - except Exception as e: + sensor = request.args["sensor"] + except Exception: sensor = None if not sensor or len(sensor) == 0: - return Response("Invalid 'sensor' supplied.", - status=400, mimetype='text/plain', headers = {'X-Dalton-Webapp':'OK'}) - return Response(get_engine_conf_file(sensor), status=200, mimetype='text/plain', headers = {'X-Dalton-Webapp':'OK'}) + return Response( + "Invalid 'sensor' supplied.", + status=400, + mimetype="text/plain", + headers={"X-Dalton-Webapp": "OK"}, + ) + return Response( + get_engine_conf_file(sensor), + status=200, + mimetype="text/plain", + headers={"X-Dalton-Webapp": "OK"}, + ) + def get_engine_conf_file(sensor): - """ return the corresponding configuration file for passed in sensor (engine and version) - """ + """return the corresponding configuration file for passed in sensor (engine and version)""" # User's browser should be making request to dynamically update 'coverage' submission page # Also called by API handler try: conf_file = None - vars_file = None custom_config = None try: # if custom config used # 'sensor' variable format example: suricata/5.0.0/mycustomfilename - (engine, version, custom_config) = sensor.split('/', 2) + (engine, version, custom_config) = sensor.split("/", 2) epath = os.path.join(CONF_STORAGE_PATH, clean_path(engine)) if os.path.isfile(os.path.join(epath, "%s" % custom_config)): conf_file = "%s" % custom_config @@ -452,7 +598,7 @@ def get_engine_conf_file(sensor): elif os.path.isfile(os.path.join(epath, "%s.conf" % custom_config)): conf_file = "%s.conf" % custom_config if conf_file: - conf_file = (os.path.join(epath, clean_path(conf_file))) + conf_file = os.path.join(epath, clean_path(conf_file)) logger.debug(f"Found custom config file: '{conf_file}'") else: logger.error(f"Unable to find custom config file '{custom_config}'") @@ -460,102 +606,122 @@ def get_engine_conf_file(sensor): return engine_config except ValueError: # no custom config - (engine, version) = sensor.split('/', 1) + (engine, version) = sensor.split("/", 1) version = prefix_strip(version, prefixes="rust_") sensor2 = f"{engine}-{version}" epath = os.path.join(CONF_STORAGE_PATH, clean_path(engine)) - filelist = [f for f in os.listdir(epath) if os.path.isfile(os.path.join(epath, f))] + filelist = [ + f for f in os.listdir(epath) if os.path.isfile(os.path.join(epath, f)) + ] # assumes an extension (e.g. '.yaml', '.conf') on engine config files # if exact match, just use that instead of relying on LooseVersion files = [f for f in filelist if os.path.splitext(f)[0] == sensor2] if len(files) == 0: - files = [f for f in filelist if LooseVersion(os.path.splitext(f)[0]) <= LooseVersion(sensor2)] + files = [ + f + for f in filelist + if LooseVersion(os.path.splitext(f)[0]) <= LooseVersion(sensor2) + ] if len(files) > 0: - files.sort(key=lambda v:LooseVersion(os.path.splitext(v)[0]), reverse=True) + files.sort( + key=lambda v: LooseVersion(os.path.splitext(v)[0]), reverse=True + ) conf_file = os.path.join(epath, files[0]) - logger.debug("in get_engine_conf_file(): passed sensor value: '%s', conf file used: '%s'", sensor, os.path.basename(conf_file)) + logger.debug( + "in get_engine_conf_file(): passed sensor value: '%s', conf file used: '%s'", + sensor, + os.path.basename(conf_file), + ) - engine_config = '' + engine_config = "" if conf_file: # open, read, return # Unix newline is \n but for display on web page, \r\n is desired in some # browsers/OSes. Note: currently not converted back on job submit. - with open(conf_file, 'r') as fh: + with open(conf_file, "r") as fh: # want to parse each line so put it into a list contents = fh.readlines() logger.debug("Loading config file %s", conf_file) - engine_config = '\r\n'.join([x.rstrip('\r\n') for x in contents]) + engine_config = "\r\n".join([x.rstrip("\r\n") for x in contents]) else: - logger.warn("No suitable configuration file found for sensor '%s'.", sensor) - engine_config = f"# No suitable configuration file found for sensor '{sensor}'." + logger.warning( + "No suitable configuration file found for sensor '%s'.", sensor + ) + engine_config = ( + f"# No suitable configuration file found for sensor '{sensor}'." + ) return engine_config except Exception as e: - logger.error("Problem getting configuration file for sensor '%s'. Error: %s\n%s", sensor, e, traceback.format_exc()) + logger.error( + "Problem getting configuration file for sensor '%s'. Error: %s\n%s", + sensor, + e, + traceback.format_exc(), + ) engine_config = f"# Exception getting configuration file for sensor '{sensor}'." if DEBUG: engine_config += f" Error: {e}\r\n{traceback.format_exc()}" return engine_config -@dalton_blueprint.route('/dalton/sensor_api/update/', methods=['POST']) -#@auth_required('write') + +@dalton_blueprint.route("/dalton/sensor_api/update/", methods=["POST"]) +# @auth_required('write') # status update from Dalton Agent def sensor_update(): - """ a sensor has submitted an api update""" - global r - global STAT_CODE_DONE + """a sensor has submitted an api update""" + redis = get_redis() - uid = request.form.get('uid') - msg = request.form.get('msg') - job = request.form.get('job') + uid = request.form.get("uid") + msg = request.form.get("msg") + job = request.form.get("job") - if int(get_job_status(job)) != STAT_CODE_DONE: - set_job_status_msg(job, msg) + if int(get_job_status(redis, job)) != STAT_CODE_DONE: + set_job_status_msg(redis, job, msg) logger.debug("Dalton Agent %s sent update for job %s; msg: %s" % (uid, job, msg)) return "OK" -@dalton_blueprint.route('/dalton/sensor_api/request_job', methods=['GET']) -#@auth_required('read') +@dalton_blueprint.route("/dalton/sensor_api/request_job", methods=["GET"]) +# @auth_required('read') def sensor_request_job(): """Sensor API. Called when a sensor wants a new job""" # job request from Dalton Agent - global r - global STAT_CODE_RUNNING + redis = get_redis() try: - SENSOR_UID = request.args['SENSOR_UID'] - except Exception as e: - SENSOR_UID = 'unknown' + SENSOR_UID = request.args["SENSOR_UID"] + except Exception: + SENSOR_UID = "unknown" SENSOR_IP = request.remote_addr try: - AGENT_VERSION = request.args['AGENT_VERSION'] - except Exception as e: - AGENT_VERSION = 'unknown' + AGENT_VERSION = request.args["AGENT_VERSION"] + except Exception: + AGENT_VERSION = "unknown" try: - SENSOR_ENGINE = request.args['SENSOR_ENGINE'] - except Exception as e: - SENSOR_ENGINE = 'unknown' + SENSOR_ENGINE = request.args["SENSOR_ENGINE"] + except Exception: + SENSOR_ENGINE = "unknown" try: - SENSOR_ENGINE_VERSION = request.args['SENSOR_ENGINE_VERSION'] - except Exception as e: - SENSOR_ENGINE_VERSION = 'unknown' + SENSOR_ENGINE_VERSION = request.args["SENSOR_ENGINE_VERSION"] + except Exception: + SENSOR_ENGINE_VERSION = "unknown" sensor_tech = f"{SENSOR_ENGINE}/{SENSOR_ENGINE_VERSION}" SENSOR_CONFIG = None - if 'SENSOR_CONFIG' in request.args.keys(): + if "SENSOR_CONFIG" in request.args.keys(): try: - SENSOR_CONFIG = request.args['SENSOR_CONFIG'] - except Exception as e: + SENSOR_CONFIG = request.args["SENSOR_CONFIG"] + except Exception: SENSOR_CONFIG = None if SENSOR_CONFIG and len(SENSOR_CONFIG) > 0: @@ -565,138 +731,162 @@ def sensor_request_job(): # note: sensor keys are expired by function clear_old_agents() which removes the sensor # when it has not checked in in amount of time (expire time configurable via # 'agent_purge_time' parameter in dalton.conf). - hash = hashlib.md5() - hash.update(SENSOR_UID.encode('utf-8')) - hash.update(SENSOR_IP.encode('utf-8')) - SENSOR_HASH = hash.hexdigest() - r.sadd("sensors", SENSOR_HASH) - r.set(f"{SENSOR_HASH}-uid", SENSOR_UID) - r.set(f"{SENSOR_HASH}-ip", SENSOR_IP) - r.set(f"{SENSOR_HASH}-time", datetime.datetime.now().strftime("%b %d %H:%M:%S")) - r.set(f"{SENSOR_HASH}-epoch", int(time.mktime(time.localtime()))) - r.set(f"{SENSOR_HASH}-tech", sensor_tech) - r.set(f"{SENSOR_HASH}-agent_version", AGENT_VERSION) - - #grab a job! If it doesn't exist, return sleep. - response = r.lpop(sensor_tech) - if (response == None): + SENSOR_HASH = create_hash([SENSOR_UID, SENSOR_IP]) + redis.sadd("sensors", SENSOR_HASH) + redis.set(f"{SENSOR_HASH}-uid", SENSOR_UID) + redis.set(f"{SENSOR_HASH}-ip", SENSOR_IP) + redis.set(f"{SENSOR_HASH}-time", datetime.datetime.now().strftime("%b %d %H:%M:%S")) + redis.set(f"{SENSOR_HASH}-epoch", int(time.mktime(time.localtime()))) + redis.set(f"{SENSOR_HASH}-tech", sensor_tech) + redis.set(f"{SENSOR_HASH}-agent_version", AGENT_VERSION) + + # grab a job! If it doesn't exist, return sleep. + response = redis.lpop(sensor_tech) + if response is None: return "sleep" else: respobj = json.loads(response) - new_jobid = respobj['id'] - logger.info("Dalton Agent %s grabbed job %s for %s" % (SENSOR_UID, new_jobid, sensor_tech)) + new_jobid = respobj["id"] + logger.info( + "Dalton Agent %s grabbed job %s for %s" + % (SENSOR_UID, new_jobid, sensor_tech) + ) # there is a key for each sensor which is ("%s-current_job" % SENSOR_HASH) and has # the value of the current job id it is running. This value is set when a job is - # requested and set to 'None' when the results are posted. A sensor can only run - # one job at a time so if there is an exiting job when the sensor requests a new + # requested and set to "" when the results are posted. A sensor can only run + # one job at a time so if there is an existing job when the sensor requests a new # job then that means the sensor was interrupted while processing a job and could - # did not communicate back with the controller. - existing_job = r.get("%s-current_job" % SENSOR_HASH) - #logger.debug("Dalton in sensor_request_job(): job requested, sensor hash %s, new job: %s, existing job: %s" % (SENSOR_HASH, new_jobid, existing_job)) + # not communicate back with the controller. + existing_job = redis.get("%s-current_job" % SENSOR_HASH) + # logger.debug("Dalton in sensor_request_job(): job requested, sensor hash %s, new job: %s, existing job: %s" % (SENSOR_HASH, new_jobid, existing_job)) if existing_job and existing_job != new_jobid: - set_job_status(existing_job, STAT_CODE_INTERRUPTED) - set_job_status_msg(existing_job, "Job %s was unexpectedly interrupted while running on the agent; please try submitting the job again." % existing_job) + set_job_status(redis, existing_job, STAT_CODE_INTERRUPTED) + set_job_status_msg( + redis, + existing_job, + "Job %s was unexpectedly interrupted while running on the agent; please try submitting the job again." + % existing_job, + ) # these shouldn't be populated but set them to expire just in case to prevent redis memory build up - set_keys_timeout(existing_job) - r.set("%s-current_job" % SENSOR_HASH, new_jobid) + set_keys_timeout(redis, existing_job) + redis.set("%s-current_job" % SENSOR_HASH, new_jobid) EXPIRE_VALUE = REDIS_EXPIRE - if r.get("%s-teapotjob" % new_jobid): + if redis.get("%s-teapotjob" % new_jobid): EXPIRE_VALUE = TEAPOT_REDIS_EXPIRE - r.expire("%s-current_job" % SENSOR_HASH, EXPIRE_VALUE) - r.set("%s-start_time" % new_jobid, int(time.time())) - r.expire("%s-start_time" % new_jobid, EXPIRE_VALUE) - set_job_status(new_jobid,STAT_CODE_RUNNING) + redis.expire("%s-current_job" % SENSOR_HASH, EXPIRE_VALUE) + redis.set("%s-start_time" % new_jobid, int(time.time())) + redis.expire("%s-start_time" % new_jobid, EXPIRE_VALUE) + set_job_status(redis, new_jobid, STAT_CODE_RUNNING) # if a user sees the "Running" message for more than a few dozen seconds (depending on # the size of the pcap(s) and ruleset), then the job is hung on the agent or is going to # timeout. Most likely the agent was killed or died during the job run. - set_job_status_msg(new_jobid, "Running...") + set_job_status_msg(redis, new_jobid, "Running...") # set expire times for keys that are stored on server until job is requested - r.expire("%s-submission_time" % new_jobid, EXPIRE_VALUE) - r.expire("%s-user" % new_jobid, EXPIRE_VALUE) - r.expire("%s-tech" % new_jobid, EXPIRE_VALUE) + redis.expire("%s-submission_time" % new_jobid, EXPIRE_VALUE) + redis.expire("%s-user" % new_jobid, EXPIRE_VALUE) + redis.expire("%s-tech" % new_jobid, EXPIRE_VALUE) return response -@dalton_blueprint.route('/dalton/sensor_api/results/', methods=['POST']) -#@auth_required('write') +@dalton_blueprint.route("/dalton/sensor_api/results/", methods=["POST"]) +# @auth_required('write') def post_job_results(jobid): - """ called by Dalton Agent sending job results """ + """called by Dalton Agent sending job results""" # no authentication or authorization so this is easily abused; anyone with jobid # can overwrite results if they submit first. - global STAT_CODE_DONE, STAT_CODE_RUNNING, STAT_CODE_QUEUED, DALTON_URL, REDIS_EXPIRE, TEAPOT_REDIS_EXPIRE, TEMP_STORAGE_PATH - global r + redis = get_redis() # check and make sure job results haven't already been posted in order to prevent # abuse/overwriting. This still isn't foolproof. - if r.exists("%s-time" % jobid) and (int(get_job_status(jobid)) not in [STAT_CODE_RUNNING, STAT_CODE_QUEUED]): - logger.error("Data for jobid %s already exists in database; not overwriting. Source IP: %s. job_status_code code: %d" % (jobid, request.remote_addr, int(get_job_status(jobid)))) - #typically this would go back to Agent who then ignores it - return Response("Error: job results already exist.", mimetype='text/plain', headers = {'X-Dalton-Webapp':'Error'}) - - jsons = request.form.get('json_data') + if redis.exists("%s-time" % jobid) and ( + int(get_job_status(redis, jobid)) not in [STAT_CODE_RUNNING, STAT_CODE_QUEUED] + ): + logger.error( + "Data for jobid %s already exists in database; not overwriting. Source IP: %s. job_status_code code: %d" + % (jobid, request.remote_addr, int(get_job_status(redis, jobid))) + ) + # typically this would go back to Agent who then ignores it + return Response( + "Error: job results already exist.", + mimetype="text/plain", + headers={"X-Dalton-Webapp": "Error"}, + ) + + jsons = request.form.get("json_data") result_obj = json.loads(jsons) - set_job_status_msg(jobid, "Final Job Status: %s" % result_obj['status']) + set_job_status_msg(redis, jobid, "Final Job Status: %s" % result_obj["status"]) # get sensor hash and update ("%s-current_job" % SENSOR_HASH) with 'None' SENSOR_IP = request.remote_addr - SENSOR_UID = 'unknown' + SENSOR_UID = "unknown" try: - SENSOR_UID = request.args['SENSOR_UID'] - except Exception as e: - SENSOR_UID = 'unknown' - hash = hashlib.md5() - hash.update(SENSOR_UID.encode('utf-8')) - hash.update(SENSOR_IP.encode('utf-8')) - SENSOR_HASH = hash.hexdigest() - r.set(f"{SENSOR_HASH}-current_job", None) - r.expire(f"{SENSOR_HASH}-current_job", REDIS_EXPIRE) - - logger.info("Dalton Agent %s submitted results for job %s. Result: %s", SENSOR_UID, jobid, result_obj['status']) - - #save results to db - if 'ids' in result_obj: - ids = result_obj['ids'] - elif 'snort' in result_obj: - ids = result_obj['snort'] + SENSOR_UID = request.args["SENSOR_UID"] + except Exception: + SENSOR_UID = "unknown" + SENSOR_HASH = create_hash([SENSOR_UID, SENSOR_IP]) + redis.set(f"{SENSOR_HASH}-current_job", "") + redis.expire(f"{SENSOR_HASH}-current_job", REDIS_EXPIRE) + + logger.info( + "Dalton Agent %s submitted results for job %s. Result: %s", + SENSOR_UID, + jobid, + result_obj["status"], + ) + + # save results to db + if "ids" in result_obj: + ids = result_obj["ids"] + elif "snort" in result_obj: + ids = result_obj["snort"] else: ids = "" - if 'performance' in result_obj: - perf = result_obj['performance'] + if "performance" in result_obj: + perf = result_obj["performance"] else: perf = "" - if 'alert' in result_obj: - alert = result_obj['alert'] + if "alert" in result_obj: + alert = result_obj["alert"] else: alert = "" - if 'error' in result_obj: - error = result_obj['error'] + if "error" in result_obj: + error = result_obj["error"] else: error = "" - if 'debug' in result_obj: - debug = result_obj['debug'] + if "debug" in result_obj: + debug = result_obj["debug"] else: debug = "" - if 'total_time' in result_obj: - time = result_obj['total_time'] + if "total_time" in result_obj: + time = result_obj["total_time"] else: time = "" # alert_detailed is base64 encoded unified2 binary data alert_detailed = "" - if 'alert_detailed' in result_obj: + if "alert_detailed" in result_obj: try: # write to disk and pass to u2spewfoo.py; we could do # myriad other things here like modify or import that # code but this works and should be compatible and # incorporate any future changes/improvements to the # script - u2_file = os.path.join(TEMP_STORAGE_PATH, "%s_unified2_%s" % (jobid, SENSOR_HASH)) + u2_file = os.path.join( + TEMP_STORAGE_PATH, "%s_unified2_%s" % (jobid, SENSOR_HASH) + ) u2_fh = open(u2_file, "wb") - u2_fh.write(base64.b64decode(result_obj['alert_detailed'])) + u2_fh.write(base64.b64decode(result_obj["alert_detailed"])) u2_fh.close() u2spewfoo_command = "%s %s" % (U2_ANALYZER, u2_file) - logger.debug("Processing unified2 data with command: '%s'" % u2spewfoo_command) - alert_detailed = subprocess.Popen(u2spewfoo_command, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).stdout.read() + logger.debug( + "Processing unified2 data with command: '%s'" % u2spewfoo_command + ) + alert_detailed = subprocess.Popen( + u2spewfoo_command, + shell=True, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + ).stdout.read() # delete u2 file os.unlink(u2_file) except Exception as e: @@ -708,155 +898,221 @@ def post_job_results(jobid): # other_logs only supported on Suricata for now if "other_logs" in result_obj: logger.debug("Accessing other_log data from agent POST...") - other_logs = result_obj['other_logs'] + other_logs = result_obj["other_logs"] else: other_logs = "" # EVE is Suricata only if "eve" in result_obj: logger.debug("Accessing EVE data from agent POST...") - eve = result_obj['eve'] + eve = result_obj["eve"] else: eve = "" # Use JSON logs for Zeek if "zeek_json" in result_obj: logger.debug("Accessing Zeek JSON data from agent POST...") - zeek_json = result_obj['zeek_json'] + zeek_json = result_obj["zeek_json"] else: zeek_json = False logger.debug("Saving job data to redis...") - r.set("%s-ids" % jobid, ids) - r.set("%s-perf" % jobid, perf) - r.set("%s-alert" % jobid, alert) - r.set("%s-error" % jobid, error) - r.set("%s-debug" % jobid, debug) - r.set("%s-time" % jobid, time) - r.set("%s-alert_detailed" % jobid, alert_detailed) - r.set("%s-other_logs" % jobid, other_logs) - r.set("%s-eve" % jobid, eve) - r.set("%s-zeek_json" % jobid, zeek_json) - set_keys_timeout(jobid) + redis.set("%s-ids" % jobid, ids) + redis.set("%s-perf" % jobid, perf) + redis.set("%s-alert" % jobid, alert) + redis.set("%s-error" % jobid, error) + redis.set("%s-debug" % jobid, debug) + redis.set("%s-time" % jobid, time) + redis.set("%s-alert_detailed" % jobid, alert_detailed) + redis.set("%s-other_logs" % jobid, other_logs) + redis.set("%s-eve" % jobid, eve) + if zeek_json is not None: + redis.set("%s-zeek_json" % jobid, str(zeek_json)) + set_keys_timeout(redis, jobid) logger.debug("Done saving job data to redis.") if error: - set_job_status_msg(jobid, '
ERROR!
Click here for details' % jobid) + set_job_status_msg( + redis, + jobid, + '
ERROR!
Click here for details' + % jobid, + ) else: - set_job_status_msg(jobid, 'Click here to view your results' % jobid) + set_job_status_msg( + redis, + jobid, + 'Click here to view your results' % jobid, + ) - set_job_status(jobid, STAT_CODE_DONE) + set_job_status(redis, jobid, STAT_CODE_DONE) logger.debug("Returning from post_job_results()") - return Response("OK", mimetype='text/plain', headers = {'X-Dalton-Webapp':'OK'}) + return Response("OK", mimetype="text/plain", headers={"X-Dalton-Webapp": "OK"}) -@dalton_blueprint.route('/dalton/controller_api/job_status/', methods=['GET']) -#@login_required() + +@dalton_blueprint.route("/dalton/controller_api/job_status/", methods=["GET"]) +# @login_required() def get_ajax_job_status_msg(jobid): """return the job status msg (as a string)""" + redis = get_redis() # user's browser requesting job status msg - global STAT_CODE_RUNNING if not validate_jobid(jobid): - return Response("Invalid Job ID: %s" % jobid, mimetype='text/plain', headers = {'X-Dalton-Webapp':'OK'}) - stat_code = get_job_status(jobid) + return Response( + "Invalid Job ID: %s" % jobid, + mimetype="text/plain", + headers={"X-Dalton-Webapp": "OK"}, + ) + stat_code = get_job_status(redis, jobid) if stat_code: if int(stat_code) == STAT_CODE_RUNNING: - check_for_timeout(jobid) - r_status_msg = get_job_status_msg(jobid) + check_for_timeout(redis, jobid) + r_status_msg = get_job_status_msg(redis, jobid) if r_status_msg: - return Response(r_status_msg, mimetype='text/plain', headers = {'X-Dalton-Webapp':'OK'}) + return Response( + r_status_msg, mimetype="text/plain", headers={"X-Dalton-Webapp": "OK"} + ) else: - return Response('Unknown', mimetype='text/plain', headers = {'X-Dalton-Webapp':'OK'}) + return Response( + "Unknown", mimetype="text/plain", headers={"X-Dalton-Webapp": "OK"} + ) else: - return Response("Invalid Job ID: %s" % jobid, mimetype='text/plain', headers = {'X-Dalton-Webapp':'OK'}) + return Response( + "Invalid Job ID: %s" % jobid, + mimetype="text/plain", + headers={"X-Dalton-Webapp": "OK"}, + ) + -@dalton_blueprint.route('/dalton/controller_api/job_status_code/', methods=['GET']) -#@login_required() +@dalton_blueprint.route( + "/dalton/controller_api/job_status_code/", methods=["GET"] +) +# @login_required() def get_ajax_job_status_code(jobid): """return the job status code (AS A STRING! -- you need to cast the return value as an int if you want to use it as an int)""" + redis = get_redis() # user's browser requesting job status code if not validate_jobid(jobid): return "%d" % STAT_CODE_INVALID - r_status_code = get_job_status(jobid) + r_status_code = get_job_status(redis, jobid) if not r_status_code: # invalid jobid return "%d" % STAT_CODE_INVALID else: if int(r_status_code) == STAT_CODE_RUNNING: - check_for_timeout(jobid) - return get_job_status(jobid) + check_for_timeout(redis, jobid) + return get_job_status(redis, jobid) -@dalton_blueprint.route('/dalton/sensor_api/get_job/', methods=['GET']) -#@auth_required('read') +@dalton_blueprint.route("/dalton/sensor_api/get_job/", methods=["GET"]) +# @auth_required('read') def sensor_get_job(id): """user or agent requesting a job zip file""" # get the user (for logging) logger.debug("Dalton in sensor_get_job(): request for job zip file %s", id) if not validate_jobid(id): logger.error("Bad jobid given: '%s'. Possible hacking attempt.", id) - return render_template('/dalton/error.html', jid=id, msg=[f"Bad jobid, invalid characters in: '{id}'"]) + return render_template( + "/dalton/error.html", + jid=id, + msg=[f"Bad jobid, invalid characters in: '{id}'"], + ) path = f"{JOB_STORAGE_PATH}/{id}.zip" if os.path.exists(path): - with open(path, 'rb') as fh: + with open(path, "rb") as fh: logger.debug(f"Dalton in sensor_get_job(): sending job zip file {id}") - return Response(fh.read(),mimetype="application/zip", headers={"Content-Disposition":f"attachment;filename={id}.zip"}) + return Response( + fh.read(), + mimetype="application/zip", + headers={"Content-Disposition": f"attachment;filename={id}.zip"}, + ) else: logger.error(f"Dalton in sensor_get_job(): could not find job {id} at {path}.") - return render_template('/dalton/error.html', jid=id, msg=[f"Job {id} does not exist on disk. It is either invalid or has been deleted."]) - - -def clear_old_agents(): - global r, AGENT_PURGE_TIME - if r.exists('sensors'): - for sensor in r.smembers('sensors'): + return render_template( + "/dalton/error.html", + jid=id, + msg=[ + f"Job {id} does not exist on disk. It is either invalid or has been deleted." + ], + ) + + +def clear_old_agents(redis): + if redis.exists("sensors"): + for sensor in redis.smembers("sensors"): try: - minutes_ago = int(round((int(time.mktime(time.localtime())) - int(r.get(f"{sensor}-epoch"))) / 60)) -# minutes_ago = AGENT_PURGE_TIME + minutes_ago = int( + round( + ( + int(time.mktime(time.localtime())) + - int(redis.get(f"{sensor}-epoch")) + ) + / 60 + ) + ) + # minutes_ago = AGENT_PURGE_TIME except Exception as e: logger.error("Error in clear_old_agents(): %s", e) # screwed something up, perhaps with Python3 strings... if minutes_ago >= AGENT_PURGE_TIME: # delete old agents - r.delete(f"{sensor}-uid") - r.delete(f"{sensor}-ip") - r.delete(f"{sensor}-time") - r.delete(f"{sensor}-epoch") - r.delete(f"{sensor}-tech") - r.delete(f"{sensor}-agent_version") - r.srem("sensors", sensor) - - -@dalton_blueprint.route('/dalton/sensor', methods=['GET']) -#@login_required() -def page_sensor_default(return_dict = False): + redis.delete(f"{sensor}-uid") + redis.delete(f"{sensor}-ip") + redis.delete(f"{sensor}-time") + redis.delete(f"{sensor}-epoch") + redis.delete(f"{sensor}-tech") + redis.delete(f"{sensor}-agent_version") + redis.srem("sensors", sensor) + + +@dalton_blueprint.route("/dalton/sensor", methods=["GET"]) +# @login_required() +def page_sensor_default(return_dict=False): """the default sensor page""" - global r + redis = get_redis() sensors = {} # first clear out old agents ('sensors') - clear_old_agents() - if r.exists('sensors'): - for sensor in r.smembers('sensors'): + clear_old_agents(redis) + if redis.exists("sensors"): + for sensor in redis.smembers("sensors"): # looks like redis keys are byte - minutes_ago = int(round((int(time.mktime(time.localtime())) - int(r.get(f"{sensor}-epoch"))) / 60)) + minutes_ago = int( + round( + ( + int(time.mktime(time.localtime())) + - int(redis.get(f"{sensor}-epoch")) + ) + / 60 + ) + ) sensors[sensor] = {} - sensors[sensor]['uid'] = r.get(f"{sensor}-uid") - sensors[sensor]['ip'] = r.get(f"{sensor}-ip") - sensors[sensor]['time'] = "{} ({} minutes ago)".format(r.get(f"{sensor}-time"), minutes_ago) - sensors[sensor]['tech'] = "{}".format(r.get(f"{sensor}-tech")) - sensors[sensor]['agent_version'] = "{}".format(r.get(f"{sensor}-agent_version")) + sensors[sensor]["uid"] = redis.get(f"{sensor}-uid") + sensors[sensor]["ip"] = redis.get(f"{sensor}-ip") + sensors[sensor]["time"] = "{} ({} minutes ago)".format( + redis.get(f"{sensor}-time"), minutes_ago + ) + sensors[sensor]["tech"] = "{}".format(redis.get(f"{sensor}-tech")) + sensors[sensor]["agent_version"] = "{}".format( + redis.get(f"{sensor}-agent_version") + ) if return_dict: return sensors else: - return render_template('/dalton/sensor.html', page='', sensors=sensors) + return render_template("/dalton/sensor.html", page="", sensors=sensors) + -# validates passed in filename (should be from Flowsynth) to verify -# that it exists and isn't trying to do something nefarious like -# directory traversal def verify_fs_pcap(fspcap): - global FS_PCAP_PATH + """Validate the filename for the pcap. + + Validates passed in filename (should be from Flowsynth) to verify + that it exists and isn't trying to do something nefarious like + directory traversal. + """ # require fspcap to be POSIX fully portable filename if not re.match(r"^[A-Za-z0-9\x5F\x2D\x2E]+$", fspcap): - logger.error("Bad fspcap filename provided: '%s'. Filename must be POSIX fully portable." % fspcap) + logger.error( + "Bad fspcap filename provided: '%s'. Filename must be POSIX fully portable." + % fspcap + ) return "Bad pcap filename provided: '%s'" % (fspcap) fspcap_path = os.path.join(FS_PCAP_PATH, os.path.basename(fspcap)) logger.debug("Flowsynth pcap file passed: %s" % fspcap_path) @@ -865,109 +1121,149 @@ def verify_fs_pcap(fspcap): return "File not found: '%s'" % os.path.basename(fspcap) return None -"""validate that job_id has expected characters; prevent directory traversal""" + def validate_jobid(jid): - if not re.match (r'^(teapot_)?[a-zA-Z\d]+$', jid): + """validate that job_id has expected characters; prevent directory traversal""" + if not re.match(r"^(teapot_)?[a-zA-Z\d]+$", jid): return False else: return True -@dalton_blueprint.route('/dalton/coverage/job/', methods=['GET']) +@dalton_blueprint.route("/dalton/coverage/job/", methods=["GET"]) def page_coverage_jid(jid, error=None): - global JOB_STORAGE_PATH - global TEMP_STORAGE_PATH - global RULESET_STORAGE_PATH + redis = get_redis() if not re.match(r"^[a-f0-9]{16}$", jid): - return render_template('/dalton/error.html', jid='', msg=["Not a valid job ID."]) + return render_template( + "/dalton/error.html", jid="", msg=["Not a valid job ID."] + ) jobzip_path = os.path.join(f"{JOB_STORAGE_PATH}", f"{jid}.zip") if not os.path.isfile(jobzip_path): - return render_template('/dalton/error.html', jid=jid, msg=[f"Job with ID {jid} does not exist."]) + return render_template( + "/dalton/error.html", jid=jid, msg=[f"Job with ID {jid} does not exist."] + ) custom_rules = None with zipfile.ZipFile(jobzip_path) as zf: - manifest = json.loads(zf.read('manifest.json').decode()) - sensor_tech = manifest['sensor-tech'].split('/')[0] + manifest = json.loads(zf.read("manifest.json").decode()) + sensor_tech = manifest["sensor-tech"].split("/")[0] for f in zf.namelist(): - if f.endswith(f".conf") or f.endswith(f".yaml"): + if f.endswith(".conf") or f.endswith(".yaml"): engine_conf = zf.read(f).decode() - elif f == "dalton-custom.rules" and manifest['custom-rules'] == True: + elif f == "dalton-custom.rules" and manifest["custom-rules"] is True: custom_rules = zf.read(f).decode() - + # extend job life by moving file mod date into the future, thereby delaying the usual expiry process # subtracting REDIS_EXPIRE so SHARE_EXPIRE matches expectations - # example: SHARE_EXPIRE = 30 days, REDIS_EXPIRE = 5 days. + # example: SHARE_EXPIRE = 30 days, REDIS_EXPIRE = 5 days. # The queue delete logic deletes after now + REDIS_EXPIRE. If we don't subtract it now jobs will last 35 days now = time.time() newtime = now + (SHARE_EXPIRE - REDIS_EXPIRE) - os.utime(jobzip_path, (newtime,newtime)) + os.utime(jobzip_path, (newtime, newtime)) rulesets = get_rulesets(sensor_tech) - if sensor_tech.lower().startswith('zeek'): + if sensor_tech.lower().startswith("zeek"): engine_conf = None # enumerate sensor versions based on available sensors and pass them to coverage.html # This way we can dynamically update the submission page as soon as new sensor versions check in - clear_old_agents() + clear_old_agents(redis) sensors = [] - if r.exists('sensors'): - for sensor in r.smembers('sensors'): + if redis.exists("sensors"): + for sensor in redis.smembers("sensors"): try: - tech = r.get("%s-tech" % sensor) + tech = redis.get("%s-tech" % sensor) if tech.startswith(sensor_tech): if tech not in sensors: sensors.append(tech) except Exception as e: - return render_template('/dalton/error.html', jid=None, msg="Error getting sensor list for %s. Error:\n%s" % (tech, e)) + return render_template( + "/dalton/error.html", + jid=None, + msg="Error getting sensor list for %s. Error:\n%s" % (tech, e), + ) try: # May 2019 - DRW - I'd prefer that non-rust sensors of the same version get listed before # rust enabled sensors so adding this extra sort. Can/should probably be removed in year or two. sensors.sort(reverse=False) # sort by version number; ignore "rust_" prefix - sensors.sort(key=lambda v:LooseVersion(prefix_strip(v.split('/', 2)[1], prefixes=["rust_"])), reverse=True) - except Exception as e: + sensors.sort( + key=lambda v: LooseVersion( + prefix_strip(v.split("/", 2)[1], prefixes=["rust_"]) + ), + reverse=True, + ) + except Exception: try: sensors.sort(key=LooseVersion, reverse=True) - except Exception as ee: + except Exception: sensors.sort(reverse=True) logger.debug(f"In page_coverage_default() - sensors:\n{sensors}") - job_ruleset = manifest.get('prod-ruleset') + job_ruleset = manifest.get("prod-ruleset") if job_ruleset: rulesets.insert(0, [f"{jid} ruleset", jobzip_path]) - return render_template('/dalton/coverage.html', sensor_tech=sensor_tech, rulesets=rulesets, error=error, engine_conf=engine_conf, sensors=sensors, fspcap=None, max_pcaps=MAX_PCAP_FILES, manifest=manifest, custom_rules=custom_rules) - -@dalton_blueprint.route('/dalton/coverage//', methods=['GET']) -#@login_required() + return render_template( + "/dalton/coverage.html", + sensor_tech=sensor_tech, + rulesets=rulesets, + error=error, + engine_conf=engine_conf, + sensors=sensors, + fspcap=None, + max_pcaps=MAX_PCAP_FILES, + manifest=manifest, + custom_rules=custom_rules, + ) + + +@dalton_blueprint.route("/dalton/coverage//", methods=["GET"]) +# @login_required() def page_coverage_default(sensor_tech, error=None): """the default coverage wizard page""" - global CONF_STORAGE_PATH, MAX_PCAP_FILES - global r - ruleset_dirs = [] - sensor_tech = sensor_tech.split('-')[0] + redis = get_redis() + sensor_tech = sensor_tech.split("-")[0] conf_dir = os.path.join(CONF_STORAGE_PATH, clean_path(sensor_tech)) if sensor_tech is None: - return render_template('/dalton/error.html', jid='', msg=["No Sensor technology selected for job."]) + return render_template( + "/dalton/error.html", jid="", msg=["No Sensor technology selected for job."] + ) elif not re.match(r"^[a-zA-Z0-9\_\-\.]+$", sensor_tech): - return render_template('/dalton/error.html', jid='', msg=[f"Invalid Sensor technology requested: {sensor_tech}"]) - elif sensor_tech == 'summary': - return render_template('/dalton/error.html', jid='', msg=["Page expired. Please resubmit your job or access it from the queue."]) - - if not os.path.isdir(conf_dir) and not sensor_tech.startswith('zeek'): - return render_template('/dalton/error.html', jid='', msg=[f"No engine configuration directory for '{sensor_tech}' found ({conf_dir})."]) + return render_template( + "/dalton/error.html", + jid="", + msg=[f"Invalid Sensor technology requested: {sensor_tech}"], + ) + elif sensor_tech == "summary": + return render_template( + "/dalton/error.html", + jid="", + msg=[ + "Page expired. Please resubmit your job or access it from the queue." + ], + ) + + if not os.path.isdir(conf_dir) and not sensor_tech.startswith("zeek"): + return render_template( + "/dalton/error.html", + jid="", + msg=[ + f"No engine configuration directory for '{sensor_tech}' found ({conf_dir})." + ], + ) # pcap filename passed in from Flowsynth fspcap = None try: - fspcap = request.args['fspcap'] + fspcap = request.args["fspcap"] err_msg = verify_fs_pcap(fspcap) - if err_msg != None: - return render_template('/dalton/error.html', jid='', msg=[f"{err_msg}"]) - except: + if err_msg is not None: + return render_template("/dalton/error.html", jid="", msg=[f"{err_msg}"]) + except Exception: fspcap = None # get list of rulesets based on engine @@ -975,27 +1271,36 @@ def page_coverage_default(sensor_tech, error=None): # enumerate sensor versions based on available sensors and pass them to coverage.html # This way we can dynamically update the submission page as soon as new sensor versions check in - clear_old_agents() + clear_old_agents(redis) sensors = [] - if r.exists('sensors'): - for sensor in r.smembers('sensors'): + if redis.exists("sensors"): + for sensor in redis.smembers("sensors"): try: - tech = r.get("%s-tech" % sensor) + tech = redis.get("%s-tech" % sensor) if tech.startswith(sensor_tech): if tech not in sensors: sensors.append(tech) except Exception as e: - return render_template('/dalton/error.html', jid=None, msg="Error getting sensor list for %s. Error:\n%s" % (tech, e)) + return render_template( + "/dalton/error.html", + jid=None, + msg="Error getting sensor list for %s. Error:\n%s" % (tech, e), + ) try: # May 2019 - DRW - I'd prefer that non-rust sensors of the same version get listed before # rust enabled sensors so adding this extra sort. Can/should probably be removed in year or two. sensors.sort(reverse=False) # sort by version number; ignore "rust_" prefix - sensors.sort(key=lambda v:LooseVersion(prefix_strip(v.split('/', 2)[1], prefixes=["rust_"])), reverse=True) - except Exception as e: + sensors.sort( + key=lambda v: LooseVersion( + prefix_strip(v.split("/", 2)[1], prefixes=["rust_"]) + ), + reverse=True, + ) + except Exception: try: sensors.sort(key=LooseVersion, reverse=True) - except Exception as ee: + except Exception: sensors.sort(reverse=True) logger.debug(f"In page_coverage_default() - sensors:\n{sensors}") # get conf or yaml file if sensor supports it @@ -1008,65 +1313,89 @@ def page_coverage_default(sensor_tech, error=None): logger.debug("call to get_engine_conf_file(%s)", sensors[0]) engine_conf = get_engine_conf_file(sensors[0]) except Exception as e: - logger.error("Could not process response from get_engine_conf_file(): %s", e) + logger.error( + "Could not process response from get_engine_conf_file(): %s", e + ) engine_conf = "# not found" else: # no sensors available. engine_conf = "# not found" - return render_template('/dalton/coverage.html', sensor_tech=sensor_tech, rulesets=rulesets, error=error, engine_conf=engine_conf, sensors=sensors, fspcap=fspcap, max_pcaps=MAX_PCAP_FILES) - -@dalton_blueprint.route('/dalton/job/') -#@auth_required() + return render_template( + "/dalton/coverage.html", + sensor_tech=sensor_tech, + rulesets=rulesets, + error=error, + engine_conf=engine_conf, + sensors=sensors, + fspcap=fspcap, + max_pcaps=MAX_PCAP_FILES, + ) + + +@dalton_blueprint.route("/dalton/job/") +# @auth_required() def page_show_job(jid): - global r - tech = r.get("%s-tech" % jid) - status = get_job_status(jid) + redis = get_redis() + tech = redis.get("%s-tech" % jid) + status = get_job_status(redis, jid) if not status: # job doesn't exist # expire (delete) all keys related to the job just in case to prevent memory leaks - expire_all_keys(jid) - return render_template('/dalton/error.html', jid=jid, msg=["Invalid Job ID. Job may have expired.", "By default, jobs are only kept for %d seconds; teapot jobs are kept for %s seconds." % (REDIS_EXPIRE, TEAPOT_REDIS_EXPIRE)]) + expire_all_keys(redis, jid) + return render_template( + "/dalton/error.html", + jid=jid, + msg=[ + "Invalid Job ID. Job may have expired.", + "By default, jobs are only kept for %d seconds; teapot jobs are kept for %s seconds." + % (REDIS_EXPIRE, TEAPOT_REDIS_EXPIRE), + ], + ) elif int(status) != STAT_CODE_DONE: # job is queued or running - return render_template('/dalton/coverage-summary.html', page='', job_id=jid, tech=tech) + return render_template( + "/dalton/coverage-summary.html", page="", job_id=jid, tech=tech + ) else: # job exists and is done - ids = r.get(f"{jid}-ids") - perf = r.get(f"{jid}-perf") - alert = r.get(f"{jid}-alert") - error = r.get(f"{jid}-error") - total_time = r.get(f"{jid}-time") - alert_detailed = r.get(f"{jid}-alert_detailed") + ids = redis.get(f"{jid}-ids") + perf = redis.get(f"{jid}-perf") + alert = redis.get(f"{jid}-alert") + error = redis.get(f"{jid}-error") + total_time = redis.get(f"{jid}-time") + alert_detailed = redis.get(f"{jid}-alert_detailed") try: - zeek_json = r.get(f"{jid}-zeek_json") - except Exception as e: - #logger.debug(f"Problem getting {jid}-zeek_json:\n{e}") + zeek_json = redis.get(f"{jid}-zeek_json") + except Exception: + # logger.debug(f"Problem getting {jid}-zeek_json:\n{e}") zeek_json = "False" try: # this gets passed as json with log description as key and log contents as value # attempt to load it as json before we pass it to job.html - other_logs = json.loads(r.get(f"{jid}-other_logs")) - if tech.startswith('zeek') and zeek_json == "False": + other_logs = json.loads(redis.get(f"{jid}-other_logs")) + if tech.startswith("zeek") and zeek_json == "False": for other_log in other_logs: other_logs[other_log] = parseZeekASCIILog(other_logs[other_log]) - except Exception as e: + except Exception: # if -other_logs is empty then error, "No JSON object could be decoded" will be thrown so just handling it cleanly other_logs = "" - #logger.error("could not load json other_logs:\n%s\n\nvalue:\n%s" % (e,r.get("%s-other_logs" % jid))) + # logger.error("could not load json other_logs:\n%s\n\nvalue:\n%s" % (e,r.get("%s-other_logs" % jid))) try: - eve = r.get(f"{jid}-eve") - except Exception as e: - #logger.debug(f"Problem getting {jid}-eve log:\n{e}") + eve = redis.get(f"{jid}-eve") + except Exception: + # logger.debug(f"Problem getting {jid}-eve log:\n{e}") eve = "" event_types = [] if len(eve) > 0: # pull out all the EVE event types try: eve_list = [json.loads(line) for line in eve.splitlines()] - event_types = set([item['event_type'] for item in eve_list if 'event_type' in item]) + event_types = set( + [item["event_type"] for item in eve_list if "event_type" in item] + ) if len(event_types) > 0: event_types = sorted(event_types) except Exception as e: @@ -1074,65 +1403,102 @@ def page_show_job(jid): # parse out custom rules option and pass it? custom_rules = False try: - debug = r.get("%s-debug" % jid) - except Exception as e: - debug = '' + debug = redis.get("%s-debug" % jid) + except Exception: + debug = "" overview = {} - if (alert != None): - overview['alert_count'] = get_alert_count(jid) + if alert is not None: + overview["alert_count"] = get_alert_count(redis, jid) else: - overview['alert_count'] = 0 - if (error == ""): - overview['status'] = 'Success' + overview["alert_count"] = 0 + if error == "": + overview["status"] = "Success" else: - overview['status'] = 'Error' + overview["status"] = "Error" + + return render_template( + "/dalton/job.html", + overview=overview, + page="", + jobid=jid, + ids=ids, + perf=perf, + alert=alert, + error=error, + debug=debug, + total_time=total_time, + tech=tech, + custom_rules=custom_rules, + alert_detailed=alert_detailed, + other_logs=other_logs, + eve_json=eve, + event_types=event_types, + zeek_json=zeek_json, + ) - return render_template('/dalton/job.html', overview=overview,page = '', - jobid = jid, ids=ids, perf=perf, alert=alert, - error=error, debug=debug, total_time=total_time, - tech=tech, custom_rules=custom_rules, - alert_detailed=alert_detailed, other_logs=other_logs, - eve_json=eve, event_types=event_types, zeek_json=zeek_json) -# sanitize passed in filename (string) and make it POSIX (fully portable) def clean_filename(filename): + """sanitize passed in filename (string) and make it POSIX (fully portable).""" return re.sub(r"[^a-zA-Z0-9\_\-\.]", "_", filename) -# handle duplicate filenames (e.g. same pcap submitted more than once) -# by renaming pcaps with same name + def handle_dup_names(filename, pcap_files, job_id, dupcount): + """Handle duplicate filenames. + + Handle duplicate filenames (e.g. same pcap submitted more than once) + by renaming pcaps with same name. + """ for pcap in pcap_files: - if pcap['filename'] == filename: - filename = "%s_%s_%d.pcap" % (os.path.splitext(filename)[0], job_id, dupcount[0]) + if pcap["filename"] == filename: + filename = "%s_%s_%d.pcap" % ( + os.path.splitext(filename)[0], + job_id, + dupcount[0], + ) dupcount[0] += 1 break return filename -# extracts files from an archive and add them to the list to be -# included with the Dalton job + def extract_pcaps(archivename, pcap_files, job_id, dupcount): - global TEMP_STORAGE_PATH + """Extract the packet capture files. + + extracts files from an archive and add them to the list to be + included with the Dalton job. + """ # Note: archivename already sanitized - logger.debug("Attempting to extract pcaps from file '%s'" % os.path.basename(archivename)) - if archivename.lower().endswith('.zip'): + logger.debug( + "Attempting to extract pcaps from file '%s'" % os.path.basename(archivename) + ) + if archivename.lower().endswith(".zip"): # Apparently python zipfile module does extraction using Python and not something # like C and it is super slow for a zipfile that isn't small in size. So # to speed things up, kick out to 7z on the system which is quite fast but not my # first choice. Still use zipfile module to process archive and get filenames. try: if not zipfile.is_zipfile(archivename): - msg = "File '%s' is not recognized as a valid zip file." % os.path.basename(archivename) + msg = ( + "File '%s' is not recognized as a valid zip file." + % os.path.basename(archivename) + ) logger.error(msg) return msg files_to_extract = [] - zf = zipfile.ZipFile(archivename, mode='r') + zf = zipfile.ZipFile(archivename, mode="r") for file in zf.namelist(): logger.debug("Processing file '%s' from ZIP archive" % file) - if file.endswith('/') or "__MACOSX/" in file: + if file.endswith("/") or "__MACOSX/" in file: continue filename = clean_filename(os.path.basename(file)) - if os.path.splitext(filename)[1].lower() not in ['.pcap', '.pcapng', '.cap']: - logger.warn("Not adding file '%s' from archive '%s': '.pcap', '.cap', or '.pcapng' extension required." % (file, os.path.basename(archivename))) + if os.path.splitext(filename)[1].lower() not in [ + ".pcap", + ".pcapng", + ".cap", + ]: + logger.warning( + "Not adding file '%s' from archive '%s': '.pcap', '.cap', or '.pcapng' extension required." + % (file, os.path.basename(archivename)) + ) # just skip the file, and move on (and log it) continue files_to_extract.append(file) @@ -1143,13 +1509,28 @@ def extract_pcaps(archivename, pcap_files, job_id, dupcount): tempd = tempfile.mkdtemp() logger.debug("temp directory for 7z: %s" % tempd) # try password 'infected' if password on archive - p7z_command = ['7z', 'x', archivename, '-pinfected', '-y', "-o%s" % tempd] + files_to_extract + p7z_command = [ + "7z", + "x", + archivename, + "-pinfected", + "-y", + "-o%s" % tempd, + ] + files_to_extract # does 7z handle invalid/filenames or should more sanitization be attempted? logger.debug("7z command: %s" % p7z_command) # I'm not convinced that 7z outputs to stderr - p7z_out = subprocess.Popen(p7z_command, shell=False, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).stdout.read() + p7z_out = subprocess.Popen( + p7z_command, + shell=False, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + ).stdout.read() if b"Everything is Ok" not in p7z_out and b"Errors: " in p7z_out: - logger.error("Problem extracting ZIP archive '%s': %s" % (os.path.basename(archivename), p7z_out)) + logger.error( + "Problem extracting ZIP archive '%s': %s" + % (os.path.basename(archivename), p7z_out) + ) raise Exception("p7zip error. See logs for details") logger.debug("7z out: %s" % p7z_out) @@ -1161,51 +1542,66 @@ def extract_pcaps(archivename, pcap_files, job_id, dupcount): pcapsrc = os.path.join(tempd, file) # copy shutil.move(pcapsrc, pcappath) - pcap_files.append({'filename': filename, 'pcappath': pcappath}) - logger.debug("Successfully extracted and added pcap file '%s'" % os.path.basename(filename)) + pcap_files.append({"filename": filename, "pcappath": pcappath}) + logger.debug( + "Successfully extracted and added pcap file '%s'" + % os.path.basename(filename) + ) # cleanup shutil.rmtree(tempd) except Exception as e: - msg = "Problem extracting ZIP file '%s': %s" % (os.path.basename(archivename), e) + msg = "Problem extracting ZIP file '%s': %s" % ( + os.path.basename(archivename), + e, + ) logger.error(msg) logger.debug("%s" % traceback.format_exc()) return msg - elif os.path.splitext(archivename)[1].lower() in ['.gz', '.gzip'] and \ - os.path.splitext(os.path.splitext(archivename)[0])[1].lower() not in ['.tar']: + elif os.path.splitext(archivename)[1].lower() in [ + ".gz", + ".gzip", + ] and os.path.splitext(os.path.splitext(archivename)[0])[1].lower() not in [".tar"]: # gzipped file try: - filename = os.path.basename(os.path.splitext(archivename)[0]) + filename = os.path.basename(os.path.splitext(archivename)[0]) logger.debug("Decompressing gzipped file '%s'" % filename) - with gzip.open(archivename, 'rb') as gz: + with gzip.open(archivename, "rb") as gz: filename = handle_dup_names(filename, pcap_files, job_id, dupcount) pcappath = os.path.join(TEMP_STORAGE_PATH, job_id, filename) - fh = open(pcappath, 'wb') + fh = open(pcappath, "wb") fh.write(gz.read()) fh.close() - pcap_files.append({'filename': filename, 'pcappath': pcappath}) + pcap_files.append({"filename": filename, "pcappath": pcappath}) logger.debug("Added %s" % filename) except Exception as e: - msg = "Problem extracting gzip file '%s': %s" % (os.path.basename(archivename), e) + msg = "Problem extracting gzip file '%s': %s" % ( + os.path.basename(archivename), + e, + ) logger.error(msg) logger.debug("%s" % traceback.format_exc()) return msg - elif os.path.splitext(archivename)[1].lower() in ['.bz2'] and \ - os.path.splitext(os.path.splitext(archivename)[0])[1].lower() not in ['.tar']: + elif os.path.splitext(archivename)[1].lower() in [".bz2"] and os.path.splitext( + os.path.splitext(archivename)[0] + )[1].lower() not in [".tar"]: # bzip2 file try: - filename = os.path.basename(os.path.splitext(archivename)[0]) + filename = os.path.basename(os.path.splitext(archivename)[0]) logger.debug("Decompressing bzip2 file '%s'" % filename) - with bz2.BZ2File(archivename, 'rb') as bz: + with bz2.BZ2File(archivename, "rb") as bz: filename = handle_dup_names(filename, pcap_files, job_id, dupcount) pcappath = os.path.join(TEMP_STORAGE_PATH, job_id, filename) - fh = open(pcappath, 'wb') + fh = open(pcappath, "wb") fh.write(bz.read()) fh.close() - pcap_files.append({'filename': filename, 'pcappath': pcappath}) + pcap_files.append({"filename": filename, "pcappath": pcappath}) logger.debug("Added %s" % filename) except Exception as e: - msg = "Problem extracting bzip2 file '%s': %s" % (os.path.basename(archivename), e) + msg = "Problem extracting bzip2 file '%s': %s" % ( + os.path.basename(archivename), + e, + ) logger.error(msg) logger.debug("%s" % traceback.format_exc()) return msg @@ -1215,29 +1611,43 @@ def extract_pcaps(archivename, pcap_files, job_id, dupcount): for file in archive.getmembers(): logger.debug("Processing file '%s' from archive" % file.name) if not file.isfile(): - logger.warn("Not adding member '%s' from archive '%s': not a file." % (file.name, os.path.basename(archivename))) + logger.warning( + "Not adding member '%s' from archive '%s': not a file." + % (file.name, os.path.basename(archivename)) + ) continue filename = clean_filename(os.path.basename(file.name)) - if os.path.splitext(filename)[1].lower() not in ['.pcap', '.pcapng', '.cap']: - logger.warn("Not adding file '%s' from archive '%s': '.pcap', '.cap', or '.pcapng' extension required." % (file.name, os.path.basename(archivename))) + if os.path.splitext(filename)[1].lower() not in [ + ".pcap", + ".pcapng", + ".cap", + ]: + logger.warning( + "Not adding file '%s' from archive '%s': '.pcap', '.cap', or '.pcapng' extension required." + % (file.name, os.path.basename(archivename)) + ) # just skip the file, and move on (and log it) continue filename = handle_dup_names(filename, pcap_files, job_id, dupcount) pcappath = os.path.join(TEMP_STORAGE_PATH, job_id, filename) - fh = open(pcappath, 'wb') + fh = open(pcappath, "wb") contentsfh = archive.extractfile(file) fh.write(contentsfh.read()) fh.close() - pcap_files.append({'filename': filename, 'pcappath': pcappath}) + pcap_files.append({"filename": filename, "pcappath": pcappath}) logger.debug("Added %s" % filename) archive.close() except Exception as e: - msg = "Problem extracting archive file '%s': %s" % (os.path.basename(archivename), e) + msg = "Problem extracting archive file '%s': %s" % ( + os.path.basename(archivename), + e, + ) logger.error(msg) logger.debug("%s" % traceback.format_exc()) return msg return None + # abstracting the job submission method away from the HTTP POST and creating this # function so that it can be called easier (e.g. from an API) def submit_job(): @@ -1246,20 +1656,14 @@ def submit_job(): # TODO: API call that accepts a job zipfile and queues it up for an agent? # would have to beef up input validation on agent probably.... -@dalton_blueprint.route('/dalton/coverage/summary', methods=['POST']) -#@auth_required() + +@dalton_blueprint.route("/dalton/coverage/summary", methods=["POST"]) +# @auth_required() # ^^ can change and add resource and group permissions if we want to restrict who can submit jobs def page_coverage_summary(): - """ Handle job submission from UI. - """ + """Handle job submission from UI.""" # user submitting a job to Dalton via the web interface - global JOB_STORAGE_PATH - global TEMP_STORAGE_PATH - global RULESET_STORAGE_PATH - global r - global STAT_CODE_QUEUED - global FS_PCAP_PATH - global MAX_PCAP_FILES + redis = get_redis() verify_temp_storage_path() digest = hashlib.md5() @@ -1269,10 +1673,10 @@ def page_coverage_summary(): # get the user who submitted the job .. not implemented user = "undefined" - #generate job_id based of pcap filenames and timestamp - digest.update(str(datetime.datetime.now()).encode('utf-8')) - digest.update(str(random.randrange(96313375)).encode('utf-8')) - job_id = digest.hexdigest()[0:16] #this is a temporary job id for the filename + # generate job_id based of pcap filenames and timestamp + digest.update(str(datetime.datetime.now()).encode("utf-8")) + digest.update(str(random.randrange(96313375)).encode("utf-8")) + job_id = digest.hexdigest()[0:16] # this is a temporary job id for the filename # store the pcaps offline temporarily # make temp job directory so there isn't a race condition if more @@ -1290,11 +1694,16 @@ def page_coverage_summary(): err_msg = verify_fs_pcap(fspcap) if err_msg: delete_temp_files(job_id) - return render_template('/dalton/error.html', jid='', msg=[err_msg]) - pcap_files.append({'filename': fspcap, 'pcappath': os.path.join(FS_PCAP_PATH, os.path.basename(fspcap))}) - - #zeek-customscript - uiupload_file_path = os.path.join(SCRIPT_STORAGE_PATH, 'uiupload.zeek') + return render_template("/dalton/error.html", jid="", msg=[err_msg]) + pcap_files.append( + { + "filename": fspcap, + "pcappath": os.path.join(FS_PCAP_PATH, os.path.basename(fspcap)), + } + ) + + # zeek-customscript + uiupload_file_path = os.path.join(SCRIPT_STORAGE_PATH, "uiupload.zeek") if os.path.isfile(uiupload_file_path): os.remove(uiupload_file_path) @@ -1305,23 +1714,23 @@ def page_coverage_summary(): bCustomscriptfile = True file.save(uiupload_file_path) - uiwrite_file_path = os.path.join(SCRIPT_STORAGE_PATH, 'uiwrite.zeek') + uiwrite_file_path = os.path.join(SCRIPT_STORAGE_PATH, "uiwrite.zeek") checkbox_value = request.form.get("optionCustomScript") - custom_script = request.form.get('custom_script', '') + custom_script = request.form.get("custom_script", "") if not checkbox_value or not custom_script: if os.path.isfile(uiwrite_file_path): os.remove(uiwrite_file_path) bCustomscriptwrite = False if checkbox_value and custom_script: bCustomscriptwrite = True - with open(uiwrite_file_path, 'w') as file: + with open(uiwrite_file_path, "w") as file: file.write(custom_script) - + bSplitCap = False try: if request.form.get("optionSplitcap"): bSplitCap = True - except: + except Exception: pass # grab the user submitted files from the web form (max number of arbitrary files allowed on the web form @@ -1330,141 +1739,235 @@ def page_coverage_summary(): # make this a list so I can pass by reference dupcount = [0] job_zip = request.form.get("job-zip") - if (job_zip != None and re.match(r"^[a-f0-9]{16}\.zip$", job_zip)): + if job_zip is not None and re.match(r"^[a-f0-9]{16}\.zip$", job_zip): filename = clean_filename(os.path.basename(job_zip)) filepath = os.path.join(JOB_STORAGE_PATH, filename) if not os.path.isfile(filepath): - return render_template('/dalton/error.html', jid='', msg=[f"Zip file for {filename} does not exist"]) + return render_template( + "/dalton/error.html", + jid="", + msg=[f"Zip file for {filename} does not exist"], + ) err_msg = extract_pcaps(filepath, pcap_files, job_id, dupcount) if err_msg: delete_temp_files(job_id) - return render_template('/dalton/error.html', jid='', msg=[err_msg]) + return render_template("/dalton/error.html", jid="", msg=[err_msg]) for i in range(MAX_PCAP_FILES): try: coverage_pcaps = request.files.getlist("coverage-pcap%d" % i) for pcap_file in coverage_pcaps: - if (pcap_file != None and pcap_file.filename != None and pcap_file.filename != '' and (len(pcap_file.filename) > 0) ): - if os.path.splitext(pcap_file.filename)[1].lower() in ['.zip', '.tar', '.gz', '.tgz', '.gzip', '.bz2']: + if ( + pcap_file is not None + and pcap_file.filename is not None + and pcap_file.filename != "" + and (len(pcap_file.filename) > 0) + ): + if os.path.splitext(pcap_file.filename)[1].lower() in [ + ".zip", + ".tar", + ".gz", + ".tgz", + ".gzip", + ".bz2", + ]: filename = clean_filename(os.path.basename(pcap_file.filename)) filename = os.path.join(TEMP_STORAGE_PATH, job_id, filename) pcap_file.save(filename) err_msg = extract_pcaps(filename, pcap_files, job_id, dupcount) if err_msg: delete_temp_files(job_id) - return render_template('/dalton/error.html', jid='', msg=[err_msg]) + return render_template( + "/dalton/error.html", jid="", msg=[err_msg] + ) else: form_pcap_files.append(pcap_file) - except: + except Exception: logger.debug("%s" % traceback.format_exc()) pass - #get the sensor technology and queue name - sensor_tech = request.form.get('sensor_tech') + # get the sensor technology and queue name + sensor_tech = request.form.get("sensor_tech") - #verify that we have a sensor that can handle the submitted sensor_tech + # verify that we have a sensor that can handle the submitted sensor_tech valid_sensor_tech = False - if r.exists('sensors'): - for sensor in r.smembers('sensors'): - if r.get("%s-tech" % sensor) == sensor_tech: + if redis.exists("sensors"): + for sensor in redis.smembers("sensors"): + if redis.get("%s-tech" % sensor) == sensor_tech: valid_sensor_tech = True break if not valid_sensor_tech: - logger.error("Dalton in page_coverage_summary(): Error: user %s submitted a job for invalid sensor tech, '%s'", user, sensor_tech) + logger.error( + "Dalton in page_coverage_summary(): Error: user %s submitted a job for invalid sensor tech, '%s'", + user, + sensor_tech, + ) delete_temp_files(job_id) - return render_template('/dalton/error.html', jid='', msg=[f"There are no sensors that support sensor technology '{sensor_tech}'."]) + return render_template( + "/dalton/error.html", + jid="", + msg=[ + f"There are no sensors that support sensor technology '{sensor_tech}'." + ], + ) if len(form_pcap_files) == 0 and len(pcap_files) == 0: - #throw an error, no pcaps submitted + # throw an error, no pcaps submitted delete_temp_files(job_id) - return render_template('/dalton/error.html', jid='', msg=["You must specify a PCAP file."]) - elif (request.form.get('optionProdRuleset') == None and request.form.get('optionCustomRuleset') == None) and not sensor_tech.startswith('zeek'): - #throw an error, no rules defined + return render_template( + "/dalton/error.html", jid="", msg=["You must specify a PCAP file."] + ) + elif ( + request.form.get("optionProdRuleset") is None + and request.form.get("optionCustomRuleset") is None + ) and not sensor_tech.startswith("zeek"): + # throw an error, no rules defined delete_temp_files(job_id) - return render_template('/dalton/error.html', jid='', msg=["You must specify at least one ruleset."]) + return render_template( + "/dalton/error.html", jid="", msg=["You must specify at least one ruleset."] + ) else: # process files from web form for pcap_file in form_pcap_files: filename = os.path.basename(pcap_file.filename) # do some input validation on the filename and try to do some accommodation to preserve original pcap filename filename = clean_filename(filename) - if os.path.splitext(filename)[1] != '.pcap': - filename = f"{filename}.pcap" + if os.path.splitext(filename)[1] != ".pcap": + filename = f"{filename}.pcap" # handle duplicate filenames (e.g. same pcap submitted more than once) filename = handle_dup_names(filename, pcap_files, job_id, dupcount) pcappath = os.path.join(TEMP_STORAGE_PATH, job_id, filename) - pcap_files.append({'filename': filename, 'pcappath': pcappath}) + pcap_files.append({"filename": filename, "pcappath": pcappath}) pcap_file.save(pcappath) (sensor_tech_engine, sensor_tech_version) = get_engine_and_version(sensor_tech) if sensor_tech_engine is None or sensor_tech_version is None: - logger.error("Dalton in page_coverage_summary(): Error: user %s submitted a job with invalid sensor tech string, '%s'", user, sensor_tech) + logger.error( + "Dalton in page_coverage_summary(): Error: user %s submitted a job with invalid sensor tech string, '%s'", + user, + sensor_tech, + ) delete_temp_files(job_id) - return render_template('/dalton/error.html', jid='', msg=[f"Bad sensor_tech string submitted: '{sensor_tech}'."]) + return render_template( + "/dalton/error.html", + jid="", + msg=[f"Bad sensor_tech string submitted: '{sensor_tech}'."], + ) # If multiple files submitted to Suricata, merge them here if the # Suricata version is < 4.1 since that is when support for multiple pcaps # was added. - if len(pcap_files) > 1 and sensor_tech.startswith("suri") and LooseVersion(sensor_tech_version) < LooseVersion("4.1") and not bSplitCap: + if ( + len(pcap_files) > 1 + and sensor_tech.startswith("suri") + and LooseVersion(sensor_tech_version) < LooseVersion("4.1") + and not bSplitCap + ): if not MERGECAP_BINARY: - logger.error("No mergecap binary; unable to merge pcaps for Suricata job.") + logger.error( + "No mergecap binary; unable to merge pcaps for Suricata job." + ) delete_temp_files(job_id) - return render_template('/dalton/error.html', jid=job_id, msg=["No mergecap binary found on Dalton Controller.", "Unable to process multiple pcaps for this Suricata job."]) - combined_file = "%s/combined-%s.pcap" % (os.path.join(TEMP_STORAGE_PATH, job_id), job_id) + return render_template( + "/dalton/error.html", + jid=job_id, + msg=[ + "No mergecap binary found on Dalton Controller.", + "Unable to process multiple pcaps for this Suricata job.", + ], + ) + combined_file = "%s/combined-%s.pcap" % ( + os.path.join(TEMP_STORAGE_PATH, job_id), + job_id, + ) mergecap_command = f"{MERGECAP_BINARY} -w {combined_file} -a -F pcap {' '.join([p['pcappath'] for p in pcap_files])}" - logger.debug("Multiple pcap files submitted to Suricata, combining the following into one file: %s", ', '.join([p['filename'] for p in pcap_files])) + logger.debug( + "Multiple pcap files submitted to Suricata, combining the following into one file: %s", + ", ".join([p["filename"] for p in pcap_files]), + ) try: # validation on pcap filenames done above; otherwise OS command injection here - mergecap_output = subprocess.Popen(mergecap_command, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE).stdout.read() + mergecap_output = subprocess.Popen( + mergecap_command, + shell=True, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + ).stdout.read() if len(mergecap_output) > 0: # return error? - logger.error("Error merging pcaps with command:\n%s\n\nOutput:\n%s", mergecap_command, mergecap_output) + logger.error( + "Error merging pcaps with command:\n%s\n\nOutput:\n%s", + mergecap_command, + mergecap_output, + ) delete_temp_files(job_id) - return render_template('/dalton/error.html', jid="", msg=["Error merging pcaps with command:", f"{mergecap_command}", "Output:", f"{mergecap_output}"]) - pcap_files = [{'filename': os.path.basename(combined_file), 'pcappath': combined_file}] + return render_template( + "/dalton/error.html", + jid="", + msg=[ + "Error merging pcaps with command:", + f"{mergecap_command}", + "Output:", + f"{mergecap_output}", + ], + ) + pcap_files = [ + { + "filename": os.path.basename(combined_file), + "pcappath": combined_file, + } + ] except Exception as e: - logger.error("Could not merge pcaps. Error: %s", e) + logger.error("Could not merge pcaps. Error: %s", e) delete_temp_files(job_id) - return render_template('/dalton/error.html', jid='', msg=["Could not merge pcaps. Error:", f"{e}"]) + return render_template( + "/dalton/error.html", + jid="", + msg=["Could not merge pcaps. Error:", f"{e}"], + ) # get use Suricata Socket Control option bSuricataSC = False - if sensor_tech.startswith("suri") : + if sensor_tech.startswith("suri"): try: if request.form.get("optionUseSC"): bSuricataSC = True - except: + except Exception: pass # get enable all rules option bEnableAllRules = False - if request.form.get('optionProdRuleset') and request.form.get('optionEnableAllRules'): + if request.form.get("optionProdRuleset") and request.form.get( + "optionEnableAllRules" + ): bEnableAllRules = True # get showFlowbitAlerts option bShowFlowbitAlerts = False - if request.form.get('optionProdRuleset') and request.form.get('optionShowFlowbitAlerts'): + if request.form.get("optionProdRuleset") and request.form.get( + "optionShowFlowbitAlerts" + ): bShowFlowbitAlerts = True # get track performance option bTrackPerformance = False - if request.form.get('optionPerf'): + if request.form.get("optionPerf"): bTrackPerformance = True # get return engine statistics option bGetEngineStats = False try: - if request.form.get('optionStats'): + if request.form.get("optionStats"): bGetEngineStats = True - except: + except Exception: pass # get generate fast pattern option - bGetFastPattern = False + bGetFastPattern = False try: - if request.form.get('optionFastPattern'): + if request.form.get("optionFastPattern"): bGetFastPattern = True - except: + except Exception: pass # A 'teapot' job is one that shouldn't be stored for a long period of time; it can be used by @@ -1474,9 +1977,9 @@ def page_coverage_summary(): bteapotJob = False # if teapotJob is set, set 'bteapotJob' to 'True' try: - if request.form.get('teapotJob'): + if request.form.get("teapotJob"): bteapotJob = True - except: + except Exception: pass # used to tell the agent to return pcap data from alerts. @@ -1484,111 +1987,177 @@ def page_coverage_summary(): # and return pcap details from them. bGetAlertDetailed = False try: - if request.form.get('optionAlertDetailed'): + if request.form.get("optionAlertDetailed"): bGetAlertDetailed = True - if sensor_tech_engine == "suricata" and int(sensor_tech_version.split('.')[0]) >= 6: + if ( + sensor_tech_engine == "suricata" + and int(sensor_tech_version.split(".")[0]) >= 6 + ): bGetAlertDetailed = False - except: + except Exception: pass # generate EVE log (only supported by Suricata) bGetEveLog = False try: - if request.form.get('optionEveLog'): + if request.form.get("optionEveLog"): bGetEveLog = True - if sensor_tech_engine == "suricata" and int(sensor_tech_version.split('.')[0]) < 2: + if ( + sensor_tech_engine == "suricata" + and int(sensor_tech_version.split(".")[0]) < 2 + ): bGetEveLog = False - except: + except Exception: pass # get other logs (only supported in Suricata for now) bGetOtherLogs = False try: - if request.form.get('optionOtherLogs'): + if request.form.get("optionOtherLogs"): bGetOtherLogs = True # Dump Buffer option valid for Suri >= version 2.1 and Snort >= 2.9.9.0 - if sensor_tech_engine == "suricata" and LooseVersion(sensor_tech_version) < LooseVersion("2.1"): + if sensor_tech_engine == "suricata" and LooseVersion( + sensor_tech_version + ) < LooseVersion("2.1"): bGetBufferDumps = False - if sensor_tech_engine == "snort" and LooseVersion(sensor_tech_version) < LooseVersion("2.9.9.0"): + if sensor_tech_engine == "snort" and LooseVersion( + sensor_tech_version + ) < LooseVersion("2.9.9.0"): bGetBufferDumps = False - except: + except Exception: pass # get dumps from buffers bGetBufferDumps = False try: - if request.form.get('optionDumpBuffers'): + if request.form.get("optionDumpBuffers"): bGetBufferDumps = True - except: + except Exception: pass # JSON output for Zeek logs boptionZeekJSON = False try: - if request.form.get('optionZeekJSON'): + if request.form.get("optionZeekJSON"): boptionZeekJSON = True - except: + except Exception: pass - #get custom rules (if defined) + # get custom rules (if defined) bCustomRules = False custom_rules_file = os.path.join(TEMP_STORAGE_PATH, f"{job_id}_custom.rules") - if request.form.get('optionCustomRuleset') and request.form.get('custom_ruleset'): + if request.form.get("optionCustomRuleset") and request.form.get( + "custom_ruleset" + ): bCustomRules = True - custom_rules = request.form.get('custom_ruleset') + custom_rules = request.form.get("custom_ruleset") # strip out leading newlines and CRLFCRLF in case the sensor does not like it for some reason - custom_rules = custom_rules.lstrip('\x0A\x0D') - while re.search(r'\x0D\x0A\x0D\x0A', custom_rules): - custom_rules = custom_rules.replace('\x0D\x0A\x0D\x0A', '\x0D\x0A') + custom_rules = custom_rules.lstrip("\x0a\x0d") + while re.search(r"\x0D\x0A\x0D\x0A", custom_rules): + custom_rules = custom_rules.replace("\x0d\x0a\x0d\x0a", "\x0d\x0a") # used for automatically generating SID values for ad-hoc rules that don't include them sid_base = 806421600 sid_offset = 1 # file we will write the custom rules to - fh = open(custom_rules_file, 'w') + fh = open(custom_rules_file, "w") # check for rule errors (very simple right now) - for line in custom_rules.split('\n'): + for line in custom_rules.split("\n"): # strip out trailing whitespace (note: this removes the newline chars too so have to add them back when we write to file) line = line.rstrip() # strip out leading whitespace to make subsequent matching easier (snort won't complain about leading whitespace though) line = line.lstrip() # if empty or comment line, continue - if line == '' or re.search(r'^\s+$', line) or line.startswith('#'): + if line == "" or re.search(r"^\s+$", line) or line.startswith("#"): continue - if (len(line) > 0) and not re.search(r'^[\x00-\x7F]+$', line): + if (len(line) > 0) and not re.search(r"^[\x00-\x7F]+$", line): fh.close() delete_temp_files(job_id) - return render_template('/dalton/error.html', jid='', msg=["Invalid rule. Only ASCII characters are allowed in the literal representation of custom rules.", "Please encode necessary non-ASCII characters appropriately. Rule:", f"{line}"]) + return render_template( + "/dalton/error.html", + jid="", + msg=[ + "Invalid rule. Only ASCII characters are allowed in the literal representation of custom rules.", + "Please encode necessary non-ASCII characters appropriately. Rule:", + f"{line}", + ], + ) # some rule validation for Snort and Suricata - if sensor_tech.startswith('snort') or sensor_tech.startswith('suri'): + if sensor_tech.startswith("snort") or sensor_tech.startswith("suri"): # rule must start with alert|log|pass|activate|dynamic|drop|reject|sdrop - if not re.search(r'^(alert|log|pass|activate|dynamic|drop|reject|sdrop|event_filter|threshold|suppress|rate_filter|detection_filter)\s', line): + if not re.search( + r"^(alert|log|pass|activate|dynamic|drop|reject|sdrop|event_filter|threshold|suppress|rate_filter|detection_filter)\s", + line, + ): fh.close() delete_temp_files(job_id) - return render_template('/dalton/error.html', jid='', msg=[f"Invalid rule, action (first word in rule) of '{line.split()[0]}' not supported. Rule:", f"line"]) + return render_template( + "/dalton/error.html", + jid="", + msg=[ + f"Invalid rule, action (first word in rule) of '{line.split()[0]}' not supported. Rule:", + "line", + ], + ) # rule must end in closing parenthesis - if not line.endswith(')') and not line.startswith("event_filter") and not line.startswith("threshold") \ - and not line.startswith("suppress") and not line.startswith("rate_filter") and not line.startswith("detection_filter"): + if ( + not line.endswith(")") + and not line.startswith("event_filter") + and not line.startswith("threshold") + and not line.startswith("suppress") + and not line.startswith("rate_filter") + and not line.startswith("detection_filter") + ): fh.close() delete_temp_files(job_id) - return render_template('/dalton/error.html', jid='', msg=["Invalid rule; does not end with closing parenthesis. Rule:", f"{line}"]) + return render_template( + "/dalton/error.html", + jid="", + msg=[ + "Invalid rule; does not end with closing parenthesis. Rule:", + f"{line}", + ], + ) # last keyword in the rule must be terminated by a semicolon - if not line[:-1].rstrip().endswith(';') and not line.startswith("event_filter") and not line.startswith("threshold") \ - and not line.startswith("suppress") and not line.startswith("rate_filter") and not line.startswith("detection_filter"): + if ( + not line[:-1].rstrip().endswith(";") + and not line.startswith("event_filter") + and not line.startswith("threshold") + and not line.startswith("suppress") + and not line.startswith("rate_filter") + and not line.startswith("detection_filter") + ): fh.close() delete_temp_files(job_id) - return render_template('/dalton/error.html', jid='', msg=["Invalid rule, last rule option must end with semicolon. Rule:", f"{line}"]) + return render_template( + "/dalton/error.html", + jid="", + msg=[ + "Invalid rule, last rule option must end with semicolon. Rule:", + f"{line}", + ], + ) # add sid if not included - if not re.search(r'(^[^\x28]+\x28\s*|\s|\x3B)sid\s*\:\s*\d+\s*\x3B', line) and not line.startswith("event_filter") and not line.startswith("threshold") \ - and not line.startswith("suppress") and not line.startswith("rate_filter") and not line.startswith("detection_filter"): + if ( + not re.search( + r"(^[^\x28]+\x28\s*|\s|\x3B)sid\s*\:\s*\d+\s*\x3B", line + ) + and not line.startswith("event_filter") + and not line.startswith("threshold") + and not line.startswith("suppress") + and not line.startswith("rate_filter") + and not line.startswith("detection_filter") + ): # if no sid in rule, fix automatically instead of throwing an error - #return render_template('/dalton/error.html', jid='', msg=["\'sid\' not specified in rule, this will error. Rule:", "%s" % line]) - line = re.sub(r'\x29$', " sid:%d;)" % (sid_base + sid_offset), line) + # return render_template('/dalton/error.html', jid='', msg=["\'sid\' not specified in rule, this will error. Rule:", "%s" % line]) + line = re.sub( + r"\x29$", " sid:%d;)" % (sid_base + sid_offset), line + ) sid_offset += 1 # including newline because it was removed earlier with rstrip() fh.write("%s\n" % line) @@ -1596,63 +2165,87 @@ def page_coverage_summary(): if not sensor_tech: delete_temp_files(job_id) - return render_template('/dalton/error.html', jid="", msg=["Variable 'sensor_tech' not specified. Please reload the submission page and try again."]) + return render_template( + "/dalton/error.html", + jid="", + msg=[ + "Variable 'sensor_tech' not specified. Please reload the submission page and try again." + ], + ) # get 'Override External_NET - set to any' option bOverrideExternalNet = False try: - if request.form.get('overrideExternalNet'): + if request.form.get("overrideExternalNet"): bOverrideExternalNet = True - except: + except Exception: pass # pre-set IP vars to add to the config if they don't exist. # this helps with some rulesets that may use these variables # but the variables aren't in the default config. - ipv2add = {'RFC1918': "[10.0.0.0/8,192.168.0.0/16,172.16.0.0/12]" - } + ipv2add = {"RFC1918": "[10.0.0.0/8,192.168.0.0/16,172.16.0.0/12]"} - conf_file = request.form.get('custom_engineconf') - if not conf_file and not sensor_tech.startswith('zeek'): + conf_file = request.form.get("custom_engineconf") + if not conf_file and not sensor_tech.startswith("zeek"): delete_temp_files(job_id) - return render_template('/dalton/error.html', jid='', msg=["No configuration file provided."]) + return render_template( + "/dalton/error.html", jid="", msg=["No configuration file provided."] + ) - bLockConfig = False - - if sensor_tech.startswith('suri'): + if sensor_tech.startswith("suri"): # just in case someone edited and didn't quote a boolean - conf_file = re.sub(r'(\w):\x20+(yes|no)([\x20\x0D\x0A\x23])', '\g<1>: "\g<2>"\g<3>', conf_file) + conf_file = re.sub( + r"(\w):\x20+(yes|no)([\x20\x0D\x0A\x23])", + '\g<1>: "\g<2>"\g<3>', + conf_file, + ) try: # read in yaml - config = yaml.round_trip_load(conf_file, version=(1,1), preserve_quotes=True) + config = yaml.round_trip_load( + conf_file, version=(1, 1), preserve_quotes=True + ) # add some IP vars common to some rulesets try: for v in ipv2add: - if v not in config['vars']['address-groups']: - config['vars']['address-groups'][v] = ipv2add[v] + if v not in config["vars"]["address-groups"]: + config["vars"]["address-groups"][v] = ipv2add[v] except Exception as e: - logger.warn("(Not Fatal) Problem customizing Suricata variables; your YAML may be bad. %s", e) + logger.warning( + "(Not Fatal) Problem customizing Suricata variables; your YAML may be bad. %s", + e, + ) logger.debug(f"{traceback.format_exc()}") # set EXTERNAL_NET to 'any' if option set try: if bOverrideExternalNet: - if not 'EXTERNAL_NET' in config['vars']['address-groups']: - logger.warn("EXTERNAL_NET IP variable not set in config; setting to 'any'") - config['vars']['address-groups']['EXTERNAL_NET'] = 'any' + if "EXTERNAL_NET" not in config["vars"]["address-groups"]: + logger.warning( + "EXTERNAL_NET IP variable not set in config; setting to 'any'" + ) + config["vars"]["address-groups"]["EXTERNAL_NET"] = "any" logger.debug("Set 'EXTERNAL_NET' IP variable to 'any'") except Exception as e: - logger.warn("(Not Fatal) Problem overriding EXTERNAL_NET: %s" % e) + logger.warning( + "(Not Fatal) Problem overriding EXTERNAL_NET: %s" % e + ) logger.debug("%s" % traceback.format_exc()) # first, do rule includes # should references to other rule files be removed? removeOtherRuleFiles = True - if not 'rule-files' in config or removeOtherRuleFiles: - config['rule-files'] = [] - if request.form.get('optionProdRuleset'): + if "rule-files" not in config or removeOtherRuleFiles: + config["rule-files"] = [] + if request.form.get("optionProdRuleset"): # some code re-use here - prod_ruleset_name = os.path.basename(request.form.get('prod_ruleset')) - if prod_ruleset_name.startswith(JOB_STORAGE_PATH) and prod_ruleset_name.endswith('.zip'): - jobzip_path = os.path.join(JOB_STORAGE_PATH, os.path.basename(prod_ruleset_name)) + prod_ruleset_name = os.path.basename( + request.form.get("prod_ruleset") + ) + if prod_ruleset_name.startswith( + JOB_STORAGE_PATH + ) and prod_ruleset_name.endswith(".zip"): + jobzip_path = os.path.join( + JOB_STORAGE_PATH, os.path.basename(prod_ruleset_name) + ) with zipfile.ZipFile(jobzip_path) as zf: for f in zf.namelist(): if f.endswith(".rules") and f != "dalton-custom.rules": @@ -1660,143 +2253,213 @@ def page_coverage_summary(): break elif not prod_ruleset_name.endswith(".rules"): prod_ruleset_name = "%s.rules" % prod_ruleset_name - config['rule-files'].append("%s" % prod_ruleset_name) + config["rule-files"].append("%s" % prod_ruleset_name) if bCustomRules: - config['rule-files'].append("dalton-custom.rules") + config["rule-files"].append("dalton-custom.rules") # remove default rule path; added back on agent - if 'default-rule-path' in config: - config.pop('default-rule-path', None) + if "default-rule-path" in config: + config.pop("default-rule-path", None) # make minimum log level "info" - if "logging" in config and "default-log-level" in config['logging'] and config['logging']['default-log-level'] == "notice": - config['logging']['default-log-level'] = "info" - - for citem in ['outputs', 'logging']: + if ( + "logging" in config + and "default-log-level" in config["logging"] + and config["logging"]["default-log-level"] == "notice" + ): + config["logging"]["default-log-level"] = "info" + + for citem in ["outputs", "logging"]: # set outputs if citem not in config: - logger.warn(f"No '{citem}' section in Suricata YAML. This may be a problem....") + logger.warning( + f"No '{citem}' section in Suricata YAML. This may be a problem...." + ) # going to try to build this from scratch but Suri still may not like it config[f"{citem}"] = [] # apparently with this version of ruamel.yaml and the round trip load, some things aren't # an ordered dict but a list... - llist =[list(config['logging']['outputs'][i].keys())[0] for i in range(0, len(config['logging']['outputs']))] - olist =[list(config['outputs'][i].keys())[0] for i in range(0, len(config['outputs']))] + llist = [ + list(config["logging"]["outputs"][i].keys())[0] + for i in range(0, len(config["logging"]["outputs"])) + ] + olist = [ + list(config["outputs"][i].keys())[0] + for i in range(0, len(config["outputs"])) + ] # Suricata log. Hard code location for use in socket control slog_level = "info" - if 'file' in llist: + if "file" in llist: try: - slog_level = config['logging']['outputs'][llist.index('file')]['file']['level'] + slog_level = config["logging"]["outputs"][llist.index("file")][ + "file" + ]["level"] except Exception as e: - logger.warn("Unable to get log level from config (logging->outputs->file->level): %s" % e) + logger.warning( + "Unable to get log level from config (logging->outputs->file->level): %s" + % e + ) pass - file_config = {'file': {'enabled': True, \ - 'filename': "/tmp/dalton-suricata.log", \ - 'level': f"{slog_level}"}} - if 'file' in llist: - config['logging']['outputs'][llist.index('file')] = file_config + file_config = { + "file": { + "enabled": True, + "filename": "/tmp/dalton-suricata.log", + "level": f"{slog_level}", + } + } + if "file" in llist: + config["logging"]["outputs"][llist.index("file")] = file_config else: - config['logging']['outputs'].append(file_config) + config["logging"]["outputs"].append(file_config) # fast.log - fast_config = {'fast': {'enabled': True, \ - 'filename': "dalton-fast.log", \ - 'append': True}} - if 'fast' in olist: - config['outputs'][olist.index('fast')] = fast_config + fast_config = { + "fast": { + "enabled": True, + "filename": "dalton-fast.log", + "append": True, + } + } + if "fast" in olist: + config["outputs"][olist.index("fast")] = fast_config else: - config['outputs'].append(fast_config) + config["outputs"].append(fast_config) # unified2 logging if bGetAlertDetailed: deployment = "reverse" header = "X-Forwarded-For" - if 'unified2-alert' in olist: + if "unified2-alert" in olist: try: - deployment = config['outputs'][olist.index('unified2-alert')]['unified2-alert']['xff']['deployment'] - except Exception as e: - logger.debug("Could not get outputs->unified2-alert->xff->deployment. Using default value of '%s'" % deployment) + deployment = config["outputs"][ + olist.index("unified2-alert") + ]["unified2-alert"]["xff"]["deployment"] + except Exception: + logger.debug( + "Could not get outputs->unified2-alert->xff->deployment. Using default value of '%s'" + % deployment + ) try: - header = config['outputs'][olist.index('unified2-alert')]['unified2-alert']['xff']['header'] - except Exception as e: - logger.debug("Could not get outputs->unified2-alert->xff->header. Using default value of '%s'" % header) - u2_config = {'unified2-alert': {'enabled': True, \ - 'filename': "unified2.dalton.alert", \ - 'xff': {'enabled': True, 'mode': 'extra-data', \ - 'deployment': deployment, 'header': header}}} - if 'unified2-alert' in olist: - config['outputs'][olist.index('unified2-alert')] = u2_config + header = config["outputs"][olist.index("unified2-alert")][ + "unified2-alert" + ]["xff"]["header"] + except Exception: + logger.debug( + "Could not get outputs->unified2-alert->xff->header. Using default value of '%s'" + % header + ) + u2_config = { + "unified2-alert": { + "enabled": True, + "filename": "unified2.dalton.alert", + "xff": { + "enabled": True, + "mode": "extra-data", + "deployment": deployment, + "header": header, + }, + } + } + if "unified2-alert" in olist: + config["outputs"][olist.index("unified2-alert")] = u2_config else: - config['outputs'].append(u2_config) - - #stats - stats_config = {'stats': {'enabled': True, \ - 'filename': "dalton-stats.log", \ - 'totals': True, \ - 'threads': False}} - if 'stats' in olist: - config['outputs'][olist.index('stats')] = stats_config + config["outputs"].append(u2_config) + + # stats + stats_config = { + "stats": { + "enabled": True, + "filename": "dalton-stats.log", + "totals": True, + "threads": False, + } + } + if "stats" in olist: + config["outputs"][olist.index("stats")] = stats_config else: - config['outputs'].append(stats_config) + config["outputs"].append(stats_config) - - if not "profiling" in config: - config['profiling'] = {} + if "profiling" not in config: + config["profiling"] = {} # always return Engine stats for Suri - config['profiling']['packets'] = {'enabled': True, \ - 'filename': "dalton-packet_stats.log", \ - 'append': True} + config["profiling"]["packets"] = { + "enabled": True, + "filename": "dalton-packet_stats.log", + "append": True, + } if bGetOtherLogs: # alert-debug - alert_debug_config = {'alert-debug': {'enabled': True, \ - 'filename': "dalton-alert_debug.log", \ - 'append': True}} - if 'alert-debug' in olist: - config['outputs'][olist.index('alert-debug')] = alert_debug_config + alert_debug_config = { + "alert-debug": { + "enabled": True, + "filename": "dalton-alert_debug.log", + "append": True, + } + } + if "alert-debug" in olist: + config["outputs"][olist.index("alert-debug")] = ( + alert_debug_config + ) else: - config['outputs'].append(alert_debug_config) + config["outputs"].append(alert_debug_config) # http - http_config = {'http-log': {'enabled': True, \ - 'filename': "dalton-http.log", \ - 'append': True}} - if 'http-log' in olist: - config['outputs'][olist.index('http-log')] = http_config + http_config = { + "http-log": { + "enabled": True, + "filename": "dalton-http.log", + "append": True, + } + } + if "http-log" in olist: + config["outputs"][olist.index("http-log")] = http_config else: - config['outputs'].append(http_config) + config["outputs"].append(http_config) # tls - tls_config = {'tls-log': {'enabled': True, \ - 'filename': "dalton-tls.log", \ - 'append': True}} - if 'tls-log' in olist: - config['outputs'][olist.index('tls-log')] = tls_config + tls_config = { + "tls-log": { + "enabled": True, + "filename": "dalton-tls.log", + "append": True, + } + } + if "tls-log" in olist: + config["outputs"][olist.index("tls-log")] = tls_config else: - config['outputs'].append(tls_config) + config["outputs"].append(tls_config) # dns - dns_config = {'dns-log': {'enabled': True, \ - 'filename': "dalton-dns.log", \ - 'append': True}} + dns_config = { + "dns-log": { + "enabled": True, + "filename": "dalton-dns.log", + "append": True, + } + } # Support for DNS log dropped in Suricata 5 :( if LooseVersion(sensor_tech_version) < LooseVersion("5"): - if 'dns-log' in olist: - config['outputs'][olist.index('dns-log')] = dns_config + if "dns-log" in olist: + config["outputs"][olist.index("dns-log")] = dns_config else: - config['outputs'].append(dns_config) + config["outputs"].append(dns_config) # EVE Log try: if bGetEveLog: # Enable EVE Log - config['outputs'][olist.index('eve-log')]['eve-log']['enabled'] = True + config["outputs"][olist.index("eve-log")]["eve-log"][ + "enabled" + ] = True # set filename - config['outputs'][olist.index('eve-log')]['eve-log']['filename'] = "dalton-eve.json" + config["outputs"][olist.index("eve-log")]["eve-log"][ + "filename" + ] = "dalton-eve.json" # disable EVE TLS logging if Suricata version is < 3.1 which doesn't support multiple # loggers. This mixing of dicts and lists is onerous.... @@ -1806,81 +2469,137 @@ def page_coverage_summary(): # Instead of trying to check everything every time, just catch the exception(s) and move on. The # stuff we want disabled will still get disabled despite the exceptions along the way. if LooseVersion(sensor_tech_version) < LooseVersion("3.1"): - for i in range(0,len(config['outputs'][olist.index('eve-log')]['eve-log']['types'])): + for i in range( + 0, + len( + config["outputs"][olist.index("eve-log")][ + "eve-log" + ]["types"] + ), + ): try: - if list(config['outputs'][olist.index('eve-log')]['eve-log']['types'][i].keys())[0] == 'alert': + if ( + list( + config["outputs"][olist.index("eve-log")][ + "eve-log" + ]["types"][i].keys() + )[0] + == "alert" + ): # apparently this is supported -- http://suricata.readthedocs.io/en/latest/output/eve/eve-json-output.html - config['outputs'][olist.index('eve-log')]['eve-log']['types'][i]['alert'].pop('tls', None) - logger.debug("Removed outputs->eve-log->types->alert->tls") + config["outputs"][olist.index("eve-log")][ + "eve-log" + ]["types"][i]["alert"].pop("tls", None) + logger.debug( + "Removed outputs->eve-log->types->alert->tls" + ) break - except Exception as e: - #logger.debug("Possible issue when removing outputs->eve-log->types->alert->tls (EVE TLS log). Error: %s" % e) + except Exception: + # logger.debug("Possible issue when removing outputs->eve-log->types->alert->tls (EVE TLS log). Error: %s" % e) pass - for i in range(0,len(config['outputs'][olist.index('eve-log')]['eve-log']['types'])): + for i in range( + 0, + len( + config["outputs"][olist.index("eve-log")][ + "eve-log" + ]["types"] + ), + ): try: - if list(config['outputs'][olist.index('eve-log')]['eve-log']['types'][i].keys())[0] == 'tls': - del config['outputs'][olist.index('eve-log')]['eve-log']['types'][i] - logger.debug("Removed outputs->eve-log->types->tls") + if ( + list( + config["outputs"][olist.index("eve-log")][ + "eve-log" + ]["types"][i].keys() + )[0] + == "tls" + ): + del config["outputs"][olist.index("eve-log")][ + "eve-log" + ]["types"][i] + logger.debug( + "Removed outputs->eve-log->types->tls" + ) break - except Exception as e: - #logger.debug("Possible issue when removing outputs->eve-log->types->tls (EVE TLS log). Error: %s" % e) + except Exception: + # logger.debug("Possible issue when removing outputs->eve-log->types->tls (EVE TLS log). Error: %s" % e) pass else: # disable EVE Log if Suricata version supports it - if int(sensor_tech_version.split('.')[0]) >= 2: + if int(sensor_tech_version.split(".")[0]) >= 2: # disable EVE Log here - config['outputs'][olist.index('eve-log')]['eve-log']['enabled'] = False + config["outputs"][olist.index("eve-log")]["eve-log"][ + "enabled" + ] = False except Exception as e: - logger.warn("Problem editing eve-log section of config: %s" % e) + logger.warning("Problem editing eve-log section of config: %s" % e) pass # set filename for rule and keyword profiling if bTrackPerformance: # rule profiling - if not "rules" in config['profiling']: - config['profiling']['rules'] = {'enabled': True, \ - 'filename': "dalton-rule_perf.log", \ - 'append': True, \ - 'sort': "avgticks", \ - 'limit': 1000, \ - 'json': False} + if "rules" not in config["profiling"]: + config["profiling"]["rules"] = { + "enabled": True, + "filename": "dalton-rule_perf.log", + "append": True, + "sort": "avgticks", + "limit": 1000, + "json": False, + } else: - config['profiling']['rules']['enabled'] = True - config['profiling']['rules']['filename'] = "dalton-rule_perf.log" - config['profiling']['rules']['json'] = False + config["profiling"]["rules"]["enabled"] = True + config["profiling"]["rules"]["filename"] = ( + "dalton-rule_perf.log" + ) + config["profiling"]["rules"]["json"] = False # keyword profiling # is this supported by older Suri versions? If not Suri will ignore when loading YAML - if 'keywords' in config['profiling']: - config['profiling']['keywords'] = {'enabled': True, \ - 'filename': "dalton-keyword_perf.log", \ - 'append': True} + if "keywords" in config["profiling"]: + config["profiling"]["keywords"] = { + "enabled": True, + "filename": "dalton-keyword_perf.log", + "append": True, + } if bGetBufferDumps: - buff_dump_config = {'lua': {'enabled': True, \ - 'scripts-dir': "/opt/dalton-agent", \ - 'scripts': ["http.lua","tls.lua","dns.lua"]}} - - if 'lua' in olist: # someone added something - config['outputs'][olist.index('lua')] = buff_dump_config + buff_dump_config = { + "lua": { + "enabled": True, + "scripts-dir": "/opt/dalton-agent", + "scripts": ["http.lua", "tls.lua", "dns.lua"], + } + } + + if "lua" in olist: # someone added something + config["outputs"][olist.index("lua")] = buff_dump_config else: - config['outputs'].append(buff_dump_config) + config["outputs"].append(buff_dump_config) # write out - engine_conf_file = os.path.join(TEMP_STORAGE_PATH, f"{job_id}_suricata.yaml") + engine_conf_file = os.path.join( + TEMP_STORAGE_PATH, f"{job_id}_suricata.yaml" + ) engine_conf_fh = open(engine_conf_file, "w") - engine_conf_fh.write(yaml.round_trip_dump(config, version=(1,1), explicit_start=True)) + engine_conf_fh.write( + yaml.round_trip_dump(config, version=(1, 1), explicit_start=True) + ) engine_conf_fh.close() except Exception as e: logger.error("Problem processing YAML file(s): %s", e) logger.debug("%s", traceback.format_exc()) delete_temp_files(job_id) - return render_template('/dalton/error.html', jid='', msg=["Error processing YAML file(s):", f"{e}"]) + return render_template( + "/dalton/error.html", + jid="", + msg=["Error processing YAML file(s):", f"{e}"], + ) else: engine_conf_file = None - if sensor_tech.startswith('snort'): + if sensor_tech.startswith("snort"): # tweak Snort conf file - new_conf = '' + new_conf = "" perf_found = False external_net_found = False ipv2add_copy = copy.deepcopy(ipv2add) @@ -1890,14 +2609,16 @@ def page_coverage_summary(): try: line = next(lines) # don't bother keeping comments or empty lines.... - if line.lstrip(' ').startswith('#') or line.lstrip(' ').rstrip(' ') == '': + if ( + line.lstrip(" ").startswith("#") + or line.lstrip(" ").rstrip(" ") == "" + ): # uncomment below to keep comments and empty lines - #new_conf += f"{line}\n" + # new_conf += f"{line}\n" continue # tweak variables - if re.search(r'^(var|portvar|ipvar)\s', line): - + if re.search(r"^(var|portvar|ipvar)\s", line): # add some IP vars common to some rulesets try: for v in ipv2add: @@ -1905,7 +2626,10 @@ def page_coverage_summary(): # can't modify list we are iterating over so delete from copy ipv2add_copy.pop(v) except Exception as e: - logger.warn("(Not Fatal) Problem customizing Snort variables: %s", e) + logger.warning( + "(Not Fatal) Problem customizing Snort variables: %s", + e, + ) logger.debug("%s" % traceback.format_exc()) if line.startswith("ipvar EXTERNAL_NET "): external_net_found = True @@ -1923,9 +2647,13 @@ def page_coverage_summary(): if line.startswith("config profile_rules:"): perf_found = True while line.endswith("\\"): - line = line.rstrip('\\') + next(lines) + line = line.rstrip("\\") + next(lines) if "filename " in line: - line = re.sub(r'filename\s+[^\s\x2C]+', 'filename dalton-rule_perf.log', line) + line = re.sub( + r"filename\s+[^\s\x2C]+", + "filename dalton-rule_perf.log", + line, + ) else: line += ", filename dalton-rule_perf.log append" @@ -1945,31 +2673,37 @@ def page_coverage_summary(): new_conf += f"ipvar {v} {ipv2add_copy[v]}\n" conf_file = new_conf - engine_conf_file = os.path.join(TEMP_STORAGE_PATH, f"{job_id}_snort.conf") - elif sensor_tech.startswith('zeek'): + engine_conf_file = os.path.join( + TEMP_STORAGE_PATH, f"{job_id}_snort.conf" + ) + elif sensor_tech.startswith("zeek"): engine_conf_file = None else: - logger.warn("Unexpected sensor_tech value submitted: %s", sensor_tech) - engine_conf_file = os.path.join(TEMP_STORAGE_PATH, f"{job_id}_engine.conf") - - if not sensor_tech.startswith('zeek'): + logger.warning( + "Unexpected sensor_tech value submitted: %s", sensor_tech + ) + engine_conf_file = os.path.join( + TEMP_STORAGE_PATH, f"{job_id}_engine.conf" + ) + + if not sensor_tech.startswith("zeek"): # write it out with open(engine_conf_file, "w") as engine_conf_fh: engine_conf_fh.write(conf_file) # create jid (job identifier) value digest = hashlib.md5() - digest.update(job_id.encode('utf-8')) - digest.update(sensor_tech.encode('utf-8')) + digest.update(job_id.encode("utf-8")) + digest.update(sensor_tech.encode("utf-8")) splitcap_jid_list = [] for splitcap in pcap_files: - digest.update(splitcap['filename'].encode('utf-8')) + digest.update(splitcap["filename"].encode("utf-8")) jid = digest.hexdigest()[0:16] - #Create the job zipfile. This will contain the file 'manifest.json', which is also queued. - #And place the rules file, config file, and test PCAPs within the zip file + # Create the job zipfile. This will contain the file 'manifest.json', which is also queued. + # And place the rules file, config file, and test PCAPs within the zip file # for splitcap (creating separate jobs for each pcap), only the pcap file and manifest are # modified but seems easier and just as fast to go thru the whole (new) zip file creation # process here. @@ -1981,183 +2715,266 @@ def page_coverage_summary(): # makes it so cron or whatever can easily delete teapot jobs on a different schedule if need be. jid = f"teapot_{jid}" if bSplitCap: - set_job_status_msg(jid, f"Creating job for pcap '{os.path.basename(splitcap['filename'])}'...") + set_job_status_msg( + redis, + jid, + f"Creating job for pcap '{os.path.basename(splitcap['filename'])}'...", + ) zf_path = os.path.join(f"{JOB_STORAGE_PATH}", f"{jid}.zip") - zf = zipfile.ZipFile(zf_path, mode='w') + zf = zipfile.ZipFile(zf_path, mode="w") try: if bSplitCap: - zf.write(splitcap['pcappath'], arcname=os.path.basename(splitcap['filename'])) + zf.write( + splitcap["pcappath"], + arcname=os.path.basename(splitcap["filename"]), + ) else: for pcap in pcap_files: - zf.write(pcap['pcappath'], arcname=os.path.basename(pcap['filename'])) - if request.form.get('optionProdRuleset'): - ruleset_path = request.form.get('prod_ruleset') + zf.write( + pcap["pcappath"], arcname=os.path.basename(pcap["filename"]) + ) + if request.form.get("optionProdRuleset"): + ruleset_path = request.form.get("prod_ruleset") if not ruleset_path: delete_temp_files(job_id) - return render_template('/dalton/error.html', jid=jid, msg=["No defined ruleset provided."]) - if ruleset_path.startswith(JOB_STORAGE_PATH) and ruleset_path.endswith('.zip'): - jobzip_path = os.path.join(JOB_STORAGE_PATH, os.path.basename(ruleset_path)) + return render_template( + "/dalton/error.html", + jid=jid, + msg=["No defined ruleset provided."], + ) + if ruleset_path.startswith( + JOB_STORAGE_PATH + ) and ruleset_path.endswith(".zip"): + jobzip_path = os.path.join( + JOB_STORAGE_PATH, os.path.basename(ruleset_path) + ) if not os.path.exists(jobzip_path): delete_temp_files(job_id) - return render_template('/dalton/error.html', jid=jid, msg=["Ruleset does not exist on Dalton Controller: %s; ruleset-path: %s" % (prod_ruleset_name, ruleset_path)]) + return render_template( + "/dalton/error.html", + jid=jid, + msg=[ + "Ruleset does not exist on Dalton Controller: %s; ruleset-path: %s" + % (prod_ruleset_name, ruleset_path) + ], + ) with zipfile.ZipFile(jobzip_path) as jobzf: for f in jobzf.namelist(): if f.endswith(".rules") and f != "dalton-custom.rules": - ruleset_path = os.path.join(TEMP_STORAGE_PATH, job_id, f) - open(ruleset_path, 'w').write(jobzf.read(f).decode()) + ruleset_path = os.path.join( + TEMP_STORAGE_PATH, job_id, f + ) + open(ruleset_path, "w").write( + jobzf.read(f).decode() + ) prod_ruleset_name = os.path.basename(ruleset_path) break - if not prod_ruleset_name: # if Suri job, this is already set above + if not prod_ruleset_name: # if Suri job, this is already set above prod_ruleset_name = os.path.basename(ruleset_path) if not prod_ruleset_name.endswith(".rules"): prod_ruleset_name = "%s.rules" % prod_ruleset_name logger.debug("ruleset_path = %s" % ruleset_path) - logger.debug("Dalton in page_coverage_summary(): prod_ruleset_name: %s" % (prod_ruleset_name)) - if (not ruleset_path.startswith(RULESET_STORAGE_PATH) and not ruleset_path.startswith(TEMP_STORAGE_PATH)) or ".." in ruleset_path or not re.search(r'^[a-z0-9\/\_\-\.]+$', ruleset_path, re.IGNORECASE): + logger.debug( + "Dalton in page_coverage_summary(): prod_ruleset_name: %s" + % (prod_ruleset_name) + ) + if ( + ( + not ruleset_path.startswith(RULESET_STORAGE_PATH) + and not ruleset_path.startswith(TEMP_STORAGE_PATH) + ) + or ".." in ruleset_path + or not re.search( + r"^[a-z0-9\/\_\-\.]+$", ruleset_path, re.IGNORECASE + ) + ): delete_temp_files(job_id) - return render_template('/dalton/error.html', jid=jid, msg=["Invalid ruleset submitted: '%s'." % prod_ruleset_name, "Path/name invalid."]) + return render_template( + "/dalton/error.html", + jid=jid, + msg=[ + "Invalid ruleset submitted: '%s'." % prod_ruleset_name, + "Path/name invalid.", + ], + ) elif not os.path.exists(ruleset_path): delete_temp_files(job_id) - return render_template('/dalton/error.html', jid=jid, msg=["Ruleset does not exist on Dalton Controller: %s; ruleset-path: %s" % (prod_ruleset_name, ruleset_path)]) + return render_template( + "/dalton/error.html", + jid=jid, + msg=[ + "Ruleset does not exist on Dalton Controller: %s; ruleset-path: %s" + % (prod_ruleset_name, ruleset_path) + ], + ) else: # if these options are set, modify ruleset accordingly if bEnableAllRules or bShowFlowbitAlerts: - modified_rules_path = "%s/%s_prod_modified.rules" % (TEMP_STORAGE_PATH, job_id) - regex = re.compile(r"^#+\s*(alert|log|pass|activate|dynamic|drop|reject|sdrop)\s") - prod_rules_fh = open(ruleset_path, 'r') - modified_rules_fh = open(modified_rules_path, 'w') + modified_rules_path = "%s/%s_prod_modified.rules" % ( + TEMP_STORAGE_PATH, + job_id, + ) + regex = re.compile( + r"^#+\s*(alert|log|pass|activate|dynamic|drop|reject|sdrop)\s" + ) + prod_rules_fh = open(ruleset_path, "r") + modified_rules_fh = open(modified_rules_path, "w") for line in prod_rules_fh: # if Enable disabled rules checked, do the needful if bEnableAllRules: if regex.search(line): - line = line.lstrip('# \t') + line = line.lstrip("# \t") # if show all flowbit alerts set, strip out 'flowbits:noalert;' if bShowFlowbitAlerts: - line = re.sub(r'([\x3B\s])flowbits\s*\x3A\s*noalert\s*\x3B', '\g<1>', line) + line = re.sub( + r"([\x3B\s])flowbits\s*\x3A\s*noalert\s*\x3B", + "\g<1>", + line, + ) modified_rules_fh.write(line) prod_rules_fh.close() modified_rules_fh.close() ruleset_path = modified_rules_path zf.write(ruleset_path, arcname=prod_ruleset_name) try: - if request.form.get('optionCustomRuleset') and request.form.get('custom_ruleset'): - zf.write(custom_rules_file, arcname='dalton-custom.rules') - except: - logger.warn("Problem adding custom rules: %s", e) + if request.form.get("optionCustomRuleset") and request.form.get( + "custom_ruleset" + ): + zf.write(custom_rules_file, arcname="dalton-custom.rules") + except Exception as e: + logger.exception("Problem adding custom rules: %s", e) pass vars_file = None if vars_file is not None: - zf.write(vars_file, arcname='variables.conf') + zf.write(vars_file, arcname="variables.conf") if engine_conf_file: - zf.write(engine_conf_file, arcname=os.path.basename(engine_conf_file)) + zf.write( + engine_conf_file, arcname=os.path.basename(engine_conf_file) + ) - #build the json job + # build the json job json_job = {} - json_job['id'] = jid - json_job['pcaps']= [] + json_job["id"] = jid + json_job["pcaps"] = [] if bSplitCap: - json_job['pcaps'].append(os.path.basename(splitcap['filename'])) + json_job["pcaps"].append(os.path.basename(splitcap["filename"])) else: for pcap in pcap_files: - json_job['pcaps'].append(os.path.basename(pcap['filename'])) + json_job["pcaps"].append(os.path.basename(pcap["filename"])) if engine_conf_file: - json_job['engine-conf'] = os.path.basename(engine_conf_file) - json_job['user'] = user - json_job['enable-all-rules'] = bEnableAllRules - json_job['show-flowbit-alerts'] = bShowFlowbitAlerts - json_job['custom-rules'] = bCustomRules - json_job['track-performance'] = bTrackPerformance - json_job['get-engine-stats'] = bGetEngineStats - json_job['teapot-job'] = bteapotJob - json_job['split-pcaps'] = bSplitCap - json_job['use-suricatasc'] = bSuricataSC - json_job['alert-detailed'] = bGetAlertDetailed - json_job['get-fast-pattern'] = bGetFastPattern - json_job['get-other-logs'] = bGetOtherLogs - json_job['get-buffer-dumps'] = bGetBufferDumps - json_job['sensor-tech'] = sensor_tech - json_job['prod-ruleset'] = prod_ruleset_name - json_job['override-external-net'] = bOverrideExternalNet - json_job['suricata-eve'] = bGetEveLog - json_job['zeek-json-logs'] = boptionZeekJSON - json_job['custom-script-file'] = bCustomscriptfile - json_job['custom-script-write'] = bCustomscriptwrite + json_job["engine-conf"] = os.path.basename(engine_conf_file) + json_job["user"] = user + json_job["enable-all-rules"] = bEnableAllRules + json_job["show-flowbit-alerts"] = bShowFlowbitAlerts + json_job["custom-rules"] = bCustomRules + json_job["track-performance"] = bTrackPerformance + json_job["get-engine-stats"] = bGetEngineStats + json_job["teapot-job"] = bteapotJob + json_job["split-pcaps"] = bSplitCap + json_job["use-suricatasc"] = bSuricataSC + json_job["alert-detailed"] = bGetAlertDetailed + json_job["get-fast-pattern"] = bGetFastPattern + json_job["get-other-logs"] = bGetOtherLogs + json_job["get-buffer-dumps"] = bGetBufferDumps + json_job["sensor-tech"] = sensor_tech + json_job["prod-ruleset"] = prod_ruleset_name + json_job["override-external-net"] = bOverrideExternalNet + json_job["suricata-eve"] = bGetEveLog + json_job["zeek-json-logs"] = boptionZeekJSON + json_job["custom-script-file"] = bCustomscriptfile + json_job["custom-script-write"] = bCustomscriptwrite # add var and other fields too str_job = json.dumps(json_job) - #build the manifest file + # build the manifest file manifest_path = os.path.join(f"{TEMP_STORAGE_PATH}", f"{job_id}.json") - f = open(manifest_path, 'w') + f = open(manifest_path, "w") f.write(str_job) f.close() - zf.write(manifest_path, arcname='manifest.json') + zf.write(manifest_path, arcname="manifest.json") finally: zf.close() - logger.debug("Dalton in page_coverage_summary(): created job zip file %s for user %s" % (zf_path, user)) + logger.debug( + "Dalton in page_coverage_summary(): created job zip file %s for user %s" + % (zf_path, user) + ) # Note: any redis sets here are not given expire times; these should # be set when job is requested by agent - #store user name - r.set("%s-user" % jid, user) + # store user name + redis.set("%s-user" % jid, user) - #store sensor tech for job - r.set("%s-tech" % jid, sensor_tech) + # store sensor tech for job + redis.set("%s-tech" % jid, sensor_tech) # store submission time for job - r.set("%s-submission_time" % jid, datetime.datetime.now().strftime("%b %d %H:%M:%S")) + redis.set( + "%s-submission_time" % jid, + datetime.datetime.now().strftime("%b %d %H:%M:%S"), + ) # if this is a teapot job, if bteapotJob: - r.set("%s-teapotjob" % jid, bteapotJob) + redis.set("%s-teapotjob" % jid, bteapotJob) # set job as queued and write to the Redis queue - set_job_status(jid, STAT_CODE_QUEUED) - set_job_status_msg(jid, f"Queued Job {jid}") - logger.info("Dalton user '%s' submitted Job %s to queue %s" % (user, jid, sensor_tech)) - r.rpush(sensor_tech, str_job) + set_job_status(redis, jid, STAT_CODE_QUEUED) + set_job_status_msg(redis, jid, f"Queued Job {jid}") + logger.info( + "Dalton user '%s' submitted Job %s to queue %s" + % (user, jid, sensor_tech) + ) + redis.rpush(sensor_tech, str_job) # add to list for queue web page - r.lpush("recent_jobs", jid) + redis.lpush("recent_jobs", jid) if bSplitCap: splitcap_jid_list.append(jid) else: break - #remove the temp files from local storage now that everything has been written to the zip file(s) + # remove the temp files from local storage now that everything has been written to the zip file(s) delete_temp_files(job_id) if bteapotJob: if bSplitCap: - return ','.join(splitcap_jid_list) + return ",".join(splitcap_jid_list) else: return jid else: # make sure redirect is set to use http or https as appropriate if bSplitCap: # TODO: something better than just redirect to queue page - rurl = url_for('dalton_blueprint.page_queue_default', _external=True) + rurl = url_for("dalton_blueprint.page_queue_default", _external=True) else: - rurl = url_for('dalton_blueprint.page_show_job', jid=jid, _external=True) - if rurl.startswith('http'): + rurl = url_for( + "dalton_blueprint.page_show_job", jid=jid, _external=True + ) + if rurl.startswith("http"): if "HTTP_X_FORWARDED_PROTO" in request.environ: # if original request was https, make sure redirect uses https - rurl = rurl.replace('http', request.environ['HTTP_X_FORWARDED_PROTO']) + rurl = rurl.replace( + "http", request.environ["HTTP_X_FORWARDED_PROTO"] + ) else: - logger.warn("Could not find request.environ['HTTP_X_FORWARDED_PROTO']. Make sure the web server (proxy) is configured to send it.") + logger.warning( + "Could not find request.environ['HTTP_X_FORWARDED_PROTO']. Make sure the web server (proxy) is configured to send it." + ) else: # this shouldn't be the case with '_external=True' passed to url_for() - logger.warn("URL does not start with 'http': %s" % rurl) + logger.warning("URL does not start with 'http': %s" % rurl) return redirect(rurl) -@dalton_blueprint.route('/dalton/queue') -#@login_required() + +@dalton_blueprint.route("/dalton/queue") +# @login_required() def page_queue_default(): """the default queue page""" - global r + redis = get_redis() num_jobs_to_show_default = 25 # clear old job files from disk @@ -2167,8 +2984,8 @@ def page_queue_default(): Thread(target=delete_old_job_files).start() try: - num_jobs_to_show = int(request.args['numjobs']) - except: + num_jobs_to_show = int(request.args["numjobs"]) + except Exception: num_jobs_to_show = num_jobs_to_show_default if not num_jobs_to_show or num_jobs_to_show < 0: @@ -2176,35 +2993,37 @@ def page_queue_default(): # use a list of dictionaries instead of a dict of dicts to preserve order when it gets passed to render_template queue = [] - queued_jobs = 0; - running_jobs = 0; - if r.exists('recent_jobs') and r.llen('recent_jobs') > 0: + queued_jobs = 0 + running_jobs = 0 + if redis.exists("recent_jobs") and redis.llen("recent_jobs") > 0: # get the last num_jobs_to_show jobs; can adjust if you want (default set above in exception handler) count = 0 - jobs = r.lrange("recent_jobs", 0, -1) + jobs = redis.lrange("recent_jobs", 0, -1) for jid in jobs: - # iterate thru all jobs and get total number of queued and running but only return + # iterate thru all jobs and get total number of queued and running but only return # the most recent num_jobs_to_show jobs # do some cleanup on the list to remove jobs where the data has expired (been deleted). # Using 'jid-submission_time' and jid=status as tests -- if these don't exist the other keys associated # with that jid should be expired or will expire shortly. That key gets set to expire # after a job is requested/sent to a sensor so we won't clear out queued jobs. - if not r.exists("%s-submission_time" % jid) or not r.exists("%s-status" % jid): + if not redis.exists("%s-submission_time" % jid) or not redis.exists( + "%s-status" % jid + ): # job has expired logger.debug("Dalton in page_queue_default(): removing job: %s" % jid) - r.lrem("recent_jobs", jid) + redis.lrem("recent_jobs", jid) # just in case, expire all keys associated with jid - expire_all_keys(jid) + expire_all_keys(redis, jid) else: - status = int(get_job_status(jid)) + status = int(get_job_status(redis, jid)) # ^^ have to cast as an int since it gets stored as a string (everything in redis is a string apparently....) - #logger.debug("Dalton in page_queue_default(): Job %s, stat code: %d" % (jid, status)) + # logger.debug("Dalton in page_queue_default(): Job %s, stat code: %d" % (jid, status)) status_msg = "Unknown" if status == STAT_CODE_QUEUED: status_msg = "Queued" queued_jobs += 1 elif status == STAT_CODE_RUNNING: - if check_for_timeout(jid): + if check_for_timeout(redis, jid): status_msg = "Timeout" else: running_jobs += 1 @@ -2212,7 +3031,7 @@ def page_queue_default(): if count < num_jobs_to_show: if status == STAT_CODE_DONE: status_msg = "Complete" - if r.get("%s-error" % jid): + if redis.get("%s-error" % jid): status_msg += " (Error)" else: status_msg += " (Success)" @@ -2222,57 +3041,82 @@ def page_queue_default(): status_msg = "Timeout" # Note: could add logic to not show teapot jobs?; add if teapotjob: job['teapot'] = "True" else: "False" job = {} - job['jid'] = jid - job ['tech'] = "%s" % r.get("%s-tech" % jid) - job['time'] = "%s" % r.get("%s-submission_time" % jid) - job['user'] = "%s" % r.get("%s-user" % jid) - job['status'] = status_msg - alert_count = get_alert_count(jid) + job["jid"] = jid + job["tech"] = "%s" % redis.get("%s-tech" % jid) + job["time"] = "%s" % redis.get("%s-submission_time" % jid) + job["user"] = "%s" % redis.get("%s-user" % jid) + job["status"] = status_msg + alert_count = get_alert_count(redis, jid) if status != STAT_CODE_DONE: - job['alert_count'] = '-' + job["alert_count"] = "-" elif alert_count is not None: - job['alert_count'] = alert_count + job["alert_count"] = alert_count else: - job['alert_count'] = '?' + job["alert_count"] = "?" queue.append(job) count += 1 - return render_template('/dalton/queue.html', queue=queue, queued_jobs=queued_jobs, running_jobs=running_jobs, num_jobs=num_jobs_to_show) + return render_template( + "/dalton/queue.html", + queue=queue, + queued_jobs=queued_jobs, + running_jobs=running_jobs, + num_jobs=num_jobs_to_show, + ) -@dalton_blueprint.route('/dalton/about') -#@login_required() + +@dalton_blueprint.route("/dalton/about") +# @login_required() def page_about_default(): """the about/help page""" - return render_template('/dalton/about.html', page='') + # Need to `import app` here, not at the top of the file. + import app + + return render_template("dalton/about.html", version=app.__version__) + ######################################### # API handling code (some of it) ######################################### -def controller_api_get_job_data(jid, requested_data): - global r + +def controller_api_get_job_data(redis, jid, requested_data): # add to as necessary - valid_keys = ('alert', 'alert_detailed', 'ids', 'other_logs', 'eve', - 'perf', 'tech', 'error', 'time', 'statcode', 'debug', - 'status', 'submission_time', 'start_time', 'user', 'all', - 'zeek_json' - ) - json_response = {'error':False, 'error_msg':None, 'data':None} + valid_keys = ( + "alert", + "alert_detailed", + "ids", + "other_logs", + "eve", + "perf", + "tech", + "error", + "time", + "statcode", + "debug", + "status", + "submission_time", + "start_time", + "user", + "all", + "zeek_json", + ) + json_response = {"error": False, "error_msg": None, "data": None} # some input validation if not validate_jobid(jid): json_response["error"] = True json_response["error_msg"] = "Invalid Job ID value: %s" % jid - elif not re.match(r'^[a-zA-Z\d\_\.\-]+$', requested_data): + elif not re.match(r"^[a-zA-Z\d\_\.\-]+$", requested_data): json_response["error"] = True json_response["error_msg"] = "Invalid request for data: %s" % requested_data else: try: - status = get_job_status(jid) - except: + status = get_job_status(redis, jid) + except Exception: status = None if not status: # job doesn't exist # expire (delete) all keys related to the job just in case to prevent memory leaks - expire_all_keys(jid) + expire_all_keys(redis, jid) json_response["error"] = True json_response["error_msg"] = "Job ID %s does not exist" % jid else: @@ -2281,22 +3125,27 @@ def controller_api_get_job_data(jid, requested_data): if requested_data not in valid_keys: # check other_logs try: - ologs = r.get("%s-%s" % (jid, 'other_logs')) + ologs = redis.get("%s-%s" % (jid, "other_logs")) if len(ologs) > 0: ologs = json.loads(ologs) for k in ologs.keys(): kkey = k.lower().strip() - kkey = kkey.replace(' ', '_') + kkey = kkey.replace(" ", "_") if kkey == requested_data: json_response["data"] = ologs[k] break if json_response["data"] is None: json_response["error"] = True - json_response["error_msg"] = f"No data found for '{requested_data}' for Job ID {jid}" + json_response["error_msg"] = ( + f"No data found for '{requested_data}' for Job ID {jid}" + ) except Exception as e: json_response["error"] = True - json_response["error_msg"] = "Unexpected error1: cannot pull '%s' data for Job ID %s" % (requested_data, jid) + json_response["error_msg"] = ( + "Unexpected error1: cannot pull '%s' data for Job ID %s" + % (requested_data, jid) + ) logger.debug(f"{json_response['error_msg']}: {e}") else: ret_data = None @@ -2309,39 +3158,60 @@ def controller_api_get_job_data(jid, requested_data): continue elif key == "other_logs": # go thru other_logs struct and make each top-level entries in the response - ologs = r.get("%s-%s" % (jid, key)) + ologs = redis.get("%s-%s" % (jid, key)) if len(ologs) > 0: ologs = json.loads(ologs) for k in ologs.keys(): kdata = ologs[k] k = k.lower().strip() - k = k.replace(' ', '_') + k = k.replace(" ", "_") ret_data[k] = kdata else: - ret_data[key] = r.get("%s-%s" % (jid, key)) + ret_data[key] = redis.get("%s-%s" % (jid, key)) except Exception as e: json_response["error"] = True - json_response["error_msg"] = "Unexpected error: cannot pull '%s' data for Job ID %s" % (requested_data, jid) + json_response["error_msg"] = ( + "Unexpected error: cannot pull '%s' data for Job ID %s" + % (requested_data, jid) + ) logger.debug(f"{json_response['error_msg']}: {e}") else: try: - ret_data = r.get("%s-%s" % (jid, requested_data)) - except: + ret_data = redis.get("%s-%s" % (jid, requested_data)) + except Exception: json_response["error"] = True - json_response["error_msg"] = "Unexpected error: cannot pull '%s' for jobid %s," % (requested_data, jid) + json_response["error_msg"] = ( + "Unexpected error: cannot pull '%s' for jobid %s," + % (requested_data, jid) + ) if requested_data == "other_logs" and len(ret_data) > 0: ret_data = json.loads(ret_data) json_response["data"] = ret_data return json_response -@dalton_blueprint.route('/dalton/controller_api/v2//', defaults={'raw': ''}) -@dalton_blueprint.route('/dalton/controller_api/v2///', methods=['GET']) -#@auth_required() + +@dalton_blueprint.route( + "/dalton/controller_api/v2//", defaults={"raw": ""} +) +@dalton_blueprint.route( + "/dalton/controller_api/v2///", methods=["GET"] +) +# @auth_required() def controller_api_get_request(jid, requested_data, raw): - logger.debug(f"controller_api_get_request() called, raw: {'True' if raw == 'raw' else 'False'}") - json_response = controller_api_get_job_data(jid=jid, requested_data=requested_data) - if raw != 'raw' or json_response['error']: - return Response(json.dumps(json_response), status=200, mimetype='application/json', headers = {'X-Dalton-Webapp':'OK'}) + logger.debug( + f"controller_api_get_request() called, raw: {'True' if raw == 'raw' else 'False'}" + ) + redis = get_redis() + json_response = controller_api_get_job_data( + redis, jid=jid, requested_data=requested_data + ) + if raw != "raw" or json_response["error"]: + return Response( + json.dumps(json_response), + status=200, + mimetype="application/json", + headers={"X-Dalton-Webapp": "OK"}, + ) else: filename = f"{jid}_{requested_data}" if requested_data in ["eve", "all"]: @@ -2350,77 +3220,112 @@ def controller_api_get_request(jid, requested_data, raw): else: mimetype = "text/plain" filename = f"{filename}.txt" - return Response(f"{json_response['data']}", status=200, mimetype=mimetype, headers = {'X-Dalton-Webapp':'OK', "Content-Disposition":f"attachment; filename={filename}"}) - -@dalton_blueprint.route('/dalton/controller_api/get-current-sensors/', methods=['GET']) + return Response( + f"{json_response['data']}", + status=200, + mimetype=mimetype, + headers={ + "X-Dalton-Webapp": "OK", + "Content-Disposition": f"attachment; filename={filename}", + }, + ) + + +@dalton_blueprint.route( + "/dalton/controller_api/get-current-sensors/", methods=["GET"] +) def controller_api_get_current_sensors(engine): """Returns a list of current active sensors""" - global r, supported_engines + redis = get_redis() sensors = [] - if engine is None or engine == '' or engine not in supported_engines: - return Response("Invalid 'engine' supplied. Must be one of %s.\nExample URI:\n\n/dalton/controller_api/get-current-sensors/suricata" % supported_engines, - status=400, mimetype='text/plain', headers = {'X-Dalton-Webapp':'OK'}) + if engine is None or engine == "" or engine not in supported_engines: + return Response( + "Invalid 'engine' supplied. Must be one of %s.\nExample URI:\n\n/dalton/controller_api/get-current-sensors/suricata" + % supported_engines, + status=400, + mimetype="text/plain", + headers={"X-Dalton-Webapp": "OK"}, + ) # first, clean out old sensors - clear_old_agents() + clear_old_agents(redis) # get active sensors based on engine - if r.exists('sensors'): - for sensor in r.smembers('sensors'): - t = r.get("%s-tech" % sensor) + if redis.exists("sensors"): + for sensor in redis.smembers("sensors"): + t = redis.get("%s-tech" % sensor) if t.lower().startswith(engine.lower()): sensors.append(t) # sort so highest version number is first; ignore "rust_" prefix try: - sensors.sort(key=lambda v:LooseVersion(prefix_strip(v.split('/', 1)[1], prefixes="rust_")), reverse=True) - except Exception as e: + sensors.sort( + key=lambda v: LooseVersion( + prefix_strip(v.split("/", 1)[1], prefixes="rust_") + ), + reverse=True, + ) + except Exception: try: sensors.sort(key=LooseVersion, reverse=True) - except Exception as ee: + except Exception: sensors.sort(reverse=True) # return json - json_response = {'sensor_tech': sensors} - return Response(json.dumps(json_response), status=200, mimetype='application/json', headers = {'X-Dalton-Webapp':'OK'}) - -@dalton_blueprint.route('/dalton/controller_api/get-current-sensors-json-full', methods=['GET']) + json_response = {"sensor_tech": sensors} + return Response( + json.dumps(json_response), + status=200, + mimetype="application/json", + headers={"X-Dalton-Webapp": "OK"}, + ) + + +@dalton_blueprint.route( + "/dalton/controller_api/get-current-sensors-json-full", methods=["GET"] +) def controller_api_get_current_sensors_json_full(): """Returns json with details about all the current active sensors""" - sensors = page_sensor_default(return_dict = True) - return Response(json.dumps(sensors), status=200, mimetype='application/json', headers = {'X-Dalton-Webapp':'OK'}) + sensors = page_sensor_default(return_dict=True) + return Response( + json.dumps(sensors), + status=200, + mimetype="application/json", + headers={"X-Dalton-Webapp": "OK"}, + ) -@dalton_blueprint.route('/dalton/controller_api/get-max-pcap-files', methods=['GET']) + +@dalton_blueprint.route("/dalton/controller_api/get-max-pcap-files", methods=["GET"]) def controller_api_get_max_pcap_files(): """Returns the config value of max_pcap_files (the number of - pcap or compressed that can be uploaded per job). - This could be useful for programmatic submissions where the - submitter can ensure all the files will be processed. + pcap or compressed that can be uploaded per job). + This could be useful for programmatic submissions where the + submitter can ensure all the files will be processed. """ return str(MAX_PCAP_FILES) + def parseZeekASCIILog(logtext): log = {} rows = [] lines = logtext.splitlines() for line in lines: line = line.strip() - if line.startswith('#'): - if line.startswith('#separator'): - separator = line.split()[1].encode().decode('unicode-escape') - elif line.startswith('#fields'): - log['fields'] = line.split(separator)[1:] - elif line.startswith('#types'): - log['types'] = line.split(separator)[1:] - elif line.startswith('#close'): + if line.startswith("#"): + if line.startswith("#separator"): + separator = line.split()[1].encode().decode("unicode-escape") + elif line.startswith("#fields"): + log["fields"] = line.split(separator)[1:] + elif line.startswith("#types"): + log["types"] = line.split(separator)[1:] + elif line.startswith("#close"): break else: continue else: rows.append(line.split(separator)) - log['rows'] = rows + log["rows"] = rows return log - diff --git a/app/flowsynth.py b/app/flowsynth.py index 59f0df8..ea38650 100644 --- a/app/flowsynth.py +++ b/app/flowsynth.py @@ -16,117 +16,195 @@ from .dalton import FS_PCAP_PATH as PCAP_PATH # setup the flowsynth blueprint -flowsynth_blueprint = Blueprint('flowsynth_blueprint', __name__, template_folder='templates/') +flowsynth_blueprint = Blueprint( + "flowsynth_blueprint", __name__, template_folder="templates/" +) -# logging -file_handler = RotatingFileHandler('/var/log/flowsynth.log', 'a', 1 * 1024 * 1024, 10) -file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) logger = logging.getLogger("flowsynth") -logger.addHandler(file_handler) -logger.setLevel(logging.INFO) -logger.info("Logging started") + +def setup_flowsynth_logging(): + """Set up logging.""" + file_handler = RotatingFileHandler( + "/var/log/flowsynth.log", "a", 1 * 1024 * 1024, 10 + ) + file_handler.setFormatter( + logging.Formatter("%(asctime)s %(levelname)s: %(message)s") + ) + logger.addHandler(file_handler) + logger.setLevel(logging.INFO) + + logger.info("Logging started") + + +def get_pcap_path(): + """Return the configured path for storing pcaps.""" + return PCAP_PATH + + +def check_pcap_path(path=None): + """Ensure the pcap path exists.""" + path = path or get_pcap_path() + if not os.path.isdir(path): + os.mkdir(path) + os.chmod(path, 0o755) + + +def get_pcap_file_path(basename, path=None): + """Return a full path to a PCAP file based on basename.""" + path = path or get_pcap_path() + return os.path.join(path, f"{basename}.pcap") + def payload_raw(formobj): """parse and format a raw payload""" synth = "" - if (str(formobj['payload_ts'])) != "": - synth = 'default > (content:"%s";);' % fs_replace_badchars(str(formobj.get('payload_ts'))) + if (str(formobj["payload_ts"])) != "": + synth = 'default > (content:"%s";);' % fs_replace_badchars( + str(formobj.get("payload_ts")) + ) - if (str(formobj.get('payload_tc'))) != "": - if (synth != ""): + if (str(formobj.get("payload_tc"))) != "": + if synth != "": synth = "%s\n" % synth - tcpayload = 'default < (content:"%s";);' % fs_replace_badchars(str(formobj.get('payload_tc'))) + tcpayload = 'default < (content:"%s";);' % fs_replace_badchars( + str(formobj.get("payload_tc")) + ) synth = "%s%s" % (synth, tcpayload) return synth -def payload_http(request): - """parse and generate an http payload""" + +def payload_http(formobj): + """Parse and generate an http payload.""" # the raw flowsynth we'll return synth = "" # we must have a request header. try: - request_header = fs_replace_badchars(unicode_safe(request.form.get('request_header')).strip("\r\n")).replace("\r", '\\x0d\\x0a').replace("\n", '\\x0d\\x0a') - request_body = fs_replace_badchars(unicode_safe(request.form.get('request_body')).strip("\r\n")).replace("\r", '\\x0d\\x0a').replace("\n", '\\x0d\\x0a') + request_header = ( + fs_replace_badchars( + unicode_safe(formobj.get("request_header")).strip("\r\n") + ) + .replace("\r", "\\x0d\\x0a") + .replace("\n", "\\x0d\\x0a") + ) + request_body = ( + fs_replace_badchars(unicode_safe(formobj.get("request_body")).strip("\r\n")) + .replace("\r", "\\x0d\\x0a") + .replace("\n", "\\x0d\\x0a") + ) request_body_len = len(request_body) - (request_body.count("\\x") * 3) except Exception as e: logger.error("Problem parsing HTTP Wizard payload request content: %s" % e) return None - #the start of the flowsynth + # the start of the flowsynth synth = 'default > (content:"%s";' % request_header - if 'payload_http_request_contentlength' in request.form: + if "payload_http_request_contentlength" in formobj: # add or update request content length # doesn't add 'Content-Length: 0' if empty request body unless POST # will do inline update of Content-Length value if exists in submitted data - if re.search(r'\\x0d\\x0acontent-length\\x3a(\\x20)*\d+("|\\x0d\\x0a)', synth.lower()): - synth = re.sub(r'(\\x0d\\x0acontent-length\\x3a(?:\\x20)*)\d+("|\\x0d\\x0a)', "\g<1>%d\g<2>" % request_body_len, synth, flags=re.I) - elif (request_body != "" or request_header.lower().startswith("post\\x20")): - synth = '%s content:"\\x0d\\x0aContent-Length\x3a\x20%s";' % (synth, request_body_len) + if re.search( + r'\\x0d\\x0acontent-length\\x3a(\\x20)*\d+("|\\x0d\\x0a)', synth.lower() + ): + synth = re.sub( + r'(\\x0d\\x0acontent-length\\x3a(?:\\x20)*)\d+("|\\x0d\\x0a)', + "\g<1>%d\g<2>" % request_body_len, + synth, + flags=re.I, + ) + elif request_body != "" or request_header.lower().startswith("post\\x20"): + synth = '%s content:"\\x0d\\x0aContent-Length\x3a\x20%s";' % ( + synth, + request_body_len, + ) # add an 0d0a0d0a synth = '%s content:"\\x0d\\x0a\\x0d\\x0a";' % synth - if (request_body != ""): + if request_body != "": # add http_client_body synth = '%s content:"%s"; );\n' % (synth, request_body) else: - synth = '%s );\n' % synth + synth = "%s );\n" % synth - if 'payload_http_response' in request.form: + if "payload_http_response" in formobj: # include http response try: - response_header = fs_replace_badchars(unicode_safe(request.form.get('response_header')).strip("\r\n")).replace("\r", '\\x0d\\x0a').replace("\n", '\\x0d\\x0a') - response_body = fs_replace_badchars(unicode_safe(request.form.get('response_body')).strip("\r\n")).replace("\r", '\\x0d\\x0a').replace("\n", '\\x0d\\x0a') + response_header = ( + fs_replace_badchars( + unicode_safe(formobj.get("response_header")).strip("\r\n") + ) + .replace("\r", "\\x0d\\x0a") + .replace("\n", "\\x0d\\x0a") + ) + response_body = ( + fs_replace_badchars( + unicode_safe(formobj.get("response_body")).strip("\r\n") + ) + .replace("\r", "\\x0d\\x0a") + .replace("\n", "\\x0d\\x0a") + ) response_body_len = len(response_body) - (response_body.count("\\x") * 3) except Exception as e: logger.error("Problem parsing HTTP Wizard payload response content: %s" % e) return None - - if 'payload_http_response_contentlength' in request.form: + if "payload_http_response_contentlength" in formobj: # add or update response content length; include "Content-Length: 0" if body empty # will do inline update of Content-Length value if exists in submitted data - if re.search(r'\\x0d\\x0acontent-length\\x3a(\\x20)*\d+($|\\x0d\\x0a)', response_header.lower()): - response_header = re.sub(r'(\\x0d\\x0acontent-length\\x3a(?:\\x20)*)\d+($|\\x0d\\x0a)', "\g<1>%d\g<2>" % response_body_len, response_header, flags=re.I) + if re.search( + r"\\x0d\\x0acontent-length\\x3a(\\x20)*\d+($|\\x0d\\x0a)", + response_header.lower(), + ): + response_header = re.sub( + r"(\\x0d\\x0acontent-length\\x3a(?:\\x20)*)\d+($|\\x0d\\x0a)", + "\g<1>%d\g<2>" % response_body_len, + response_header, + flags=re.I, + ) synth = '%sdefault < (content:"%s";' % (synth, response_header) else: synth = '%sdefault < (content:"%s";' % (synth, response_header) - synth = '%s content:"\\x0d\\x0aContent-Length\x3a\x20%s";' % (synth, response_body_len) + synth = '%s content:"\\x0d\\x0aContent-Length\x3a\x20%s";' % ( + synth, + response_body_len, + ) else: synth = '%sdefault < (content:"%s";' % (synth, response_header) # add an 0d0a0d0a synth = '%s content:"\\x0d\\x0a\\x0d\\x0a";' % synth - if (response_body != ""): + if response_body != "": synth = '%s content:"%s"; );\n' % (synth, response_body) else: - synth = '%s );\n' % synth + synth = "%s );\n" % synth return synth + def payload_cert(formobj): - # make sure we have stuff we need - if not ('cert_file_type' in formobj and 'cert_file' in request.files): + # Make sure we have stuff we need + if not ("cert_file_type" in formobj and "cert_file" in request.files): logger.error("No cert submitted") return None try: - file_content = request.files['cert_file'].read() - if formobj.get('cert_file_type') == 'pem': - file_content = file_content.decode('utf-8') + file_content = request.files["cert_file"].read() + cert_file_type = formobj.get("cert_file_type") + if cert_file_type == "pem": + file_content = file_content.decode("utf-8") if certsynth.pem_cert_validate(file_content.strip()): - return certsynth.cert_to_synth(file_content.strip(), 'PEM') + return certsynth.cert_to_synth(file_content.strip(), "PEM") else: logger.error("Unable to validate submitted pem file.") return None - elif formobj.get('cert_file_type') == 'der': - return certsynth.cert_to_synth(file_content, 'DER') + elif cert_file_type == "der": + return certsynth.cert_to_synth(file_content, "DER") else: # this shouldn't happen if people are behaving logger.error(f"Invalid certificate format given: '{cert_file_type}'") @@ -135,12 +213,13 @@ def payload_cert(formobj): logger.error(f"Error processing submitted certificate file: {e}") return None + def fs_replace_badchars(payload): - """replace characters that conflict with the flowsynth syntax""" - badchars = ['"', "'", ';', ":", " "] + """Replace characters that conflict with the flowsynth syntax.""" + badchars = ['"', "'", ";", ":", " "] for char in badchars: payload = payload.replace(char, "\\x%s" % str(hex(ord(char)))[2:]) - payload = payload.replace("\r\n", '\\x0d\\x0a') + payload = payload.replace("\r\n", "\\x0d\\x0a") return payload @@ -148,144 +227,173 @@ def unicode_safe(string): """return an ascii repr of the string""" # Jun 21, 2019 - DRW - I'm not sure the reason for this # or if it is still necessary in Python3.... - return string.encode('ascii', 'ignore').decode('ascii') + return string.encode("ascii", "ignore").decode("ascii") -@flowsynth_blueprint.route('/index.html', methods=['GET', 'POST']) +@flowsynth_blueprint.route("/index.html", methods=["GET", "POST"]) def index_redirect(): - return redirect('/') + return redirect("/") @flowsynth_blueprint.route("/") def page_index(): """return the packet generator template""" - return render_template('/pcapwg/packet_gen.html', page='') + return render_template("pcapwg/packet_gen.html", page="") -@flowsynth_blueprint.route('/generate', methods=['POST', 'GET']) +@flowsynth_blueprint.route("/generate", methods=["POST", "GET"]) def generate_fs(): - """receive and handle a request to generate a PCAP""" + """Receive and handle a request to generate a PCAP.""" - packet_hexdump = "" formobj = request.form - # generate flowsynth file - - # options for the flow definition + # Generate flowsynth file flow_init_opts = "" # build src ip statement - src_ip = str(request.form.get('l3_src_ip')) - if (src_ip == "$HOME_NET"): - src_ip = '192.168.%s.%s' % (random.randint(1, 254), random.randint(1, 254)) + src_ip = str(formobj.get("l3_src_ip")) + if src_ip == "$HOME_NET": + src_ip = "192.168.%s.%s" % (random.randint(1, 254), random.randint(1, 254)) else: - src_ip = '172.16.%s.%s' % (random.randint(1, 254), random.randint(1, 254)) + src_ip = "172.16.%s.%s" % (random.randint(1, 254), random.randint(1, 254)) # build dst ip statement - dst_ip = str(request.form.get('l3_dst_ip')) - if (dst_ip == '$HOME_NET'): - dst_ip = '192.168.%s.%s' % (random.randint(1, 254), random.randint(1, 254)) + dst_ip = str(formobj.get("l3_dst_ip")) + if dst_ip == "$HOME_NET": + dst_ip = "192.168.%s.%s" % (random.randint(1, 254), random.randint(1, 254)) else: - dst_ip = '172.16.%s.%s' % (random.randint(1, 254), random.randint(1, 254)) + dst_ip = "172.16.%s.%s" % (random.randint(1, 254), random.randint(1, 254)) # build src port statement - src_port = str(request.form.get('l4_src_port')) - if (src_port.lower() == 'any'): + src_port = str(formobj.get("l4_src_port")) + if src_port.lower() == "any": src_port = random.randint(10000, 65000) # build dst port statement - dst_port = str(request.form.get('l4_dst_port')) - if (dst_port.lower() == 'any'): + dst_port = str(formobj.get("l4_dst_port")) + if dst_port.lower() == "any": dst_port = random.randint(10000, 65000) # initialize the tcp connection automatically, if requested. - if 'l3_flow_established' in formobj: + if "l3_flow_established" in formobj: flow_init_opts = " (tcp.initialize;)" # define the actual flow in the fs syntax synth = "flow default %s %s:%s > %s:%s%s;" % ( - str(request.form.get('l3_protocol')).lower(), src_ip, src_port, dst_ip, dst_port, flow_init_opts) - - payload_fmt = str(request.form.get('payload_format')) - + str(formobj.get("l3_protocol")).lower(), + src_ip, + src_port, + dst_ip, + dst_port, + flow_init_opts, + ) + + payload_fmt = str(formobj.get("payload_format")) payload_cmds = "" - - if payload_fmt == 'raw': - payload_cmds = payload_raw(request.form) - elif (payload_fmt == 'http'): - payload_cmds = payload_http(request) + if payload_fmt == "raw": + payload_cmds = payload_raw(formobj) + elif payload_fmt == "http": + payload_cmds = payload_http(formobj) if payload_cmds is None: - return render_template('/pcapwg/error.html', error_text = "Unable to process submitted HTTP Wizard content. See log for more details.") - elif (payload_fmt == 'cert'): - payload_cmds = payload_cert(request.form) + return render_template( + "pcapwg/error.html", + error_text="Unable to process submitted HTTP Wizard content. See log for more details.", + ) + elif payload_fmt == "cert": + payload_cmds = payload_cert(formobj) if payload_cmds is None: - return render_template('/pcapwg/error.html', error_text = "Unable to process submitted certificate. See log for more details.") + return render_template( + "pcapwg/error.html", + error_text="Unable to process submitted certificate. See log for more details.", + ) synth = "%s\n%s" % (synth, payload_cmds) - return render_template('/pcapwg/compile.html', page='compile', flowsynth_code=synth) + return render_template("pcapwg/compile.html", page="compile", flowsynth_code=synth) + -@flowsynth_blueprint.route('/pcap/compile_fs', methods=['POST']) +@flowsynth_blueprint.route("/pcap/compile_fs", methods=["POST"]) def compile_fs(): - """compile a flowsynth file""" - global PCAP_PATH + """Compile a flowsynth file into a downloadable PCAP.""" - if (os.path.isdir(PCAP_PATH) == False): - os.mkdir(PCAP_PATH) - os.chmod(PCAP_PATH, 0o777) + check_pcap_path() - #write flowsynth data to file - fs_code = str(request.form.get('code')) + # write flowsynth data to file + fs_code = str(request.form.get("code")) hashobj = hashlib.md5() - hashobj.update(f"{fs_code}{random.randint(1,10000)}".encode('utf-8')) + hashobj.update(f"{fs_code}{random.randint(1,10000)}".encode("utf-8")) fname = hashobj.hexdigest()[0:15] - output_url = "get_pcap/%s" % (fname) - inpath = tempfile.mkstemp()[1] - outpath = "%s/%s.pcap" % (PCAP_PATH, fname) - - #write to temp input file - fptr = open(inpath,'w') - fptr.write(fs_code) - fptr.close() - #run the flowsynth command - command = "%s %s -f pcap -w %s --display json --no-filecontent" % (BIN_PATH, inpath, outpath) - print(command) - proc = subprocess.Popen(shlex.split(command), stdout = subprocess.PIPE, stderr = subprocess.PIPE) - output = proc.communicate()[0] + inpath = tempfile.mkstemp()[1] + outpath = get_pcap_file_path(fname) + + # write to temp input file + with open(inpath, "w") as fptr: + fptr.write(fs_code) + + # run the flowsynth command + command = "%s %s -f pcap -w %s --display json --no-filecontent" % ( + BIN_PATH, + inpath, + outpath, + ) + logger.debug(command) + proc = subprocess.Popen( + shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + output, errors = proc.communicate() # parse flowsynth json try: synthstatus = json.loads(output) except ValueError: - #there was a problem producing output. + # there was a problem producing output. logger.error("Problem processing Flowsynth output: %s" % output) - return render_template('/pcapwg/error.html', error_text = output) + return render_template( + "pcapwg/error.html", error_text=output + b"\n\n" + errors + ) - #delete the tempfile + # delete the tempfile os.unlink(inpath) - #render the results page - return render_template('/pcapwg/packet.html', buildstatus = synthstatus, filename=fname) + # render the results page + return render_template( + "pcapwg/packet.html", buildstatus=synthstatus, filename=fname + ) -@flowsynth_blueprint.route('/compile', methods=['GET', 'POST']) + +@flowsynth_blueprint.route("/compile", methods=["GET", "POST"]) def compile_page(): - flowsynth = request.values.get('flowsynth', '') - return render_template('/pcapwg/compile.html', page='compile', flowsynth_code=flowsynth) + flowsynth = request.values.get("flowsynth", "") + return render_template( + "pcapwg/compile.html", page="compile", flowsynth_code=flowsynth + ) + -@flowsynth_blueprint.route('/about') +@flowsynth_blueprint.route("/about") def about_page(): - return render_template('/pcapwg/about.html', page='about') + return render_template("pcapwg/about.html", page="about") + -@flowsynth_blueprint.route('/pcap/get_pcap/') +@flowsynth_blueprint.route("/pcap/get_pcap/") def retrieve_pcap(pcapid): - """returns a PCAP to the user""" - global PCAP_PATH, logger + """Returns a PCAP to the user.""" if not re.match(r"^[A-Za-z0-9\x5F\x2D\x2E]+$", pcapid): logger.error("Bad pcapid in get_pcap request: '%s'" % pcapid) - return render_template('/pcapwg/error.html', error_text = "Bad pcapid: '%s'" % pcapid) - path = '%s/%s.pcap' % (PCAP_PATH, os.path.basename(pcapid)) + return render_template( + "pcapwg/error.html", error_text="Bad pcapid: '%s'" % pcapid + ) + path = get_pcap_file_path(os.path.basename(pcapid)) if not os.path.isfile(path): - logger.error("In get_pcap request: file not found: '%s'" % os.path.basename(path)) - return render_template('/pcapwg/error.html', error_text = "File not found: '%s'" % os.path.basename(path)) - filedata = open(path,'rb').read() - return Response(filedata,mimetype="application/vnd.tcpdump.pcap", headers={"Content-Disposition":"attachment;filename=%s.pcap" % pcapid}) + logger.error( + "In get_pcap request: file not found: '%s'" % os.path.basename(path) + ) + return render_template( + "pcapwg/error.html", + error_text="File not found: '%s'" % os.path.basename(path), + ) + filedata = open(path, "rb").read() + return Response( + filedata, + mimetype="application/vnd.tcpdump.pcap", + headers={"Content-Disposition": "attachment;filename=%s.pcap" % pcapid}, + ) diff --git a/app/templates/dalton/about.html b/app/templates/dalton/about.html index 807646d..2068583 100644 --- a/app/templates/dalton/about.html +++ b/app/templates/dalton/about.html @@ -3,4 +3,5 @@

About Dalton

Dalton allows pcaps to be run against sundry IDS sensors (e.g. Suricata, Snort) and sensor versions using a defined ruleset and/or bespoke rules.

Official repository and documentation can be found at https://github.com/secureworks/dalton

+

This is Dalton version {{ version }}

{% endblock %} diff --git a/dalton-agent/.bumpversion.toml b/dalton-agent/.bumpversion.toml new file mode 100644 index 0000000..9a4ba28 --- /dev/null +++ b/dalton-agent/.bumpversion.toml @@ -0,0 +1,21 @@ +[tool.bumpversion] +current_version = "3.1.2" +parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)" +serialize = ["{major}.{minor}.{patch}"] +search = "{current_version}" +replace = "{new_version}" +regex = false +ignore_missing_version = false +ignore_missing_files = false +tag = false +sign_tags = false +allow_dirty = false +commit = true +message = "Bump dalton-agent version: {current_version} → {new_version}" +commit_args = "--no-verify" +setup_hooks = [] +pre_commit_hooks = [] +post_commit_hooks = [] + +[[tool.bumpversion.files]] +filename = "dalton-agent/dalton-agent.py" diff --git a/dalton-agent/Dockerfiles/Dockerfile_snort b/dalton-agent/Dockerfiles/Dockerfile_snort index 592542b..52e8d5d 100644 --- a/dalton-agent/Dockerfiles/Dockerfile_snort +++ b/dalton-agent/Dockerfiles/Dockerfile_snort @@ -2,15 +2,19 @@ # Works for Snort 2.9.1.1 and later; previous versions are more # nuanced with libraries and compile dependencies so if you need # a previous version, just build your own. + +# hadolint global ignore=DL3003,SC2046 + FROM ubuntu:18.04 -MAINTAINER David Wharton ARG SNORT_VERSION ARG DAQ_VERSION # tcpdump is for pcap analysis; not *required* for # the agent but nice to have.... -RUN apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y \ +# hadolint ignore=DL3008 +RUN apt-get update -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ python3.8 \ tcpdump \ automake autoconf \ @@ -18,10 +22,13 @@ RUN apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y \ libpcap-dev libpcre3-dev \ libcap-ng-dev libdumbnet-dev \ zlib1g-dev liblzma-dev openssl libssl-dev \ - libnghttp2-dev libluajit-5.1-dev && ldconfig + libnghttp2-dev libluajit-5.1-dev && \ + ldconfig && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # for debugging agent -RUN apt-get install -y less nano +# RUN apt-get install -y less nano # download, build, and install Snort from source RUN mkdir -p /src/snort-${SNORT_VERSION} && mkdir -p /etc/snort @@ -44,4 +51,4 @@ WORKDIR /opt/dalton-agent COPY dalton-agent.py /opt/dalton-agent/dalton-agent.py COPY dalton-agent.conf /opt/dalton-agent/dalton-agent.conf -CMD python3.8 /opt/dalton-agent/dalton-agent.py -c /opt/dalton-agent/dalton-agent.conf 2>&1 +CMD ["python3.8", "/opt/dalton-agent/dalton-agent.py", "-c", "/opt/dalton-agent/dalton-agent.conf"] diff --git a/dalton-agent/Dockerfiles/Dockerfile_suricata b/dalton-agent/Dockerfiles/Dockerfile_suricata index bd95416..1c7c878 100644 --- a/dalton-agent/Dockerfiles/Dockerfile_suricata +++ b/dalton-agent/Dockerfiles/Dockerfile_suricata @@ -1,20 +1,23 @@ # Builds Suricata Dalton agent using Suricata source tarball FROM ubuntu:18.04 -MAINTAINER David Wharton ARG SURI_VERSION ARG ENABLE_RUST # tcpdump is for pcap analysis; not *required* for # the agent but nice to have.... -RUN apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y \ +# hadolint ignore=DL3008 +RUN apt-get update -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ python3.8 \ tcpdump \ - libpcre3 libpcre3-dbg libpcre3-dev libnss3-dev\ + libpcre3 libpcre3-dbg libpcre3-dev libnss3-dev \ build-essential autoconf automake libtool libpcap-dev libnet1-dev \ libyaml-0-2 libyaml-dev zlib1g zlib1g-dev libcap-ng-dev libcap-ng0 \ make libmagic-dev libjansson-dev libjansson4 pkg-config rustc cargo \ - liblua5.1-dev libevent-dev libpcre2-dev + liblua5.1-dev libevent-dev libpcre2-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # for debugging agent @@ -35,15 +38,18 @@ WORKDIR /src/suricata-${SURI_VERSION} # sed -i 's|#ifdef HAVE_AF_PACKET|#ifdef HAVE_AF_PACKET\n\n#if HAVE_LINUX_SOCKIOS_H\n#include \n#endif\n|' src/source-af-packet.c; \ # fi; # configure, make, and install +# hadolint ignore=SC2046 RUN ./configure --enable-profiling ${ENABLE_RUST} --enable-lua && make -j $(nproc) && make install && make install-conf && ldconfig RUN mkdir -p /opt/dalton-agent/ WORKDIR /opt/dalton-agent COPY dalton-agent.py /opt/dalton-agent/dalton-agent.py COPY dalton-agent.conf /opt/dalton-agent/dalton-agent.conf + COPY http.lua /opt/dalton-agent/http.lua COPY dns.lua /opt/dalton-agent/dns.lua COPY tls.lua /opt/dalton-agent/tls.lua + RUN sed -i 's/REPLACE_AT_DOCKER_BUILD-VERSION/'"${SURI_VERSION}"'/' /opt/dalton-agent/dalton-agent.conf -CMD python3.8 /opt/dalton-agent/dalton-agent.py -c /opt/dalton-agent/dalton-agent.conf 2>&1 +CMD ["python3.8", "/opt/dalton-agent/dalton-agent.py", "-c", "/opt/dalton-agent/dalton-agent.conf"] diff --git a/dalton-agent/Dockerfiles/Dockerfile_zeek b/dalton-agent/Dockerfiles/Dockerfile_zeek index 0c90df6..27f6aa1 100644 --- a/dalton-agent/Dockerfiles/Dockerfile_zeek +++ b/dalton-agent/Dockerfiles/Dockerfile_zeek @@ -1,46 +1,21 @@ -FROM ubuntu:20.04 ARG ZEEK_VERSION -# Set non-interactive frontend to avoid interactive prompts -ENV DEBIAN_FRONTEND=noninteractive +FROM zeek/zeek:$ZEEK_VERSION -# Update package list and install required packages including Node.js and Python -RUN apt-get update --fix-missing -y && \ - apt-get install -y \ +# Update package list and install useful packages +# hadolint ignore=DL3008 +RUN apt-get update -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ curl \ - gnupg \ - build-essential \ tcpdump \ - cmake \ - make \ - gcc \ - g++ \ - flex \ - bison \ - libpcap-dev \ - libssl-dev \ python3 \ python3-dev \ python3-pip \ - swig \ - zlib1g-dev \ && \ - curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \ - apt-get install -y nodejs \ - && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -ENV PATH="/usr/local/zeek/bin/:${PATH}" - -RUN mkdir -p /src/zeek-${ZEEK_VERSION} -WORKDIR /src -ADD https://download.zeek.org/zeek-${ZEEK_VERSION}.tar.gz zeek-${ZEEK_VERSION}.tar.gz -RUN tar -zxf zeek-${ZEEK_VERSION}.tar.gz -C zeek-${ZEEK_VERSION} --strip-components=1 -WORKDIR /src/zeek-${ZEEK_VERSION} -RUN ./configure && make -j $(nproc) && make install - RUN mkdir -p /opt/dalton-agent/ WORKDIR /opt/dalton-agent COPY dalton-agent.py /opt/dalton-agent/dalton-agent.py diff --git a/dalton-agent/dalton-agent.py b/dalton-agent/dalton-agent.py index 1e23eef..c33c322 100755 --- a/dalton-agent/dalton-agent.py +++ b/dalton-agent/dalton-agent.py @@ -27,29 +27,30 @@ # still present. # -import os -import sys -import traceback -import urllib.request, urllib.parse, urllib.error -import urllib.request, urllib.error, urllib.parse -import re -import time +import base64 +import binascii +import configparser import datetime import glob -import shutil -import base64 +import hashlib import json +import logging +import os +import re +import shutil +import socket +import struct import subprocess +import sys +import time +import traceback +import urllib.error +import urllib.parse +import urllib.request import zipfile -import configparser -from optparse import OptionParser -import struct -import socket -import logging -from logging.handlers import RotatingFileHandler from distutils.version import LooseVersion -import binascii -import hashlib +from logging.handlers import RotatingFileHandler +from optparse import OptionParser from pathlib import Path # urllib2 in Python < 2.6 doesn't support setting a timeout so doing it like this @@ -63,14 +64,17 @@ suricata_sc_pid_file = "/usr/local/var/run/suricata.pid" -#********************************* -#*** Parse Command Line Options *** -#********************************* +# ********************************* +# *** Parse Command Line Options *** +# ********************************* parser = OptionParser() -parser.add_option("-c", "--config", - dest="configfile", - help="path to config file [default: %default]", - default="dalton-agent.conf") +parser.add_option( + "-c", + "--config", + dest="configfile", + help="path to config file [default: %default]", + default="dalton-agent.conf", +) (options, args) = parser.parse_args() dalton_config_file = options.configfile @@ -85,40 +89,44 @@ try: config.read(dalton_config_file) -except Exception as e: +except Exception: # just print to stdout; logging hasn't started yet print(f"Error reading config file, '{dalton_config_file}'.\n\nexiting.") sys.exit(1) try: - DEBUG = config.getboolean('dalton', 'DEBUG') - STORAGE_PATH = config.get('dalton', 'STORAGE_PATH') - SENSOR_CONFIG = config.get('dalton', 'SENSOR_CONFIG') - SENSOR_ENGINE = config.get('dalton', 'SENSOR_ENGINE').lower() - SENSOR_ENGINE_VERSION = config.get('dalton', 'SENSOR_ENGINE_VERSION').lower() - SENSOR_UID = config.get('dalton', 'SENSOR_UID') - DALTON_API = config.get('dalton', 'DALTON_API') - API_KEY = config.get('dalton', 'API_KEY') - POLL_INTERVAL = int(config.get('dalton', 'POLL_INTERVAL')) - KEEP_JOB_FILES = config.getboolean('dalton', 'KEEP_JOB_FILES') - USE_SURICATA_SOCKET_CONTROL = config.getboolean('dalton', 'USE_SURICATA_SOCKET_CONTROL') - SURICATA_SC_PYTHON_MODULE = config.get('dalton', 'SURICATA_SC_PYTHON_MODULE') - SURICATA_SOCKET_NAME = config.get('dalton', 'SURICATA_SOCKET_NAME') + DEBUG = config.getboolean("dalton", "DEBUG") + STORAGE_PATH = config.get("dalton", "STORAGE_PATH") + SENSOR_CONFIG = config.get("dalton", "SENSOR_CONFIG") + SENSOR_ENGINE = config.get("dalton", "SENSOR_ENGINE").lower() + SENSOR_ENGINE_VERSION = config.get("dalton", "SENSOR_ENGINE_VERSION").lower() + SENSOR_UID = config.get("dalton", "SENSOR_UID") + DALTON_API = config.get("dalton", "DALTON_API") + API_KEY = config.get("dalton", "API_KEY") + POLL_INTERVAL = int(config.get("dalton", "POLL_INTERVAL")) + KEEP_JOB_FILES = config.getboolean("dalton", "KEEP_JOB_FILES") + USE_SURICATA_SOCKET_CONTROL = config.getboolean( + "dalton", "USE_SURICATA_SOCKET_CONTROL" + ) + SURICATA_SC_PYTHON_MODULE = config.get("dalton", "SURICATA_SC_PYTHON_MODULE") + SURICATA_SOCKET_NAME = config.get("dalton", "SURICATA_SOCKET_NAME") except Exception as e: # just print to stdout; logging hasn't started yet print(f"Error parsing config file, '{dalton_config_file}':\n\n{e}\n\nexiting.") sys.exit(1) -SENSOR_ENGINE_VERSION_ORIG = 'undefined' +SENSOR_ENGINE_VERSION_ORIG = "undefined" -#*************** -#*** Logging *** -#*************** +# *************** +# *** Logging *** +# *************** -file_handler = RotatingFileHandler('/var/log/dalton-agent.log', 'a', 1 * 1024 * 1024, 10) -file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) -#file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]')) +file_handler = RotatingFileHandler( + "/var/log/dalton-agent.log", "a", 1 * 1024 * 1024, 10 +) +file_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s")) +# file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]')) logger = logging.getLogger("dalton-agent") logger.addHandler(file_handler) if DEBUG or ("AGENT_DEBUG" in os.environ and int(os.getenv("AGENT_DEBUG"))): @@ -130,40 +138,45 @@ logger.setLevel(logging.INFO) -#************************************************ -#** Helper Functions to populate config values ** -#************************************************ +# ************************************************ +# ** Helper Functions to populate config values ** +# ************************************************ def prefix_strip(mystring, prefixes=["rust_"]): - """ strip passed in prefixes from the beginning of passed in string and return it - """ + """strip passed in prefixes from the beginning of passed in string and return it""" if not isinstance(prefixes, list): prefixes = [prefixes] for prefix in prefixes: if mystring.startswith(prefix): - return mystring[len(prefix):] + return mystring[len(prefix) :] return mystring + def find_file(name): """Returns full path to file if found on system.""" ret_path = None try: # see if it is already in the path by using the 'which' command - process = subprocess.Popen("which %s" % name, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + process = subprocess.Popen( + "which %s" % name, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ) stdout, stderr = process.communicate() if stderr: raise else: - ret_path = stdout.decode('utf-8').strip() - except: + ret_path = stdout.decode("utf-8").strip() + except Exception: # file not in PATH, try manually searching - paths = ['/usr/sbin', '/usr/bin', '/usr/local/bin', '/usr/local/sbin'] + paths = ["/usr/sbin", "/usr/bin", "/usr/local/bin", "/usr/local/sbin"] for path in paths: candidate = os.path.join(path, name) if os.path.exists(candidate): - ret_val = candidate break return ret_path + def get_engine_version(path): """returns the version of the engine given full path to binary (e.g. Suricata, Snort).""" global SENSOR_ENGINE_VERSION_ORIG @@ -171,40 +184,52 @@ def get_engine_version(path): version = "unknown" try: binary_name = os.path.basename(path).lower() - if 'zeek' in binary_name: + if "zeek" in binary_name: command = f"{path} --version" - elif 'snort' in binary_name or 'suricata' in binary_name: + elif "snort" in binary_name or "suricata" in binary_name: command = f"{path} -V" - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + process = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True + ) stdout, stderr = process.communicate() - regex = re.compile(r"(Version|(Suricata|zeek) version)\s+(?P\d+[\d\x2E\x2D\5FA-Za-z]*)") + regex = re.compile( + r"(Version|(Suricata|zeek) version)\s+(?P\d+[\d\x2E\x2D\5FA-Za-z]*)" + ) if stderr: # apparently 'Snort -V' outputs to stderr.... output = stderr else: output = stdout # get engine from output - if "Suricata" in output.decode('utf-8'): + if "Suricata" in output.decode("utf-8"): engine = "suricata" - elif "Snort" in output.decode('utf-8'): + elif "Snort" in output.decode("utf-8"): engine = "snort" - elif "zeek" in output.decode('utf-8'): + elif "zeek" in output.decode("utf-8"): engine = "zeek" else: # use filename of binary engine = os.path.basename(path).lower() - logger.warn("Could not determine engine name, using '%s' from IDS_BINARY path" % engine) + logger.warn( + "Could not determine engine name, using '%s' from IDS_BINARY path" + % engine + ) # get version from output - result = regex.search(output.decode('utf-8')) + result = regex.search(output.decode("utf-8")) if result: - version = result.group('version') + version = result.group("version") SENSOR_ENGINE_VERSION_ORIG = version # if Suricata version 4, see if Rust is enabled and add to version string - if "suricata" in engine and version.split('.')[0] == "4": - process = subprocess.Popen('%s --build-info | grep "Rust support"' % path, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + if "suricata" in engine and version.split(".")[0] == "4": + process = subprocess.Popen( + '%s --build-info | grep "Rust support"' % path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ) stdout, stderr = process.communicate() if "yes" in str(stdout): # rust support exists @@ -212,14 +237,17 @@ def get_engine_version(path): except Exception as e: logger.warn("Exception in get_engine_version(): %s" % e) pass - logger.debug("Using IDS binary '%s': engine: '%s', version '%s'" % (path, engine, version)) + logger.debug( + "Using IDS binary '%s': engine: '%s', version '%s'" % (path, engine, version) + ) return (engine, version) + def hash_file(filenames): """Returns md5sum of passed in file. If a list of files is passed, - they are concatenated together and hashed. - Remove "default-rule-path" from Suricata config since this - changes every job. + they are concatenated together and hashed. + Remove "default-rule-path" from Suricata config since this + changes every job. """ if not isinstance(filenames, list): filenames = [filenames] @@ -230,63 +258,70 @@ def hash_file(filenames): raise if filename.endswith(".yaml"): # remove "default-rule-path:" line - with open(filename, 'r') as fh: + with open(filename, "r") as fh: lines = fh.readlines() - hash.update("".join([l for l in lines if not l.startswith("default-rule-path:")]).encode('utf-8')) + hash.update( + "".join( + [ + line + for line in lines + if not line.startswith("default-rule-path:") + ] + ).encode("utf-8") + ) else: - with open(filename, 'rb') as fh: + with open(filename, "rb") as fh: data = fh.read(65536) while len(data) > 0: hash.update(data) data = fh.read(65536) return hash.hexdigest() -#************************** -#*** Constant Variables *** -#************************** -AGENT_VERSION = "3.1.1" -HTTP_HEADERS = { - "User-Agent" : f"Dalton Agent/{AGENT_VERSION}" -} +# ************************** +# *** Constant Variables *** +# ************************** + +AGENT_VERSION = "3.1.2" +HTTP_HEADERS = {"User-Agent": f"Dalton Agent/{AGENT_VERSION}"} # check options from config file # done here after logging has been set up -if SENSOR_UID == 'auto': +if SENSOR_UID == "auto": SENSOR_UID = socket.gethostname() -TCPDUMP_BINARY = 'auto' +TCPDUMP_BINARY = "auto" try: - TCPDUMP_BINARY = config.get('dalton', 'TCPDUMP_BINARY') + TCPDUMP_BINARY = config.get("dalton", "TCPDUMP_BINARY") except Exception as e: logger.warn("Unable to get config value 'TCPDUMP_BINARY': %s" % e) pass -if TCPDUMP_BINARY == 'auto': - TCPDUMP_BINARY = find_file('tcpdump') +if TCPDUMP_BINARY == "auto": + TCPDUMP_BINARY = find_file("tcpdump") if not TCPDUMP_BINARY or not os.path.exists(TCPDUMP_BINARY): - logger.warn("Could not find 'tcpdump' binary.") - TCPDUMP_BINARY = '' + logger.warn("Could not find 'tcpdump' binary.") + TCPDUMP_BINARY = "" -IDS_BINARY = 'auto' +IDS_BINARY = "auto" try: - IDS_BINARY = config.get('dalton', 'IDS_BINARY') + IDS_BINARY = config.get("dalton", "IDS_BINARY") except Exception as e: logger.warn("Unable to get config value 'IDS_BINARY': %s" % e) pass -if IDS_BINARY == 'auto': - IDS_BINARY = find_file('suricata') +if IDS_BINARY == "auto": + IDS_BINARY = find_file("suricata") if not IDS_BINARY or not os.path.exists(IDS_BINARY): - logger.info("Could not find 'suricata' binary, going to look for Snort.") - IDS_BINARY = None + logger.info("Could not find 'suricata' binary, going to look for Snort.") + IDS_BINARY = None if IDS_BINARY is None: # look for Snort - IDS_BINARY = find_file('snort') + IDS_BINARY = find_file("snort") if not IDS_BINARY or not os.path.exists(IDS_BINARY): logger.info("Could not find 'snort' binary.") IDS_BINARY = None if IDS_BINARY is None: # look for Zeek - IDS_BINARY = find_file('zeek') + IDS_BINARY = find_file("zeek") if not IDS_BINARY or not os.path.exists(IDS_BINARY): logger.info("Could not find 'snort' binary.") logger.critical("No IDS binary specified or found. Cannot continue.") @@ -307,7 +342,7 @@ def hash_file(filenames): if USE_SURICATA_SOCKET_CONTROL: # Socket Control supported in Suricata 1.4 and later - if float('.'.join(prefix_strip(eng_ver).split('.')[:2])) < 3.0: + if float(".".join(prefix_strip(eng_ver).split(".")[:2])) < 3.0: msg = f"Dalton Agent does not support Suricata Socket Control for Suricata versions before 3.0. This is running Suricata version {eng_ver}. Disabling Suricata Socket Control Mode." logger.warn(msg) USE_SURICATA_SOCKET_CONTROL = False @@ -315,23 +350,36 @@ def hash_file(filenames): if USE_SURICATA_SOCKET_CONTROL: if os.path.isdir(SURICATA_SC_PYTHON_MODULE): sys.path.append(SURICATA_SC_PYTHON_MODULE) - elif os.path.isdir(os.path.abspath(os.path.join(SURICATA_SC_PYTHON_MODULE, '..', 'scripts', 'suricatasc', 'src'))): + elif os.path.isdir( + os.path.abspath( + os.path.join( + SURICATA_SC_PYTHON_MODULE, "..", "scripts", "suricatasc", "src" + ) + ) + ): # older Suricata versions had suricatasc in "scripts" directory, not "python" directory - sys.path.append(os.path.abspath(os.path.join(SURICATA_SC_PYTHON_MODULE, '..', 'scripts', 'suricatasc', 'src'))) + sys.path.append( + os.path.abspath( + os.path.join( + SURICATA_SC_PYTHON_MODULE, "..", "scripts", "suricatasc", "src" + ) + ) + ) # Used as Suricata default-log-dir when in SC mode os.makedirs(os.path.dirname(SURICATA_SOCKET_NAME), exist_ok=True) -req_job_url = (f"{DALTON_API}/request_job?" - f"SENSOR_ENGINE={SENSOR_ENGINE}&" - f"SENSOR_ENGINE_VERSION={SENSOR_ENGINE_VERSION}&" - f"SENSOR_UID={SENSOR_UID}&" - f"AGENT_VERSION={AGENT_VERSION}&" - f"{sensor_config_variable}" - f"API_KEY={API_KEY}" - ) +req_job_url = ( + f"{DALTON_API}/request_job?" + f"SENSOR_ENGINE={SENSOR_ENGINE}&" + f"SENSOR_ENGINE_VERSION={SENSOR_ENGINE_VERSION}&" + f"SENSOR_UID={SENSOR_UID}&" + f"AGENT_VERSION={AGENT_VERSION}&" + f"{sensor_config_variable}" + f"API_KEY={API_KEY}" +) logger.info("\n*******************") -logger.info("Starting Dalton Agent version %s:"% AGENT_VERSION) +logger.info("Starting Dalton Agent version %s:" % AGENT_VERSION) logger.debug("\tDEBUG logging: enabled") logger.info("\tSENSOR_UID: %s" % SENSOR_UID) logger.info("\tSENSOR_ENGINE: %s" % SENSOR_ENGINE) @@ -347,15 +395,18 @@ def hash_file(filenames): # agent is contacting the hostname "dalton_web", then the agent and web server are # containers on the same host. dalton_web_container = "dalton_web" -if not 'no_proxy' in os.environ: - os.environ['no_proxy'] = dalton_web_container +if "no_proxy" not in os.environ: + os.environ["no_proxy"] = dalton_web_container else: - os.environ['no_proxy'] = "%s,%s" % (os.environ['no_proxy'].rstrip(','), dalton_web_container) + os.environ["no_proxy"] = "%s,%s" % ( + os.environ["no_proxy"].rstrip(","), + dalton_web_container, + ) logger.info("Added '%s' to 'no_proxy' environment variable." % dalton_web_container) -#************************ -#*** Global Variables *** -#************************ +# ************************ +# *** Global Variables *** +# ************************ JOB_ID = None PCAP_FILES = [] PCAP_DIR = "pcaps" @@ -379,7 +430,7 @@ def hash_file(filenames): # used by Snort for logs/alerts # Suricata puts every log in here IDS_LOG_DIRECTORY = None -TOTAL_PROCESSING_TIME = '' +TOTAL_PROCESSING_TIME = "" # seconds ERROR_SLEEP_TIME = 5 URLLIB_TIMEOUT = 120 @@ -390,9 +441,10 @@ def hash_file(filenames): # boolean used to work around Suricata Redmine issue 4225 SC_FIRST_RUN = True -#****************# -#*** Job Logs ***# -#****************# + +# ****************# +# *** Job Logs ***# +# ****************# # functions for populating job logs def print_error(msg): if JOB_ERROR_LOG: @@ -406,12 +458,14 @@ def print_error(msg): # throw error raise DaltonError(msg) + def print_msg(msg): print_debug(msg) # send message logger.debug(msg) send_update(msg, JOB_ID) + def print_debug(msg): global JOB_DEBUG_LOG if JOB_DEBUG_LOG: @@ -421,15 +475,18 @@ def print_debug(msg): else: logger.debug("print_debug() called but no JOB_DEBUG_LOG exists") -#********************** -#*** Custom Classes *** -#********************** + +# ********************** +# *** Custom Classes *** +# ********************** + class SocketController: - """ Basically a wrapper for Suricata socket control. - Also handles start/restart of Suricata which is - run in daemon mode. + """Basically a wrapper for Suricata socket control. + Also handles start/restart of Suricata which is + run in daemon mode. """ + def __init__(self, socket_path): try: self.sc = suricatasc.SuricataSC(socket_path) @@ -456,13 +513,16 @@ def connect(self): def send_command(self, command): try: cmd, arguments = self.sc.parse_command(command) - #logger.debug("in send_command():\n\tcmd: %s\n\targuments: %s" % (cmd, arguments)) + # logger.debug("in send_command():\n\tcmd: %s\n\targuments: %s" % (cmd, arguments)) cmdret = self.sc.send_command(cmd, arguments) except Exception as e: print_error("Problem parsing/sending command: %s" % e) if cmdret["return"] == "NOK": - print_error("\"NOK\" response received from socket command; message: %s" % json.dumps(cmdret["message"])) + print_error( + '"NOK" response received from socket command; message: %s' + % json.dumps(cmdret["message"]) + ) return json.dumps(cmdret["message"]) @@ -481,25 +541,27 @@ def shutdown(self): finally: self.suricata_is_running = False - def stop_suricata_daemon(self): """Stop Suricata daemon using socket control.""" logger.debug("stop_suricata_daemon() called") if not self.suricata_is_running: - logger.warn("stop_suricata_daemon() called but Suricata may not be running." - " Still attempting shutdown but it will likely error." - ) + logger.warn( + "stop_suricata_daemon() called but Suricata may not be running." + " Still attempting shutdown but it will likely error." + ) try: self.connect() self.shutdown() self.close() except Exception as e: - print_error(f"Problem shutting down old Suricata instance in stop_suricata_daemon(): {e}") + print_error( + f"Problem shutting down old Suricata instance in stop_suricata_daemon(): {e}" + ) finally: self.suricata_is_running = False - def reset_logging(self, delete_pid_file = True): - """ Reset log files and remove pid file if exists. """ + def reset_logging(self, delete_pid_file=True): + """Reset log files and remove pid file if exists.""" # logging if os.path.exists(suricata_logging_outputs_file): logger.debug("deleting '%s'" % suricata_logging_outputs_file) @@ -508,7 +570,7 @@ def reset_logging(self, delete_pid_file = True): # there could be a race condition Path(suricata_logging_outputs_file).touch() self.log_offset = 0 - self.suri_startup_log = '' + self.suri_startup_log = "" # pid file if delete_pid_file: @@ -524,19 +586,24 @@ def start_suricata_daemon(self, config): print_error("start_suricata_daemon() called but not initialized.") # start Suri suricata_command = f"suricata -c {config} -k none --runmode single --unix-socket={SURICATA_SOCKET_NAME} -D" - print_debug(f"Starting suricata thread with the following command:\n{suricata_command}") + print_debug( + f"Starting suricata thread with the following command:\n{suricata_command}" + ) # use Popen() instead of call() since the latter blocks which isn't what we want - subprocess.Popen(suricata_command, shell = True) + subprocess.Popen(suricata_command, shell=True) # wait for Suricata to be ready to process traffic before returning # tail suricata_logging_outputs_file (default /tmp/dalton-suricata.log), # look for "engine started." - with open(suricata_logging_outputs_file, 'r') as suri_output_fh: - logger.debug("tailing '%s' to see when engine has started up fully" % suricata_logging_outputs_file) + with open(suricata_logging_outputs_file, "r") as suri_output_fh: + logger.debug( + "tailing '%s' to see when engine has started up fully" + % suricata_logging_outputs_file + ) now = datetime.datetime.now() keep_looking = True while keep_looking: line = suri_output_fh.readline() - if not line or not line.endswith('\n'): + if not line or not line.endswith("\n"): time.sleep(0.1) continue self.suri_startup_log += line @@ -569,25 +636,30 @@ def restart_suricata_socket_mode(self, newconfig): SCONTROL.start_suricata_daemon(newconfig) SC_FIRST_RUN = True + # Error Class class DaltonError(Exception): pass -#*********************** -#*** Custom Imports **** -#*********************** + +# *********************** +# *** Custom Imports **** +# *********************** if USE_SURICATA_SOCKET_CONTROL: try: import suricatasc - except Exception as e: - logger.error(f"Unable to import 'suricatasc' module (SURICATA_SC_PYTHON_MODULE set to '{SURICATA_SC_PYTHON_MODULE}'). Suricata Socket Control will be disabled.") + except Exception: + logger.error( + f"Unable to import 'suricatasc' module (SURICATA_SC_PYTHON_MODULE set to '{SURICATA_SC_PYTHON_MODULE}'). Suricata Socket Control will be disabled." + ) USE_SURICATA_SOCKET_CONTROL = False -#**************************************** -#*** Communication/Printing Functions *** -#**************************************** -def send_update(msg, job_id = None): + +# **************************************** +# *** Communication/Printing Functions *** +# **************************************** +def send_update(msg, job_id=None): global DALTON_API global SENSOR_UID global HTTP_HEADERS @@ -596,195 +668,223 @@ def send_update(msg, job_id = None): url = f"{DALTON_API}/update/?apikey={API_KEY}" params = {} - params['uid'] = SENSOR_UID - params['msg'] = msg - params['job'] = job_id + params["uid"] = SENSOR_UID + params["msg"] = msg + params["job"] = job_id - req = urllib.request.Request(url, urllib.parse.urlencode(params).encode('utf-8'), HTTP_HEADERS) + req = urllib.request.Request( + url, urllib.parse.urlencode(params).encode("utf-8"), HTTP_HEADERS + ) try: urllib.request.urlopen(req, timeout=URLLIB_TIMEOUT) - except Exception as e: - raise Exception(f"Error in sensor '{SENSOR_UID}' while processing job {job_id}. " - "Could not communicate with controller in send_update().\n\tAttempted URL:\n\t" - + re.sub(r'\x26API_KEY=[^\x26]+', "", url) - ) + except Exception: + raise Exception( + f"Error in sensor '{SENSOR_UID}' while processing job {job_id}. " + "Could not communicate with controller in send_update().\n\tAttempted URL:\n\t" + + re.sub(r"\x26API_KEY=[^\x26]+", "", url) + ) + def request_job(): try: - data = urllib.request.urlopen(req_job_url, timeout=URLLIB_TIMEOUT).read().decode('utf-8') - except Exception as e: - raise Exception(f"Error in sensor '{SENSOR_UID}'. " - "Could not communicate with controller in request_job().\n\tAttempted URL:\n\t" - + re.sub(r'\x26API_KEY=[^\x26]+', "", req_job_url) - ) - - if (data == 'sleep'): - #sleep + data = ( + urllib.request.urlopen(req_job_url, timeout=URLLIB_TIMEOUT) + .read() + .decode("utf-8") + ) + except Exception: + raise Exception( + f"Error in sensor '{SENSOR_UID}'. " + "Could not communicate with controller in request_job().\n\tAttempted URL:\n\t" + + re.sub(r"\x26API_KEY=[^\x26]+", "", req_job_url) + ) + + if data == "sleep": + # sleep return None else: - #we got a job? + # we got a job? try: job = json.loads(data) - except Exception as e: - print_error(f"Problem loading json from Dalton Controller; could not parse job id from data: '{data}'.") + except Exception: + print_error( + f"Problem loading json from Dalton Controller; could not parse job id from data: '{data}'." + ) return job + def request_zip(jid): url = f"{DALTON_API}/get_job/{jid}?apikey={API_KEY}" - params = {} req = urllib.request.Request(url, None, HTTP_HEADERS) try: zf = urllib.request.urlopen(req, timeout=URLLIB_TIMEOUT) - except Exception as e: - raise Exception(f"Error in sensor '{SENSOR_UID}'. " - "Could not communicate with controller in request_zip().\n\tAttempted URL:\n\t" - + re.sub(r'\x26API_KEY=[^\x26]+', "", url) - ) + except Exception: + raise Exception( + f"Error in sensor '{SENSOR_UID}'. " + "Could not communicate with controller in request_zip().\n\tAttempted URL:\n\t" + + re.sub(r"\x26API_KEY=[^\x26]+", "", url) + ) zf_path = f"{STORAGE_PATH}/{jid}.zip" - f = open(zf_path,'wb') + f = open(zf_path, "wb") f.write(zf.read()) f.close() return zf_path + # takes a re match object (should be a single byte) and returns it # as printable. Example: byte 0x13 becomes string "\x13". def hexescape(matchobj): - return r'\x{0:02x}'.format(ord(matchobj.group())) + return r"\x{0:02x}".format(ord(matchobj.group())) + # send results back to server. Returns value of 'status' in results dictionary def send_results(): print_debug("send_results() called") print_msg("Sending back results") - nonprintable_re = re.compile(r'[\x80-\xFF]') + nonprintable_re = re.compile(r"[\x80-\xFF]") # create and populate results dictionary results_dict = {} # populate error and status - fh = open(JOB_ERROR_LOG, 'r') + fh = open(JOB_ERROR_LOG, "r") results = fh.read() - results_dict['error'] = results + results_dict["error"] = results # if JOB_ERROR_LOG contains data if results: - results_dict['status'] = "ERROR" + results_dict["status"] = "ERROR" else: - results_dict['status'] = "SUCCESS" + results_dict["status"] = "SUCCESS" fh.close() # populate ids log - results = '' - with open(JOB_IDS_LOG, 'r') as fh: + results = "" + with open(JOB_IDS_LOG, "r") as fh: results = fh.read() if not results: - results_dict['ids'] = "*** No Output ***\n" + results_dict["ids"] = "*** No Output ***\n" else: # make sure we have only ASCII - results_dict['ids'] = "" + results_dict["ids"] = "" for line in results: - results_dict['ids'] += nonprintable_re.sub(hexescape, line) + results_dict["ids"] += nonprintable_re.sub(hexescape, line) # populate alert - fh = open(JOB_ALERT_LOG, 'r') + fh = open(JOB_ALERT_LOG, "r") results = fh.read() if not results: - results_dict['alert'] = "*** No Alerts ***\n" + results_dict["alert"] = "*** No Alerts ***\n" else: - results_dict['alert'] = results + results_dict["alert"] = results fh.close() # populate alert detailed - fh = open(JOB_ALERT_DETAILED_LOG, 'r') + fh = open(JOB_ALERT_DETAILED_LOG, "r") results = fh.read() fh.close() - if not results: # or error identified in results? - results_dict['alert_detailed'] = "" + if not results: # or error identified in results? + results_dict["alert_detailed"] = "" else: - results_dict['alert_detailed'] = results + results_dict["alert_detailed"] = results # populate performance - fh = open(JOB_PERFORMANCE_LOG, 'r') + fh = open(JOB_PERFORMANCE_LOG, "r") results = fh.read() - results_dict['performance'] = results + results_dict["performance"] = results fh.close() # populate debug - fh = open(JOB_DEBUG_LOG, 'r') + fh = open(JOB_DEBUG_LOG, "r") results = fh.read() - results_dict['debug'] = results + results_dict["debug"] = results fh.close() # populate TOTAL_PROCESSING_TIME - results_dict['total_time'] = TOTAL_PROCESSING_TIME + results_dict["total_time"] = TOTAL_PROCESSING_TIME # populate other logs (Suricata only for now) # this file actually contains json; Dalton controller will have to (double) decode since # results_dict is json encoded before it is sent - fh = open(JOB_OTHER_LOGS, 'r') + fh = open(JOB_OTHER_LOGS, "r") results = fh.read() - results_dict['other_logs'] = results + results_dict["other_logs"] = results fh.close() # populate EVE log - fh = open(JOB_EVE_LOG, 'r') + fh = open(JOB_EVE_LOG, "r") results = fh.read() - results_dict['eve'] = results + results_dict["eve"] = results fh.close() # set Zeek JSON - results_dict['zeek_json'] = JOB_ZEEK_JSON + results_dict["zeek_json"] = JOB_ZEEK_JSON - #comment this out for prod - #logger.debug(results_dict) + # comment this out for prod + # logger.debug(results_dict) # convert the dictionary to json json_results_dict = json.dumps(results_dict) - #comment this out for prod - #logger.debug(json_results_dict) + # comment this out for prod + # logger.debug(json_results_dict) - payload = {'json_data': json_results_dict} + payload = {"json_data": json_results_dict} # send results back to server post_results(payload) - return results_dict['status'] + return results_dict["status"] + def post_results(json_data): - #logger.debug("json_data:\n%s" % json_data) - url = "%s/results/%s?SENSOR_UID=%s&apikey=%s" % (DALTON_API, JOB_ID, SENSOR_UID, API_KEY) - req = urllib.request.Request(url, urllib.parse.urlencode(json_data).encode('utf-8'), HTTP_HEADERS) + # logger.debug("json_data:\n%s" % json_data) + url = "%s/results/%s?SENSOR_UID=%s&apikey=%s" % ( + DALTON_API, + JOB_ID, + SENSOR_UID, + API_KEY, + ) + req = urllib.request.Request( + url, urllib.parse.urlencode(json_data).encode("utf-8"), HTTP_HEADERS + ) try: - response = urllib.request.urlopen(req, timeout=URLLIB_TIMEOUT) + urllib.request.urlopen(req, timeout=URLLIB_TIMEOUT) except Exception as e: try: - truncated_url = re.search('(^[^\?]*)', url).group(1) - except: - truncated_url = "unknown" + re.search("(^[^\?]*)", url).group(1) + except Exception: + pass + + raise Exception( + f"Error in sensor '{SENSOR_UID}' while processing job {JOB_ID}" + "Could not communicate with controller in post_results().\n\tAttempted URL:\n\t" + + re.sub(r"\x26API_KEY=[^\x26]+", "", url) + + "\n\tError:\n\t" + + f"{e}" + ) - raise Exception(f"Error in sensor '{SENSOR_UID}' while processing job {job_id}" - "Could not communicate with controller in post_results().\n\tAttempted URL:\n\t" - + re.sub(r'\x26API_KEY=[^\x26]+', "", url) - + "\n\tError:\n\t" - + f"{e}" - ) def error_post_results(error_msg): global SENSOR_UID results_dict = {} - results_dict['error'] = "Unexpected error on Dalton Agent \'%s\', please try your job again or contact admin with this message (see \'About\' page for contact info). Error message:\n\n%s" % (SENSOR_UID, error_msg) - results_dict['status'] = "ERROR" - results_dict['ids'] = '' - results_dict['alert'] = "*** No Alerts ***\n" - results_dict['performance'] = '' - results_dict['debug'] = '' - results_dict['total_time'] = '' + results_dict["error"] = ( + "Unexpected error on Dalton Agent '%s', please try your job again or contact admin with this message (see 'About' page for contact info). Error message:\n\n%s" + % (SENSOR_UID, error_msg) + ) + results_dict["status"] = "ERROR" + results_dict["ids"] = "" + results_dict["alert"] = "*** No Alerts ***\n" + results_dict["performance"] = "" + results_dict["debug"] = "" + results_dict["total_time"] = "" json_results_dict = json.dumps(results_dict) - payload = {'json_data': json_results_dict} + payload = {"json_data": json_results_dict} post_results(payload) + # process alert output from Snort def process_snort_alerts(): print_debug("process_snort_alerts() called") @@ -792,13 +892,16 @@ def process_snort_alerts(): os.system("chmod -R 755 %s" % IDS_LOG_DIRECTORY) job_alert_log_fh = open(JOB_ALERT_LOG, "w") - for alert_file in glob.glob(os.path.join(IDS_LOG_DIRECTORY, "alert-full_dalton-agent*")): + for alert_file in glob.glob( + os.path.join(IDS_LOG_DIRECTORY, "alert-full_dalton-agent*") + ): alert_filehandle = open(alert_file, "r") print_debug("Processing snort alert file %s" % alert_file) job_alert_log_fh.write(alert_filehandle.read()) alert_filehandle.close() job_alert_log_fh.close() + def check_pcaps(): """ Check of the pcaps and alert on potential issues. @@ -813,26 +916,61 @@ def check_pcaps(): if os.path.exists(TCPDUMP_BINARY): for pcap in PCAP_FILES: # check for TCP packets - if len(subprocess.Popen("%s -nn -q -c 1 -r %s -p tcp 2>/dev/null" % (TCPDUMP_BINARY, pcap), shell=True, stdout=subprocess.PIPE).stdout.read()) > 0: + if ( + len( + subprocess.Popen( + "%s -nn -q -c 1 -r %s -p tcp 2>/dev/null" + % (TCPDUMP_BINARY, pcap), + shell=True, + stdout=subprocess.PIPE, + ).stdout.read() + ) + > 0 + ): # check for SYN packets; this only works on IPv4 packets - if len(subprocess.Popen("%s -nn -q -c 1 -r %s \"tcp[tcpflags] & tcp-syn != 0\" 2>/dev/null" % (TCPDUMP_BINARY, pcap), shell=True, stdout=subprocess.PIPE).stdout.read()) == 0: + if ( + len( + subprocess.Popen( + '%s -nn -q -c 1 -r %s "tcp[tcpflags] & tcp-syn != 0" 2>/dev/null' + % (TCPDUMP_BINARY, pcap), + shell=True, + stdout=subprocess.PIPE, + ).stdout.read() + ) + == 0 + ): # check IPv6 packets too - if len(subprocess.Popen("%s -nn -q -c 1 -r %s \"ip6 and tcp and ip6[0x35] & 0x2 != 0\" 2>/dev/null" % (TCPDUMP_BINARY, pcap), shell=True, stdout=subprocess.PIPE).stdout.read()) == 0: - print_error("As Dalton says, \"pain don\'t hurt.\" But an incomplete pcap sure can." - "\n\n" - "The pcap file \'%s\' contains TCP traffic but does not " - "contain any TCP packets with the SYN flag set." - "\n\n" - "Almost all IDS rules that look for TCP traffic require " - "an established connection.\nYou will need to provide a more complete " - "pcap if you want accurate results." - "\n\n" - "If you need help crafting a pcap, Flowsynth may be able to help --\n" - "https://github.com/secureworks/flowsynth" - "\n\n" - "And, \"there's always barber college....\"" % os.path.basename(pcap)) + if ( + len( + subprocess.Popen( + '%s -nn -q -c 1 -r %s "ip6 and tcp and ip6[0x35] & 0x2 != 0" 2>/dev/null' + % (TCPDUMP_BINARY, pcap), + shell=True, + stdout=subprocess.PIPE, + ).stdout.read() + ) + == 0 + ): + print_error( + 'As Dalton says, "pain don\'t hurt." But an incomplete pcap sure can.' + "\n\n" + "The pcap file '%s' contains TCP traffic but does not " + "contain any TCP packets with the SYN flag set." + "\n\n" + "Almost all IDS rules that look for TCP traffic require " + "an established connection.\nYou will need to provide a more complete " + "pcap if you want accurate results." + "\n\n" + "If you need help crafting a pcap, Flowsynth may be able to help --\n" + "https://github.com/secureworks/flowsynth" + "\n\n" + 'And, "there\'s always barber college...."' + % os.path.basename(pcap) + ) else: - print_debug("In check_pcaps() -- no tcpdump binary found at %s" % TCPDUMP_BINARY) + print_debug( + "In check_pcaps() -- no tcpdump binary found at %s" % TCPDUMP_BINARY + ) except Exception as e: if not str(e).startswith("As Dalton says"): print_debug("Error doing TCP SYN check in check_pcaps():\n%s" % e) @@ -846,39 +984,46 @@ def check_pcaps(): snaplen = 65535 # get first 40 bytes of pcap file - with open(pcap, 'rb') as fh: + with open(pcap, "rb") as fh: bytes = fh.read(44) - magic = binascii.hexlify(bytes[0:4]).decode('ascii') - if magic.lower() == '0a0d0d0a': + magic = binascii.hexlify(bytes[0:4]).decode("ascii") + if magic.lower() == "0a0d0d0a": # this is pcapng and these aren't the byte-order magic bytes snaplen_offset = 40 pcapng = True # get the correct byte-order magic bytes for pcapng - magic = binascii.hexlify(bytes[8:12]).decode('ascii') + magic = binascii.hexlify(bytes[8:12]).decode("ascii") else: # this is libpcap, we have the magic pcapng = False # now determine endian-ness - if magic.lower() == 'a1b2c3d4': + if magic.lower() == "a1b2c3d4": # this is "big endian" little_endian = False - elif magic.lower() == '4d3c2b1a' or magic.lower() == 'd4c3b2a1': + elif magic.lower() == "4d3c2b1a" or magic.lower() == "d4c3b2a1": # this is little endian little_endian = True else: - print_debug("in check_pcaps() - Pcap Byte-Order Magic field not found in file \'%s\'. Is this a valid pcap?" % os.path.basename(pcap)) + print_debug( + "in check_pcaps() - Pcap Byte-Order Magic field not found in file '%s'. Is this a valid pcap?" + % os.path.basename(pcap) + ) continue # get snaplen if little_endian: - snaplen = struct.unpack('i', bytes[snaplen_offset:snaplen_offset+4])[0] + snaplen = struct.unpack( + ">i", bytes[snaplen_offset : snaplen_offset + 4] + )[0] # Python 2.4 doesn't support this so doing it the ugly way - #print_debug("Packet capture file \'%s\' is format %s, %s, and has snaplen of %d bytes." % (os.path.basename(pcap), ('pcapng' if pcapng else 'libpcap'), ('little endian' if little_endian else 'big endian'), snaplen)) - debug_msg = "Packet capture file \'%s\' is format " % os.path.basename(pcap) + # print_debug("Packet capture file \'%s\' is format %s, %s, and has snaplen of %d bytes." % (os.path.basename(pcap), ('pcapng' if pcapng else 'libpcap'), ('little endian' if little_endian else 'big endian'), snaplen)) + debug_msg = "Packet capture file '%s' is format " % os.path.basename(pcap) if pcapng: debug_msg += "pcapng, " else: @@ -890,14 +1035,20 @@ def check_pcaps(): print_debug(debug_msg) if snaplen < 65535: - print_debug("Warning: \'%s\' was captured using a snaplen of %d bytes. This may mean you have truncated packets." % (os.path.basename(pcap), snaplen)) + print_debug( + "Warning: '%s' was captured using a snaplen of %d bytes. This may mean you have truncated packets." + % (os.path.basename(pcap), snaplen) + ) # validate snaplen if snaplen < 1514: - warning_msg = '' + warning_msg = "" if not os.path.getsize(JOB_ERROR_LOG) == 0: warning_msg += "\n----------------\n\n" - warning_msg += "Warning: \'%s\' was captured using a snaplen of %d bytes. This may mean you have truncated packets." % (os.path.basename(pcap), snaplen) + warning_msg += ( + "Warning: '%s' was captured using a snaplen of %d bytes. This may mean you have truncated packets." + % (os.path.basename(pcap), snaplen) + ) if snaplen == 1500: warning_msg += "\n\nSome sandboxes (Bluecoat/Norman) will put a hardcoded snaplen of 1500 bytes\n" warning_msg += "on pcaps even when the packets are larger than 1500 bytes. This can result in the sensor throwing away these\n" @@ -910,23 +1061,36 @@ def check_pcaps(): if not str(e).startswith("Warning:"): print_debug("Error doing snaplen check in check_pcaps(): %s" % e) -#************************* -#**** Snort Functions **** -#************************* + +# ************************* +# **** Snort Functions **** +# ************************* def run_snort(): print_debug("run_snort() called") IDS_BUFFERS_LOG = os.path.join(IDS_LOG_DIRECTORY, "dalton-buffers.log") # note: if we don't have '--treat-drop-as-alert' then some alerts in a stream that has already triggered a 'drop' rule won't fire since they are assumed to already blocked by the DAQ - snort_command = "%s -Q --daq dump --daq-dir /usr/lib/daq/ --daq-var load-mode=read-file --daq-var file=/tmp/inline-out.pcap -l %s -c %s -k none -X --conf-error-out --process-all-events --treat-drop-as-alert --pcap-dir=%s --buffer-dump-alert=%s 2>&1" % (IDS_BINARY, IDS_LOG_DIRECTORY, IDS_CONFIG_FILE, os.path.split(PCAP_FILES[0])[0], IDS_BUFFERS_LOG) + snort_command = ( + "%s -Q --daq dump --daq-dir /usr/lib/daq/ --daq-var load-mode=read-file --daq-var file=/tmp/inline-out.pcap -l %s -c %s -k none -X --conf-error-out --process-all-events --treat-drop-as-alert --pcap-dir=%s --buffer-dump-alert=%s 2>&1" + % ( + IDS_BINARY, + IDS_LOG_DIRECTORY, + IDS_CONFIG_FILE, + os.path.split(PCAP_FILES[0])[0], + IDS_BUFFERS_LOG, + ) + ) print_msg("Starting Snort and Running Pcap(s)...") print_debug("Running Snort with the following command command:\n%s" % snort_command) snort_output_fh = open(JOB_IDS_LOG, "w") - subprocess.call(snort_command, shell = True, stderr=subprocess.STDOUT, stdout=snort_output_fh) + subprocess.call( + snort_command, shell=True, stderr=subprocess.STDOUT, stdout=snort_output_fh + ) snort_output_fh.close() -#************************ -#** Suricata Functions ** -#************************ + +# ************************ +# ** Suricata Functions ** +# ************************ def run_suricata_sc(): @@ -939,11 +1103,20 @@ def run_suricata_sc(): config_hash = hash_file(IDS_CONFIG_FILE) ruleset_hash = hash_file(sorted(glob.glob(os.path.join(JOB_DIRECTORY, "*.rules")))) logger.debug("NEW config_hash: %s, ruleset_hash: %s" % (config_hash, ruleset_hash)) - logger.debug("OLD config_hash: %s, ruleset_hash: %s" % (SCONTROL.config_hash, SCONTROL.ruleset_hash)) - if (not (ruleset_hash == SCONTROL.ruleset_hash and config_hash == SCONTROL.config_hash)) \ - and SCONTROL.suricata_is_running: + logger.debug( + "OLD config_hash: %s, ruleset_hash: %s" + % (SCONTROL.config_hash, SCONTROL.ruleset_hash) + ) + if ( + not ( + ruleset_hash == SCONTROL.ruleset_hash + and config_hash == SCONTROL.config_hash + ) + ) and SCONTROL.suricata_is_running: # if hashes don't match, shutdown suri via socket, start new suri, update hashes, run - print_debug("Suricata Socket Control: new hashes found, restarting Suricata.....") + print_debug( + "Suricata Socket Control: new hashes found, restarting Suricata....." + ) SCONTROL.ruleset_hash = ruleset_hash SCONTROL.config_hash = config_hash SCONTROL.restart_suricata_socket_mode(newconfig=IDS_CONFIG_FILE) @@ -958,7 +1131,15 @@ def run_suricata_sc(): SCONTROL.connect() if SC_FIRST_RUN: - bug4225_versions = ["5.0.5", "5.0.6", "5.0.7", "6.0.1", "6.0.2", "6.0.3", "7.0.0-dev"] + bug4225_versions = [ + "5.0.5", + "5.0.6", + "5.0.7", + "6.0.1", + "6.0.2", + "6.0.3", + "7.0.0-dev", + ] if SENSOR_ENGINE_VERSION_ORIG in bug4225_versions: # Re: https://redmine.openinfosecfoundation.org/issues/4225 # Certain Suricata versions will throw an error on the @@ -968,25 +1149,32 @@ def run_suricata_sc(): # To work around this for now, we make an initial dummy "pcap-file" # request with a small benign pcap so the error condition can # work itself out. - dummy_pcap_bytes = b'\xD4\xC3\xB2\xA1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00' \ - b'\xFF\xFF\x00\x00\x01\x00\x00\x00\x20\xD0\x23\x60\x84\xE8\x0C\x00' \ - b'\x2B\x00\x00\x00\x2B\x00\x00\x00\x53\x07\x1D\x71\x7F\xB3\xCB\xDA' \ - b'\x12\x16\x20\xAF\x08\x00\x45\x00\x00\x1D\x00\x01\x00\x00\x40\x11' \ - b'\xB0\x32\xC0\xA8\x7A\x95\xAC\x10\xE3\x4E\x1F\x48\x05\x39\x00\x09' \ - b'\xCC\xBD\x44' + dummy_pcap_bytes = ( + b"\xd4\xc3\xb2\xa1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\xff\xff\x00\x00\x01\x00\x00\x00\x20\xd0\x23\x60\x84\xe8\x0c\x00" + b"\x2b\x00\x00\x00\x2b\x00\x00\x00\x53\x07\x1d\x71\x7f\xb3\xcb\xda" + b"\x12\x16\x20\xaf\x08\x00\x45\x00\x00\x1d\x00\x01\x00\x00\x40\x11" + b"\xb0\x32\xc0\xa8\x7a\x95\xac\x10\xe3\x4e\x1f\x48\x05\x39\x00\x09" + b"\xcc\xbd\x44" + ) dummy_pcap_file = "/tmp/dalton-dummy-pcap" with open(dummy_pcap_file, "wb") as dfh: dfh.write(dummy_pcap_bytes) - logger.debug("Sending dummy pcap to socket control to work around Redmine 4225") + logger.debug( + "Sending dummy pcap to socket control to work around Redmine 4225" + ) resp = SCONTROL.send_command(f"pcap-file {dummy_pcap_file} /tmp") logger.debug(f"Sent dummy pcap. Response: {resp}") - while int(SCONTROL.send_command("pcap-file-number")) > 0 or SCONTROL.send_command("pcap-current") != "\"None\"": + while ( + int(SCONTROL.send_command("pcap-file-number")) > 0 + or SCONTROL.send_command("pcap-current") != '"None"' + ): # wait for dummy pcap run to finish # TODO: check for timeout/infinite loop? - time.sleep(.05) + time.sleep(0.05) SC_FIRST_RUN = False # skip over output from dummy pcap run in global suri output log - with open(suricata_logging_outputs_file, 'r') as fh: + with open(suricata_logging_outputs_file, "r") as fh: fh.seek(SCONTROL.log_offset, 0) fh.read() SCONTROL.log_offset = fh.tell() @@ -1004,16 +1192,19 @@ def run_suricata_sc(): files_remaining = 1 current_pcap = "dummy.pcap" # TODO: change dynamically based on number of pcap files? - sleep_time = .1 - while files_remaining > 0 or current_pcap != "\"None\"": + sleep_time = 0.1 + while files_remaining > 0 or current_pcap != '"None"': time.sleep(sleep_time) # TODO: try/catch ??? files_remaining = int(SCONTROL.send_command("pcap-file-number")) current_pcap = SCONTROL.send_command("pcap-current") - #logger.debug(f"files_remaining: {files_remaining}, current_pcap: {current_pcap}") - logger.debug("In run_suricata_sc(): all pcaps done running ... closing connection to socket.") + # logger.debug(f"files_remaining: {files_remaining}, current_pcap: {current_pcap}") + logger.debug( + "In run_suricata_sc(): all pcaps done running ... closing connection to socket." + ) SCONTROL.close() + def run_suricata(): print_debug("run_suricata() called") if not IDS_BINARY: @@ -1025,16 +1216,23 @@ def run_suricata(): if LooseVersion(SENSOR_ENGINE_VERSION_ORIG) >= LooseVersion("2.0"): # not sure if the '-k' option was added in Suri 2.0 or earlier but for now just doing this for v2 and later add_options = "-k none" - except Exception as e: + except Exception: add_options = "" - suricata_command = "%s -c %s -l %s %s " % (IDS_BINARY, IDS_CONFIG_FILE, IDS_LOG_DIRECTORY, add_options) + suricata_command = "%s -c %s -l %s %s " % ( + IDS_BINARY, + IDS_CONFIG_FILE, + IDS_LOG_DIRECTORY, + add_options, + ) if len(PCAP_FILES) > 1: suricata_command += "-r %s" % (os.path.dirname(PCAP_FILES[0])) else: suricata_command += "-r %s" % (PCAP_FILES[0]) print_debug("Running suricata with the following command:\n%s" % suricata_command) suri_output_fh = open(JOB_IDS_LOG, "w") - subprocess.call(suricata_command, shell = True, stderr=subprocess.STDOUT, stdout=suri_output_fh) + subprocess.call( + suricata_command, shell=True, stderr=subprocess.STDOUT, stdout=suri_output_fh + ) suri_output_fh.close() @@ -1043,16 +1241,29 @@ def run_suricata(): def generate_fast_pattern(): print_debug("generate_fast_pattern() called") print_msg("Generating Fast Pattern Info") - if SENSOR_ENGINE.startswith('suri'): + if SENSOR_ENGINE.startswith("suri"): if not IDS_BINARY: print_error("No Suricata binary found on system.") - suricata_command = "%s -c %s -l %s --engine-analysis" % (IDS_BINARY, IDS_CONFIG_FILE, IDS_LOG_DIRECTORY) - print_debug("Running suricata with the following command to get fast pattern info:\n%s" % suricata_command) + suricata_command = "%s -c %s -l %s --engine-analysis" % ( + IDS_BINARY, + IDS_CONFIG_FILE, + IDS_LOG_DIRECTORY, + ) + print_debug( + "Running suricata with the following command to get fast pattern info:\n%s" + % suricata_command + ) # send output to /dev/null for now suri_output_fh = open(os.devnull, "wb") - subprocess.call(suricata_command, shell = True, stderr=subprocess.STDOUT, stdout=suri_output_fh) + subprocess.call( + suricata_command, + shell=True, + stderr=subprocess.STDOUT, + stdout=suri_output_fh, + ) suri_output_fh.close() + def process_suri_alerts(): global JOB_ALERT_LOG, IDS_LOG_DIRECTORY print_debug("process_suri_alerts() called") @@ -1068,7 +1279,8 @@ def process_suri_alerts(): alert_filehandle.close() job_alert_log_fh.close() else: - print_debug("No alerts found. File \'%s\' does not exist." % alerts_file) + print_debug("No alerts found. File '%s' does not exist." % alerts_file) + def process_eve_log(): print_debug("process_eve_log() called") @@ -1079,7 +1291,8 @@ def process_eve_log(): shutil.copyfile(eve_file, JOB_EVE_LOG) print_debug(f"copying {eve_file} to {JOB_EVE_LOG}") else: - print_debug("No EVE JSON file found. File \'%s\' does not exist." % eve_file) + print_debug("No EVE JSON file found. File '%s' does not exist." % eve_file) + def process_other_logs(other_logs): """ @@ -1094,48 +1307,78 @@ def process_other_logs(other_logs): if not os.path.exists("%s/%s" % (IDS_LOG_DIRECTORY, other_logs[log_name])): log_name_new = other_logs[log_name].replace("-", "_") if log_name_new != other_logs[log_name]: - print_debug("Log file \'%s\' not present, trying \'%s\'..." % (other_logs[log_name], log_name_new)) + print_debug( + "Log file '%s' not present, trying '%s'..." + % (other_logs[log_name], log_name_new) + ) other_logs[log_name] = log_name_new if os.path.exists("%s/%s" % (IDS_LOG_DIRECTORY, other_logs[log_name])): log_fh = open("%s/%s" % (IDS_LOG_DIRECTORY, other_logs[log_name]), "r") all_other_logs[log_name] = log_fh.read() log_fh.close() if all_other_logs[log_name] == "": - print_debug("log \"%s\" is empty, not including" % log_name) + print_debug('log "%s" is empty, not including' % log_name) del all_other_logs[log_name] else: - print_debug("Requested log file \'%s\' not present, skipping." % other_logs[log_name]) + print_debug( + "Requested log file '%s' not present, skipping." + % other_logs[log_name] + ) other_logs_fh = open(JOB_OTHER_LOGS, "w") other_logs_fh.write(json.dumps(all_other_logs)) other_logs_fh.close() else: print_debug("No additional logs requested.") + def check_for_errors(tech): - """ checks the IDS output for error messages """ + """checks the IDS output for error messages""" print_debug("check_for_errors() called") error_lines = [] try: ids_log_fh = open(JOB_IDS_LOG, "r") for line in ids_log_fh: - if tech.startswith('suri'): - if ("" in line or "Error: " in line or line.startswith("ERROR") or line.startswith("Failed to parse configuration file")): + if tech.startswith("suri"): + if ( + "" in line + or "Error: " in line + or line.startswith("ERROR") + or line.startswith("Failed to parse configuration file") + ): error_lines.append(line) if "bad dump file format" in line or "unknown file format" in line: - error_lines.append("Bad pcap file(s) submitted to Suricata. Pcap files should be in libpcap format (pcapng is not supported in older Suricata versions).\n") - elif tech.startswith('snort'): - if "ERROR:" in line or "FATAL" in line or "Fatal Error" in line or "Segmentation fault" in line or line.startswith("Error "): + error_lines.append( + "Bad pcap file(s) submitted to Suricata. Pcap files should be in libpcap format (pcapng is not supported in older Suricata versions).\n" + ) + elif tech.startswith("snort"): + if ( + "ERROR:" in line + or "FATAL" in line + or "Fatal Error" in line + or "Segmentation fault" in line + or line.startswith("Error ") + ): error_lines.append(line) if "unknown file format" in line: - error_lines.append("Bad pcap file(s) submitted to Snort. Pcap files should be in libpcap or pcapng format.\n") + error_lines.append( + "Bad pcap file(s) submitted to Snort. Pcap files should be in libpcap or pcapng format.\n" + ) else: - logger.warn(f"Unexpected engine value passed to check_for_errors(): {tech}") + logger.warn( + f"Unexpected engine value passed to check_for_errors(): {tech}" + ) ids_log_fh.close() except Exception as e: - print_error("Error reading IDS output file \'%s\'. Error:\n\n%s" % (JOB_IDS_LOG, e)) + print_error( + "Error reading IDS output file '%s'. Error:\n\n%s" % (JOB_IDS_LOG, e) + ) if len(error_lines) > 0: - print_error("Error message(s) found in IDS output. See \"IDS Engine\" tab for more details and/or context:\n\n%s" % '\n'.join(error_lines)) + print_error( + 'Error message(s) found in IDS output. See "IDS Engine" tab for more details and/or context:\n\n%s' + % "\n".join(error_lines) + ) + # process unified2 data and populate JOB_ALERT_DETAILED_LOG (only for sensors # that generate unified2 logs such as Snort and Suricata) @@ -1163,7 +1406,9 @@ def process_unified2_logs(): unified2_files = list(unified2_files) # now, cat all the files together, base64 encode them, and write to JOB_ALERT_DETAILED_LOG # copy first file instead of catting to it to preserve original on agent if needed - u2_combined_file = os.path.join(IDS_LOG_DIRECTORY, "dalton-unified2-combined.alerts") + u2_combined_file = os.path.join( + IDS_LOG_DIRECTORY, "dalton-unified2-combined.alerts" + ) shutil.copyfile(unified2_files[0], u2_combined_file) try: combined_fh = open(u2_combined_file, "ab") @@ -1178,13 +1423,17 @@ def process_unified2_logs(): # b64 it and write! try: - with open(JOB_ALERT_DETAILED_LOG, 'wb') as job_alert_detailed_log_fh: - with open(u2_combined_file, 'rb') as u2_fh: + with open(JOB_ALERT_DETAILED_LOG, "wb") as job_alert_detailed_log_fh: + with open(u2_combined_file, "rb") as u2_fh: job_alert_detailed_log_fh.write(base64.b64encode(u2_fh.read())) except Exception as e: - print_debug("Error processing unified2 files and base64 encoding them for transmission ... bailing. Error: %s" % e) + print_debug( + "Error processing unified2 files and base64 encoding them for transmission ... bailing. Error: %s" + % e + ) return + # process performance output (Snort and Suricata) def process_performance_logs(): print_debug("process_performance_logs() called") @@ -1192,37 +1441,58 @@ def process_performance_logs(): os.system("chmod -R 755 %s" % IDS_LOG_DIRECTORY) job_performance_log_fh = open(JOB_PERFORMANCE_LOG, "w") if len(glob.glob(os.path.join(IDS_LOG_DIRECTORY, "dalton-rule_perf*"))) > 0: - for perf_file in glob.glob(os.path.join(IDS_LOG_DIRECTORY, "dalton-rule_perf*")): + for perf_file in glob.glob( + os.path.join(IDS_LOG_DIRECTORY, "dalton-rule_perf*") + ): perf_filehandle = open(perf_file, "r") print_debug("Processing rule performance log file %s" % perf_file) job_performance_log_fh.write(perf_filehandle.read()) job_performance_log_fh.write("\n") perf_filehandle.close() else: - print_debug("No rules performance log(s) found. File \'%s\' does not exist." % "dalton-rule_perf*") + print_debug( + "No rules performance log(s) found. File '%s' does not exist." + % "dalton-rule_perf*" + ) job_performance_log_fh.close() -#************************ -#**** Zeek Functions **** -#************************ +# ************************ +# **** Zeek Functions **** +# ************************ def run_zeek(json_logs): print_debug("run_zeek() called") - zeek_command = "cd %s && %s -C -r %s" % (IDS_LOG_DIRECTORY, IDS_BINARY, PCAP_FILES[0]) + zeek_command = "cd %s && %s -C -r %s" % ( + IDS_LOG_DIRECTORY, + IDS_BINARY, + PCAP_FILES[0], + ) if json_logs: zeek_command += " -e 'redef LogAscii::use_json=T;redef LogAscii::json_timestamps=JSON::TS_ISO8601;'" - if len([f for f in os.listdir('/opt/dalton-agent/zeek_scripts/') if not f.startswith('.')]) > 0: + if ( + len( + [ + f + for f in os.listdir("/opt/dalton-agent/zeek_scripts/") + if not f.startswith(".") + ] + ) + > 0 + ): zeek_command += " /opt/dalton-agent/zeek_scripts/*" print_msg("Starting Zeek and Running Pcap(s)...") print_debug("Running Zeek with the following command command:\n%s" % zeek_command) zeek_output_fh = open(JOB_IDS_LOG, "w") zeek_error_fh = open(JOB_ERROR_LOG, "w") - subprocess.call(zeek_command, shell=True, stderr=zeek_error_fh, stdout=zeek_output_fh) + subprocess.call( + zeek_command, shell=True, stderr=zeek_error_fh, stdout=zeek_output_fh + ) zeek_output_fh.close() zeek_error_fh.close() + # process logs from Zeek def process_zeek_logs(): print_debug("process_zeek_logs() called") @@ -1231,20 +1501,34 @@ def process_zeek_logs(): logs = {} for log_file in os.listdir(IDS_LOG_DIRECTORY): - logs[log_file.split('.')[0]] = log_file + logs[log_file.split(".")[0]] = log_file return logs -#**************************** -#*** Submit Job Functions *** -#**************************** +# **************************** +# *** Submit Job Functions *** +# **************************** # resets the global variables between jobs def reset_globals(): - global JOB_ID, PCAP_FILES, IDS_RULES_FILES, IDS_CONFIG_FILE, \ - JOB_DIRECTORY, JOB_LOG_DIRECTORY, JOB_ERROR_LOG, JOB_IDS_LOG, \ - JOB_DEBUG_LOG, JOB_ALERT_LOG, JOB_ALERT_DETAILED_LOG, JOB_PERFORMANCE_LOG, \ - IDS_LOG_DIRECTORY, TOTAL_PROCESSING_TIME, JOB_OTHER_LOGS, JOB_EVE_LOG, JOB_ZEEK_JSON + global \ + JOB_ID, \ + PCAP_FILES, \ + IDS_RULES_FILES, \ + IDS_CONFIG_FILE, \ + JOB_DIRECTORY, \ + JOB_LOG_DIRECTORY, \ + JOB_ERROR_LOG, \ + JOB_IDS_LOG, \ + JOB_DEBUG_LOG, \ + JOB_ALERT_LOG, \ + JOB_ALERT_DETAILED_LOG, \ + JOB_PERFORMANCE_LOG, \ + IDS_LOG_DIRECTORY, \ + TOTAL_PROCESSING_TIME, \ + JOB_OTHER_LOGS, \ + JOB_EVE_LOG, \ + JOB_ZEEK_JSON JOB_ID = None PCAP_FILES = [] @@ -1265,34 +1549,50 @@ def reset_globals(): # end dalton's logs # used by snort for logs/alerts IDS_LOG_DIRECTORY = None - TOTAL_PROCESSING_TIME = '' + TOTAL_PROCESSING_TIME = "" JOB_OTHER_LOGS = None JOB_ZEEK_JSON = False + # primary function # gets passed directory of submitted files (rules file, pcap file(s)) and job ID def submit_job(job_id, job_directory): - global JOB_ID, PCAP_FILES, IDS_RULES_FILES, IDS_CONFIG_FILE, \ - JOB_DIRECTORY, JOB_LOG_DIRECTORY, JOB_ERROR_LOG, JOB_IDS_LOG, \ - JOB_DEBUG_LOG, JOB_ALERT_LOG, JOB_ALERT_DETAILED_LOG, JOB_OTHER_LOGS, \ - JOB_PERFORMANCE_LOG, IDS_LOG_DIRECTORY, TOTAL_PROCESSING_TIME, IDS_BINARY, \ - JOB_EVE_LOG, USE_SURICATA_SOCKET_CONTROL, JOB_ZEEK_JSON + global \ + JOB_ID, \ + PCAP_FILES, \ + IDS_RULES_FILES, \ + IDS_CONFIG_FILE, \ + JOB_DIRECTORY, \ + JOB_LOG_DIRECTORY, \ + JOB_ERROR_LOG, \ + JOB_IDS_LOG, \ + JOB_DEBUG_LOG, \ + JOB_ALERT_LOG, \ + JOB_ALERT_DETAILED_LOG, \ + JOB_OTHER_LOGS, \ + JOB_PERFORMANCE_LOG, \ + IDS_LOG_DIRECTORY, \ + TOTAL_PROCESSING_TIME, \ + IDS_BINARY, \ + JOB_EVE_LOG, \ + USE_SURICATA_SOCKET_CONTROL, \ + JOB_ZEEK_JSON # reset and populate global vars reset_globals() (JOB_ID, JOB_DIRECTORY) = (job_id, job_directory) - JOB_DIRECTORY = JOB_DIRECTORY.rstrip('/') - JOB_LOG_DIRECTORY = '%s/output_logs' % JOB_DIRECTORY + JOB_DIRECTORY = JOB_DIRECTORY.rstrip("/") + JOB_LOG_DIRECTORY = "%s/output_logs" % JOB_DIRECTORY if os.path.isdir(JOB_LOG_DIRECTORY): shutil.rmtree(JOB_LOG_DIRECTORY) os.makedirs(JOB_LOG_DIRECTORY) - JOB_ERROR_LOG = '%s/error.log' % JOB_LOG_DIRECTORY - JOB_IDS_LOG = '%s/ids.log' % JOB_LOG_DIRECTORY - JOB_DEBUG_LOG = '%s/debug.log' % JOB_LOG_DIRECTORY - JOB_ALERT_LOG = '%s/alerts.log' % JOB_LOG_DIRECTORY - JOB_ALERT_DETAILED_LOG = '%s/alerts_detailed.log' % JOB_LOG_DIRECTORY - JOB_OTHER_LOGS = '%s/other_logs.json' % JOB_LOG_DIRECTORY - JOB_PERFORMANCE_LOG = '%s/performance.log' % JOB_LOG_DIRECTORY - IDS_CONFIG_FILE = '%s/snort.conf' % JOB_DIRECTORY + JOB_ERROR_LOG = "%s/error.log" % JOB_LOG_DIRECTORY + JOB_IDS_LOG = "%s/ids.log" % JOB_LOG_DIRECTORY + JOB_DEBUG_LOG = "%s/debug.log" % JOB_LOG_DIRECTORY + JOB_ALERT_LOG = "%s/alerts.log" % JOB_LOG_DIRECTORY + JOB_ALERT_DETAILED_LOG = "%s/alerts_detailed.log" % JOB_LOG_DIRECTORY + JOB_OTHER_LOGS = "%s/other_logs.json" % JOB_LOG_DIRECTORY + JOB_PERFORMANCE_LOG = "%s/performance.log" % JOB_LOG_DIRECTORY + IDS_CONFIG_FILE = "%s/snort.conf" % JOB_DIRECTORY JOB_EVE_LOG = os.path.join(JOB_LOG_DIRECTORY, "dalton-eve.json") # touch log files @@ -1306,7 +1606,9 @@ def submit_job(job_id, job_directory): open(JOB_EVE_LOG, "w").close() print_debug(datetime.datetime.now().strftime("%b %d %Y %H:%M:%S")) - print_debug(f"Agent Name: {SENSOR_UID}\nAgent Version: {AGENT_VERSION}\nIDS Engine: {SENSOR_ENGINE} {SENSOR_ENGINE_VERSION}\nDalton API: {DALTON_API}") + print_debug( + f"Agent Name: {SENSOR_UID}\nAgent Version: {AGENT_VERSION}\nIDS Engine: {SENSOR_ENGINE} {SENSOR_ENGINE_VERSION}\nDalton API: {DALTON_API}" + ) print_debug("submit_job() called") @@ -1320,11 +1622,19 @@ def submit_job(job_id, job_directory): print_debug("manifest.json: %s" % manifest_data) # use Suricata Socket Control (Suricata only) - if SENSOR_ENGINE.startswith('suri'): + if SENSOR_ENGINE.startswith("suri"): try: - useSuricataSC = manifest_data[0]['use-suricatasc'] + useSuricataSC = manifest_data[0]["use-suricatasc"] if useSuricataSC != USE_SURICATA_SOCKET_CONTROL: - if useSuricataSC and float('.'.join(prefix_strip(SENSOR_ENGINE_VERSION_ORIG).split('.')[:2])) < 3.0: + if ( + useSuricataSC + and float( + ".".join( + prefix_strip(SENSOR_ENGINE_VERSION_ORIG).split(".")[:2] + ) + ) + < 3.0 + ): msg = f"Dalton Agent does not support Suricata Socket Control for Suricata versions before 3.0. This is running Suricata version {eng_ver}. Cannot use Suricata Socket Control Mode." logger.warn(msg) # should not be necessary but just in case @@ -1339,7 +1649,7 @@ def submit_job(job_id, job_directory): trackPerformance = False try: - trackPerformance = manifest_data[0]['track-performance'] + trackPerformance = manifest_data[0]["track-performance"] except Exception: trackPerformance = False @@ -1349,47 +1659,46 @@ def submit_job(job_id, job_directory): # in the submitted engine.conf getFastPattern = False try: - getFastPattern = manifest_data[0]['get-fast-pattern'] + getFastPattern = manifest_data[0]["get-fast-pattern"] except Exception: getFastPattern = False # engine statistics, Snort only - getEngineStats = False try: - getEngineStats = manifest_data[0]['get-engine-stats'] + manifest_data[0]["get-engine-stats"] except Exception: - getEngineStats = False + pass # process unified2 alerts, supported for Snort and suricata getAlertDetailed = False try: - getAlertDetailed = manifest_data[0]['alert-detailed'] + getAlertDetailed = manifest_data[0]["alert-detailed"] except Exception: getAlertDetailed = False # get other logs (Suricata only for now) getOtherLogs = False try: - getOtherLogs = manifest_data[0]['get-other-logs'] + getOtherLogs = manifest_data[0]["get-other-logs"] except Exception: getOtherLogs = False # get dumps from buffers getBufferDumps = False try: - getBufferDumps = manifest_data[0]['get-buffer-dumps'] + getBufferDumps = manifest_data[0]["get-buffer-dumps"] except Exception: getBufferDumps = False # Zeek JSON logging JOB_ZEEK_JSON = False try: - JOB_ZEEK_JSON = manifest_data[0]['zeek-json-logs'] + JOB_ZEEK_JSON = manifest_data[0]["zeek-json-logs"] except Exception: JOB_ZEEK_JSON = False # make a directory for engine to use for alert, perf, and other sundry logs - IDS_LOG_DIRECTORY = '%s/raw_ids_logs' % JOB_DIRECTORY + IDS_LOG_DIRECTORY = "%s/raw_ids_logs" % JOB_DIRECTORY if os.path.isdir(IDS_LOG_DIRECTORY): shutil.rmtree(IDS_LOG_DIRECTORY) os.makedirs(IDS_LOG_DIRECTORY) @@ -1401,48 +1710,70 @@ def submit_job(job_id, job_directory): # Agents. cdir = "/etc/snort" if os.path.isdir(cdir): - file_list = ["classification.config", "file_magic.conf", "gen-msg.map", "reference.config", "threshold.conf", "unicode.map"] + file_list = [ + "classification.config", + "file_magic.conf", + "gen-msg.map", + "reference.config", + "threshold.conf", + "unicode.map", + ] for file in file_list: if os.path.isfile(os.path.join(cdir, file)): - shutil.copyfile(os.path.join(cdir, file), os.path.join(JOB_DIRECTORY, file)) + shutil.copyfile( + os.path.join(cdir, file), os.path.join(JOB_DIRECTORY, file) + ) # pcaps and config should be in manifest IDS_CONFIG_FILE = None - if not SENSOR_ENGINE.startswith('zeek'): + if not SENSOR_ENGINE.startswith("zeek"): try: - IDS_CONFIG_FILE = os.path.join(JOB_DIRECTORY, os.path.basename(manifest_data[0]['engine-conf'])) + IDS_CONFIG_FILE = os.path.join( + JOB_DIRECTORY, os.path.basename(manifest_data[0]["engine-conf"]) + ) except Exception: print_error("Could not extract engine configuration file from job.") try: - PCAP_FILES = [os.path.join(JOB_DIRECTORY, PCAP_DIR, os.path.basename(cap)) for cap in manifest_data[0]['pcaps']] + PCAP_FILES = [ + os.path.join(JOB_DIRECTORY, PCAP_DIR, os.path.basename(cap)) + for cap in manifest_data[0]["pcaps"] + ] for pcap_file in PCAP_FILES: # move pcaps to their own directory (PCAP_DIR) since at this point they are in the JOB_DIRECTORY; can # be easier for the engine to process when there are multiple pcaps (base, name) = os.path.split(os.path.split(pcap_file)[0]) - shutil.move(os.path.join(os.path.dirname(os.path.dirname(pcap_file)), os.path.basename(pcap_file)), pcap_file) + shutil.move( + os.path.join( + os.path.dirname(os.path.dirname(pcap_file)), + os.path.basename(pcap_file), + ), + pcap_file, + ) except Exception as e: print_error("Could not determine pcap files in job.") - logger.debug("Problem moving pcaps to directory '%s': %s" % (os.path.join(JOB_DIRECTORY, PCAP_DIR), e)) - + logger.debug( + "Problem moving pcaps to directory '%s': %s" + % (os.path.join(JOB_DIRECTORY, PCAP_DIR), e) + ) # parse job dir for configs and pcaps logger.debug("Parsing job directory: %s" % JOB_DIRECTORY) for file in glob.glob(os.path.join(JOB_DIRECTORY, "*")): if not os.path.isfile(file): continue - if os.path.splitext(file)[1] == '.rules': + if os.path.splitext(file)[1] == ".rules": IDS_RULES_FILES.append(file) # input validation (sort of) if not PCAP_FILES: print_error("No pcap files found") - if not IDS_RULES_FILES and not SENSOR_ENGINE.startswith('zeek'): + if not IDS_RULES_FILES and not SENSOR_ENGINE.startswith("zeek"): print_error("No rules files found") if not JOB_ID: print_error("job id not defined") - if SENSOR_ENGINE.startswith('snort'): + if SENSOR_ENGINE.startswith("snort"): snort_conf_fh = open(IDS_CONFIG_FILE, "a") # include rules in config file @@ -1460,20 +1791,26 @@ def submit_job(job_id, job_directory): snort_conf_fh.close() - if SENSOR_ENGINE.startswith('suri'): + if SENSOR_ENGINE.startswith("suri"): # config/YAML should already be built on the controller suri_yaml_fh = open(IDS_CONFIG_FILE, "a") suri_yaml_fh.write("\n") # set default-rule-path; this is stripped out when the controller built # the job with the expectation that it be added here. - print_debug("adding default-rule-path to yaml:\n%s" % '\n'.join(IDS_RULES_FILES)) + print_debug( + "adding default-rule-path to yaml:\n%s" % "\n".join(IDS_RULES_FILES) + ) suri_yaml_fh.write("default-rule-path: %s\n" % JOB_DIRECTORY) suri_yaml_fh.close() # reading multiple pcaps added in Suricata 4.1 - if len(PCAP_FILES) > 1 and LooseVersion("4.1") > LooseVersion(SENSOR_ENGINE_VERSION_ORIG): - print_error("Multiple pcap files were submitted to the Dalton Agent for a Suricata job.\n\nSuricata can only read a single pcap file so multiple pcaps submitted to the Dalton Controller should have been combined by the Controller when packaging the job.\n\nIf you see this, something went wrong on the Controller or you are doing something untoward.") - - if SENSOR_ENGINE.startswith('snort'): + if len(PCAP_FILES) > 1 and LooseVersion("4.1") > LooseVersion( + SENSOR_ENGINE_VERSION_ORIG + ): + print_error( + "Multiple pcap files were submitted to the Dalton Agent for a Suricata job.\n\nSuricata can only read a single pcap file so multiple pcaps submitted to the Dalton Controller should have been combined by the Controller when packaging the job.\n\nIf you see this, something went wrong on the Controller or you are doing something untoward." + ) + + if SENSOR_ENGINE.startswith("snort"): # this section applies only to Snort sensors # Snort uses DAQ dump and pcap read mode run_snort() @@ -1481,7 +1818,7 @@ def submit_job(job_id, job_directory): # process snort alerts process_snort_alerts() - elif SENSOR_ENGINE.startswith('suri'): + elif SENSOR_ENGINE.startswith("suri"): # this section for Suricata agents if getFastPattern: generate_fast_pattern() @@ -1500,7 +1837,7 @@ def submit_job(job_id, job_directory): process_suri_alerts() process_eve_log() - elif SENSOR_ENGINE.startswith('zeek'): + elif SENSOR_ENGINE.startswith("zeek"): # this section applies only to Zeek sensors run_zeek(JOB_ZEEK_JSON) @@ -1511,29 +1848,29 @@ def submit_job(job_id, job_directory): # other logs to return from the job; sensor specific other_logs = {} - if SENSOR_ENGINE.startswith('suri'): + if SENSOR_ENGINE.startswith("suri"): # always return Engine and Packet Stats for Suri - other_logs['Engine Stats'] = 'dalton-stats.log' - other_logs['Packet Stats'] = 'dalton-packet_stats.log' + other_logs["Engine Stats"] = "dalton-stats.log" + other_logs["Packet Stats"] = "dalton-packet_stats.log" if getOtherLogs: - other_logs['Alert Debug'] = 'dalton-alert_debug.log' - other_logs['HTTP Log'] = 'dalton-http.log' - other_logs['TLS Log'] = 'dalton-tls.log' - other_logs['DNS Log'] = 'dalton-dns.log' + other_logs["Alert Debug"] = "dalton-alert_debug.log" + other_logs["HTTP Log"] = "dalton-http.log" + other_logs["TLS Log"] = "dalton-tls.log" + other_logs["DNS Log"] = "dalton-dns.log" if getFastPattern: - other_logs['Fast Pattern'] = 'rules_fast_pattern.txt' + other_logs["Fast Pattern"] = "rules_fast_pattern.txt" if trackPerformance: - other_logs['Keyword Perf'] = 'dalton-keyword_perf.log' + other_logs["Keyword Perf"] = "dalton-keyword_perf.log" if getBufferDumps: - other_logs['HTTP Buffers'] = 'dalton-http-buffers.log' - other_logs['DNS Buffers'] = 'dalton-dns-buffers.log' - other_logs['TLS Buffers'] = 'dalton-tls-buffers.log' + other_logs["HTTP Buffers"] = "dalton-http-buffers.log" + other_logs["DNS Buffers"] = "dalton-dns-buffers.log" + other_logs["TLS Buffers"] = "dalton-tls-buffers.log" # elif ... can add processing of logs from other engines here - elif SENSOR_ENGINE.startswith('snort'): + elif SENSOR_ENGINE.startswith("snort"): if getBufferDumps: - other_logs['Buffer Dump'] = 'dalton-buffers.log' - elif SENSOR_ENGINE.startswith('zeek'): - other_logs = zeek_other_logs + other_logs["Buffer Dump"] = "dalton-buffers.log" + elif SENSOR_ENGINE.startswith("zeek"): + other_logs = zeek_other_logs if len(other_logs) > 0: process_other_logs(other_logs) @@ -1541,7 +1878,9 @@ def submit_job(job_id, job_directory): if getAlertDetailed: process_unified2_logs() else: - print_debug("Not processing unified2 logs (either the sensor technology does not generate these or the option was not selected).") + print_debug( + "Not processing unified2 logs (either the sensor technology does not generate these or the option was not selected)." + ) # process performance data if trackPerformance: @@ -1551,7 +1890,7 @@ def submit_job(job_id, job_directory): # Populate JOB_IDS_LOG accordingly for Suricata socket control if USE_SURICATA_SOCKET_CONTROL: - with open(JOB_IDS_LOG, 'w') as fhout: + with open(JOB_IDS_LOG, "w") as fhout: if (not SCONTROL.suricata_is_running) and SCONTROL.log_offset == 0: # this means suri Errored at startup; pass here and just # the whole output file will be included later, otherwise @@ -1560,7 +1899,7 @@ def submit_job(job_id, job_directory): else: # include initial suricata startup output fhout.write(f"{SCONTROL.suri_startup_log}\n-----\n\n") - with open(suricata_logging_outputs_file, 'r') as fh: + with open(suricata_logging_outputs_file, "r") as fh: # include relevant part of suricata log from this job fh.seek(SCONTROL.log_offset, 0) fhout.write(fh.read()) @@ -1575,6 +1914,7 @@ def submit_job(job_id, job_directory): # check the pcaps to make sure incomplete, truncated, etc. pcaps weren't submitted. check_pcaps() + ################################################# # init class to use for suricata socket control @@ -1590,19 +1930,28 @@ def submit_job(job_id, job_directory): USE_SURICATA_SOCKET_CONTROL = USE_SURICATA_SOCKET_CONTROL_DEFAULT try: job = request_job() - if (job != None): + if job is not None: start_time = int(time.time()) - JOB_ID = job['id'] + JOB_ID = job["id"] logger.info("Job %s accepted by %s" % (JOB_ID, SENSOR_UID)) send_update("Job %s Accepted by %s" % (JOB_ID, SENSOR_UID), JOB_ID) zf_path = request_zip(JOB_ID) - logger.debug("Downloaded zip for %s successfully. Extracting file %s" % (JOB_ID, zf_path)) - send_update("Downloaded zip for %s successfully; extracting..." % JOB_ID, JOB_ID) + logger.debug( + "Downloaded zip for %s successfully. Extracting file %s" + % (JOB_ID, zf_path) + ) + send_update( + "Downloaded zip for %s successfully; extracting..." % JOB_ID, JOB_ID + ) # JOB_DEBUG_LOG not defined yet so can't call print_debug() here - #print_debug("Extracting zip file for job id %s" % JOB_ID) - JOB_DIRECTORY = "%s/%s_%s" % (STORAGE_PATH, JOB_ID, datetime.datetime.now().strftime("%b-%d-%Y_%H-%M-%S")) + # print_debug("Extracting zip file for job id %s" % JOB_ID) + JOB_DIRECTORY = "%s/%s_%s" % ( + STORAGE_PATH, + JOB_ID, + datetime.datetime.now().strftime("%b-%d-%Y_%H-%M-%S"), + ) os.makedirs(os.path.join(JOB_DIRECTORY, PCAP_DIR)) - zf = zipfile.ZipFile(zf_path, 'r') + zf = zipfile.ZipFile(zf_path, "r") filenames = zf.namelist() for filename in filenames: logger.debug("extracting file, %s" % filename) @@ -1623,19 +1972,28 @@ def submit_job(job_id, job_directory): except Exception as e: # not a DaltonError, perhaps a code bug? Try to write to JOB_ERROR_LOG if JOB_ERROR_LOG: - msg = "Dalton Agent error in sensor \'%s\' while processing job %s. Exception in submit_job(). Please re-submit or contact admin with this message (see \'About\' page for contact info). Error message:\n\n%s" % (SENSOR_UID, JOB_ID, e) + msg = ( + "Dalton Agent error in sensor '%s' while processing job %s. Exception in submit_job(). Please re-submit or contact admin with this message (see 'About' page for contact info). Error message:\n\n%s" + % (SENSOR_UID, JOB_ID, e) + ) fh = open(JOB_ERROR_LOG, "a") fh.write("%s\n" % msg) fh.close() print_msg("ERROR!") print_debug("ERROR:\n%s" % msg) else: - error_post_results("Dalton Agent Error in sensor \'%s\' while processing job %s. Error:\n%s" % (SENSOR_UID, JOB_ID, e)) + error_post_results( + "Dalton Agent Error in sensor '%s' while processing job %s. Error:\n%s" + % (SENSOR_UID, JOB_ID, e) + ) logger.error("Non DaltonError Exception caught:\n%s" % e) logger.debug("%s" % traceback.format_exc()) - TOTAL_PROCESSING_TIME = int(int(time.time())-start_time) - print_debug("Total Processing Time (includes job download time): %d seconds" % TOTAL_PROCESSING_TIME) + TOTAL_PROCESSING_TIME = int(int(time.time()) - start_time) + print_debug( + "Total Processing Time (includes job download time): %d seconds" + % TOTAL_PROCESSING_TIME + ) # send results back to server logger.info("Job %s done processing, sending back results" % JOB_ID) @@ -1658,24 +2016,38 @@ def submit_job(job_id, job_directory): SCONTROL.connect() SCONTROL.shutdown() SCONTROL.close() - except: + except Exception: pass sys.exit(0) except DaltonError as e: logger.debug("DaltonError caught (in while True loop):\n%s" % e) except Exception as e: - logger.debug("General Dalton Agent exception caught. Error:\n%s\n%s" % (e, traceback.format_exc())) + logger.debug( + "General Dalton Agent exception caught. Error:\n%s\n%s" + % (e, traceback.format_exc()) + ) if JOB_ID: # unexpected error happened on agent when trying to process a job but there may not be job data so compile an empty response with the exception error message and try to send it - logger.warn("Possible communication error processing jobid %s. Attempting to send error message to controller." % JOB_ID) + logger.warn( + "Possible communication error processing jobid %s. Attempting to send error message to controller." + % JOB_ID + ) try: error_post_results(e) - logger.info("Successfully sent error message to controller for jobid %s" % JOB_ID) + logger.info( + "Successfully sent error message to controller for jobid %s" + % JOB_ID + ) except Exception as e: - logger.error("Could not communicate with controller to send error info for jobid %s; is the Dalton Controller accepting network communications? Error:\n%s" % (JOB_ID, e)) + logger.error( + "Could not communicate with controller to send error info for jobid %s; is the Dalton Controller accepting network communications? Error:\n%s" + % (JOB_ID, e) + ) time.sleep(ERROR_SLEEP_TIME) else: - logger.error("Agent Error -- Is the Dalton Controller accepting network communications?") + logger.error( + "Agent Error -- Is the Dalton Controller accepting network communications?" + ) sys.stdout.flush() time.sleep(ERROR_SLEEP_TIME) # finally: diff --git a/docker-compose.yml b/docker-compose.yml index 6429ed4..efd3397 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +--- services: controller: @@ -11,6 +11,10 @@ services: - no_proxy=${no_proxy} image: dalton:latest container_name: dalton_controller + # Uncomment the next line to run the app in debug mode + # command: flask --app app run --port=8080 --host=0.0.0.0 --debug + depends_on: + - redis environment: - CONTROLLER_DEBUG=${CONTROLLER_DEBUG} volumes: @@ -28,6 +32,8 @@ services: - DALTON_EXTERNAL_PORT_SSL=${DALTON_EXTERNAL_PORT_SSL} image: nginx-dalton:latest container_name: dalton_web + depends_on: + - controller ports: # external HTTP listen port # to change, set DALTON_EXTERNAL_PORT in the '.env' file (NOT here) @@ -41,7 +47,7 @@ services: restart: always redis: - image: redis:3.2.12 + image: redis:7.4.1 container_name: dalton_redis restart: always diff --git a/nginx-conf/nginx.conf b/nginx-conf/nginx.conf index 850191e..af476fa 100644 --- a/nginx-conf/nginx.conf +++ b/nginx-conf/nginx.conf @@ -2,7 +2,6 @@ user www-data; worker_processes 1; error_log /var/log/nginx/error.log; pid /run/nginx.pid; -daemon off; events { worker_connections 1024; diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..648ec80 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,91 @@ +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "dalton" +description = "Run pcaps against an IDS" +dynamic = ["version"] +requires-python = ">=3.10" +dependencies = [ + "Jinja2==3.1.4", + "Flask==3.1.0", + "redis==5.2.0", + # there is newer ruamel available + "ruamel.yaml<0.18.0", + "idstools==0.6.5", + "flowsynth>=1.4.1", + "Werkzeug==3.1.3", + "itsdangerous==2.2.0", +] +authors = [ + { name = "David Wharton and others" } +] +readme = "README.rst" +classifiers = [ + "Private :: Do Not Upload", + "License :: OSI Approved :: Apache Software License", + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", +] + +[project.urls] +Homepage = "https://github.com/secureworks/dalton" + +[project.optional-dependencies] +testing = [ + "pytest", +] +devtools = [ + "bump-my-version", + "coverage", + "ruff", +] + +[tool.pytest.ini_options] +pythonpath = [ + "." +] + +[tool.setuptools.packages.find] +# Include only the "app" directory. +include = ["app"] + +[tool.ruff] +include = [ + "pyproject.toml", + "app/**/*.py", + "api/**/*.py", + "dalton-agent/**/*.py", + "tests/**/*.py" +] +exclude = ["app/static/**/*.py"] + +[tool.ruff.lint] +# see https://docs.astral.sh/ruff/rules/#legend +# E = errors, F = pyflakes, I = isort, B = bugbears +select = ["E", "F", "I", "B"] +ignore = ["E501"] + +[tool.ruff.lint.per-file-ignores] +# Defer these fixes to dalton-agent until we have some unit tests +"dalton-agent/dalton-agent.py" = ["B"] + +[tool.bumpversion] +current_version = "3.4.2" + +commit = true +allow_dirty = false +message = "Bump version: {current_version} → {new_version}" +commit_args = "--no-verify" + +tag = true +sign_tags = false +tag_name = "v{new_version}" +tag_message = "Bump version: {current_version} → {new_version}" + +[[tool.bumpversion.files]] +filename = "app/__init__.py" diff --git a/requirements.txt b/requirements.txt index 750b6ad..d44989d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,3 @@ -Jinja2==3.0.3 -Flask==1.1.1 -Flask-Assets==0.12 -Flask-Script==2.0.6 -Flask-Caching==1.8.0 -Flask-Compress==1.4.0 -redis==2.10.6 -ruamel.yaml==0.16.12 -idstools==0.6.4 -flowsynth>=1.4.1 -werkzeug==0.16.1 -itsdangerous==2.0.1 +# requirements are now in pyproject.toml + +-e . diff --git a/run.py b/run.py index f6d2001..14d2f79 100644 --- a/run.py +++ b/run.py @@ -1,36 +1,5 @@ -from flask import Flask -from flask_caching import Cache -from flask_compress import Compress -from app.dalton import dalton_blueprint -from app.flowsynth import flowsynth_blueprint -import logging - -# create -daltonfs = Flask(__name__, static_folder='app/static') - -# register modules -# -# dalton -daltonfs.register_blueprint(dalton_blueprint) - -# flowsynth -daltonfs.register_blueprint(flowsynth_blueprint, url_prefix='/flowsynth') - -daltonfs.debug = True - -# Apparently the werkzeug default logger logs every HTTP request -# which bubbles up to the root logger and gets output to the -# console which ends up in the docker logs. Since each agent -# checks in every second (by default), this can be voluminous -# and is superfluous for my current needs. -try: - logging.getLogger("werkzeug").setLevel(logging.ERROR) -except Exception as e: - pass - -compress = Compress() -cache = Cache(daltonfs, config={"CACHE_TYPE": "simple"}) -compress.init_app(daltonfs) +import app if __name__ == "__main__": - daltonfs.run(host='0.0.0.0', port=8080) + app = app.create_app(None) + app.run(host='0.0.0.0', port=8080) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..df80e19 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,14 @@ +import pytest + +from app import create_app + + +@pytest.fixture(scope="class") +def client(request): + the_app = create_app( + { + "TESTING": True, + } + ) + client = the_app.test_client() + request.cls.client = client diff --git a/tests/files/pcap_98765.pcap b/tests/files/pcap_98765.pcap new file mode 100644 index 0000000..37d4e6c --- /dev/null +++ b/tests/files/pcap_98765.pcap @@ -0,0 +1 @@ +hi there diff --git a/tests/test_dalton.py b/tests/test_dalton.py new file mode 100644 index 0000000..23e5205 --- /dev/null +++ b/tests/test_dalton.py @@ -0,0 +1,115 @@ +"""Unit tests for dalton module.""" + +import json +import random +import unittest +from unittest import mock + +import pytest + +from app.dalton import ( + REDIS_EXPIRE, + STAT_CODE_DONE, + create_hash, + prefix_strip, +) + + +@pytest.mark.usefixtures("client") +class TestDalton(unittest.TestCase): + """Test dalton functionality.""" + + def test_dalton_main(self): + """Ensure the index page loads.""" + res = self.client.get("/dalton/") + self.assertEqual(res.status_code, 200, "It should render") + self.assertIn(b"Dalton", res.data) + + def test_dalton_about(self): + response = self.client.get("/dalton/about") + self.assertIn(b"About Dalton", response.data) + + def test_prefix_strip(self): + """Test the prefix_strip function.""" + prefixes_to_strip = ["abcd", "defg"] + self.assertEqual(prefix_strip("abcd1234", prefixes_to_strip), "1234") + self.assertEqual(prefix_strip("defg1234", prefixes_to_strip), "1234") + self.assertEqual(prefix_strip("12345678", prefixes_to_strip), "12345678") + # Also test with default prefix to strip - "rust_" + self.assertEqual(prefix_strip("rust_1234"), "1234") + self.assertEqual(prefix_strip("12345678"), "12345678") + + def test_create_hash(self): + """Test create_hash with bytes.""" + uid = "73756293-827f-4829-9515-f5a77c36ad0c" + ip = "127.0.0.22" + expected = "430637bdaa1e8dd5fc032dc4deb14791" + digest = create_hash([uid, ip]) + self.assertEqual(expected, digest) + + @mock.patch("app.dalton.get_redis") + def test_sensor_page(self, get_redis): + """Check if the sensor page can load.""" + redis = mock.Mock() + # Return False when asked if "sensors" exists. + redis.exists.return_value = False + get_redis.return_value = redis + res = self.client.get("/dalton/sensor") + self.assertEqual(res.status_code, 200, "A page with sensors") + + @mock.patch("app.dalton.get_redis") + def test_request_job(self, get_redis): + """Try to request a job. It should not crash.""" + redis = mock.Mock() + redis.lpop.return_value = None + get_redis.return_value = redis + versions = ["2.0.9", "3.0", "9.9.9"] + for version in versions: + url = f"/dalton/sensor_api/request_job?SENSOR_ENGINE_VERSION={version}" + res = self.client.get(url) + self.assertEqual(res.status_code, 200, res.data.decode()) + + def test_dalton_get_job(self): + res = self.client.get("/dalton/sensor_api/get_job/12345") + self.assertEqual(res.status_code, 200) + self.assertIn( + "Job 12345 does not exist on disk. It is either invalid " + "or has been deleted.", + res.data.decode(), + ) + + @mock.patch("app.dalton.get_redis") + def test_job_results(self, get_redis): + """Try to provide job results. It should not crash.""" + mock_redis = mock.Mock() + mock_redis.get.return_value = 1 + get_redis.return_value = mock_redis + job_id = 34 + url = f"/dalton/sensor_api/results/{job_id}" + job_results = {"status": "1"} + data = dict(json_data=json.dumps(job_results)) + res = self.client.post(url, data=data) + self.assertEqual(res.status_code, 200) + + @mock.patch("app.dalton.create_hash") + @mock.patch("app.dalton.get_redis") + def test_post_job_results(self, get_redis, create_hash): + """Exercise `post_job_results`.""" + redis = mock.Mock() + redis.get.return_value = 1 + get_redis.return_value = redis + the_hash = str(random.randint(1, 999999999)) + create_hash.return_value = the_hash + job_id = random.randint(1, 999999999) + job_status = str(random.randint(1, 999999999)) + job_status_dict = {"status": job_status} + data = { + "json_data": json.dumps(job_status_dict), + } + res = self.client.post(f"/dalton/sensor_api/results/{job_id}", data=data) + self.assertEqual(res.status_code, 200) + self.assertEqual("OK", res.data.decode()) + redis.set.assert_any_call(f"{job_id}-status", f"Final Job Status: {job_status}") + redis.set.assert_any_call(f"{the_hash}-current_job", "") + redis.expire.assert_any_call(f"{the_hash}-current_job", REDIS_EXPIRE) + redis.set.assert_any_call(f"{job_id}-statcode", STAT_CODE_DONE) diff --git a/tests/test_factory.py b/tests/test_factory.py new file mode 100644 index 0000000..cfc0bb5 --- /dev/null +++ b/tests/test_factory.py @@ -0,0 +1,12 @@ +import unittest + +from flask import Flask + +from app import create_app + + +class TestFactory(unittest.TestCase): + def test_testing(self): + result = create_app({"TESTING": True}) + self.assertIsInstance(result, Flask) + self.assertTrue(result.testing) diff --git a/tests/test_flowsynth.py b/tests/test_flowsynth.py new file mode 100644 index 0000000..19a37e2 --- /dev/null +++ b/tests/test_flowsynth.py @@ -0,0 +1,139 @@ +import os +import random +import shutil +import unittest +import uuid +from io import BytesIO +from unittest import mock +from urllib.parse import urlencode + +import pytest + +from app.flowsynth import ( + check_pcap_path, + get_pcap_file_path, + get_pcap_path, + unicode_safe, +) + +KNOWN_PCAP_ID = "98765" +KNOWN_PCAP_CONTENTS = b"hi there" + + +@pytest.mark.usefixtures("client") +class TestFlowsynth(unittest.TestCase): + def setUp(self): + self.pcap_base = "/tmp/pcaps" + check_pcap_path(self.pcap_base) + + def tearDown(self): + shutil.rmtree(self.pcap_base) + + def test_flowsynth_home(self): + response = self.client.get("/flowsynth/") + self.assertIn(b"Build Packet Capture", response.data) + self.assertEqual(200, response.status_code) + + def test_flowsynth_about(self): + response = self.client.get("/flowsynth/about") + self.assertEqual(200, response.status_code) + self.assertIn(b"About Flowsynth", response.data) + + def test_flowsynth_compile(self): + response = self.client.get("/flowsynth/compile") + self.assertEqual(200, response.status_code) + self.assertIn(b"Compile", response.data) + + def test_get_pcap_file_path(self): + basename = "snap" + path = get_pcap_file_path(basename) + + expected = get_pcap_path() + "/" + basename + ".pcap" + self.assertEqual(expected, path) + + @classmethod + def read_pcap_file(cls, filename=None, random=False, pcap_id=KNOWN_PCAP_ID): + """Read a PCAP file from test data.""" + filename = filename or f"pcap_{pcap_id}.pcap" + path = os.path.join( + os.path.abspath(os.path.dirname(__file__)), "files", filename + ) + with open(path, "rb") as content_file: + content = content_file.read() + if random: + content = content + str(uuid.uuid4()).encode() + return BytesIO(content) + + def test_unicode_safe(self): + testdata = "abcd1234" + self.assertEqual(testdata, unicode_safe(testdata)) + testdata = "きたない\n" + self.assertEqual("\n", unicode_safe(testdata)) + + @mock.patch("app.flowsynth.get_pcap_file_path") + def test_retrieve_pcap(self, mock_pcap_path): + """Confirm '/flowsynth/pcap/get_pcap' endpoint is working.""" + # Copy the KNOWN_PCAP file into PCAP_PATH/.pcap. + pcap_id = random.randint(100000, 999999) + path = get_pcap_file_path(pcap_id, path=self.pcap_base) + mock_pcap_path.side_effect = [path] + with open(path, "wb") as fd: + pcap_data = self.read_pcap_file(None, random=True) + fd.write(pcap_data.read()) + url = f"/flowsynth/pcap/get_pcap/{pcap_id}" + response = self.client.get(url) + self.assertEqual(200, response.status_code) + self.assertEqual("application/vnd.tcpdump.pcap", response.content_type) + self.assertEqual( + f"attachment;filename={pcap_id}.pcap", + response.headers["Content-Disposition"], + ) + self.assertIn(KNOWN_PCAP_CONTENTS, response.data) + + def test_generate(self): + """Ensure we can call the /generate endpoint.""" + # NOTE these are L's and not ones + input_data_string = "data would go here" + expected_output = r"data\x20would\x20go\x20here" + post_args = { + "l3_src_ip": "$HOME_NET", + "l3_dst_ip": "$HOME_NET", + "l3_protocol": "TCP", + "l3_flow_established": "yes", + "l4_src_port": "any", + "l4_dst_port": "any", + "payload_format": "http", # 'http', 'raw' or 'cert' + "payload_http_request_contentlength": "on", + "request_header": "", + "request_body": input_data_string, + "generate_method": "build", + } + url = "/flowsynth/generate" + response = self.client.post( + url, + data=urlencode(post_args), + content_type="application/x-www-form-urlencoded", + ) + self.assertEqual(200, response.status_code) + self.assertIn("flow default tcp 192", response.data.decode()) + self.assertIn(expected_output, response.data.decode()) + + @mock.patch("app.flowsynth.check_pcap_path") + @mock.patch("app.flowsynth.get_pcap_file_path") + def test_pcap_compile_fs(self, mock_pcap_path, mock_check_pcap_path): + """Ensure we can call the pcap/compile_fs endpoint.""" + post_args = { + "code": "flow default tcp 192.168.51.44:45348 > 172.16.146.20:45694 (tcp.initialize;);" + 'default > (content:""; content:"\x0d\x0aContent-Length: 12"; content:"\x0d\x0a\x0d\x0a";' + 'content:"blah-blah"; );', + } + path = get_pcap_file_path(KNOWN_PCAP_ID, path=self.pcap_base) + mock_pcap_path.side_effect = [path] + response = self.client.post( + "/flowsynth/pcap/compile_fs", + data=urlencode(post_args), + content_type="application/x-www-form-urlencoded", + ) + self.assertEqual(200, response.status_code, "Flowsynth command worked") + self.assertIn(b"Success", response.data) + self.assertIn(b"Click Here to Download PCAP", response.data)