Skip to content
This repository has been archived by the owner on Mar 24, 2023. It is now read-only.

Commit

Permalink
Merge branch 'dkim-and-dmarc' of https://github.com/minvws/nl-kat-boe…
Browse files Browse the repository at this point in the history
…fjes into dkim-and-dmarc
  • Loading branch information
noamblitz committed Jan 31, 2023
2 parents b345252 + eb37a2b commit 2470c29
Show file tree
Hide file tree
Showing 23 changed files with 592 additions and 66 deletions.
17 changes: 0 additions & 17 deletions .ci/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,29 +9,12 @@ services:
- ENVIRONMENT=dev
command: sh -c 'python -m pytest -vv boefjes/katalogus/tests/integration'
depends_on:
- ci_katalogus
- ci_katalogus-db
env_file:
- .ci/.env.test
volumes:
- .:/app/boefjes

ci_katalogus:
build:
context: ..
dockerfile: nl-kat-boefjes/Dockerfile.dev
args:
- ENVIRONMENT=dev
command: ["python", "-m", "uvicorn", "--host", "0.0.0.0", "boefjes.katalogus.api:app"]
depends_on:
- ci_katalogus-db
env_file:
- .ci/.env.test
environment:
- ENABLE_DB=false
volumes:
- .:/app/boefjes

ci_katalogus-db:
image: postgres:12.8
env_file:
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ sql: ## Generate raw sql for the migrations
docker-compose run katalogus python -m alembic --config /app/boefjes/boefjes/alembic.ini upgrade $(rev1):$(rev2) --sql

check:
pre-commit run --all-files --show-diff-on-failure --color always
pre-commit run --all-files --color always

##
##|------------------------------------------------------------------------|
Expand Down
16 changes: 15 additions & 1 deletion boefjes/katalogus/dependencies/plugins.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,20 @@ def delete_setting_by_key(self, key: str, organisation_id: str, plugin_id: str):
plugin = self.by_plugin_id(plugin_id, organisation_id)
self.update_by_id(plugin.repository_id, plugin_id, organisation_id, False)

def clone_settings_to_organisation(self, from_organisation: str, to_organisation: str):
# One requirement is that we also do not keep previously enabled boefjes enabled of they are not copied.
for repository_id, plugins in self.plugin_enabled_store.get_all_enabled(to_organisation).items():
for plugin_id in plugins:
self.update_by_id(repository_id, plugin_id, to_organisation, enabled=False)

for plugin in self.get_all(from_organisation):
for key, value in self.get_all_settings(from_organisation, plugin.id).items():
self.create_setting(key, value, to_organisation, plugin.id)

for repository_id, plugins in self.plugin_enabled_store.get_all_enabled(from_organisation).items():
for plugin_id in plugins:
self.update_by_id(repository_id, plugin_id, to_organisation, enabled=True)

def update_setting_by_key(self, key: str, value: str, organisation_id: str, plugin_id: str):
return self.settings_storage.update_by_key(key, value, organisation_id, plugin_id)

Expand Down Expand Up @@ -204,7 +218,7 @@ def get_plugin_service(organisation_id: str) -> Iterator[PluginService]:
yield PluginService(
store,
repository_storage,
SettingsStorageMemory(organisation_id),
SettingsStorageMemory(),
client,
local_repo,
)
Expand Down
10 changes: 10 additions & 0 deletions boefjes/katalogus/routers/plugins.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,3 +128,13 @@ def get_plugin_description(
raise HTTPException(HTTP_404_NOT_FOUND, "Unknown repository")
except HTTPError as ex:
raise HTTPException(ex.response.status_code)


@router.post("/settings/clone/{to_organisation_id}")
def clone_organisation_settings(
organisation_id: str,
to_organisation_id: str,
storage: PluginService = Depends(get_plugin_service),
):
with storage as store:
store.clone_settings_to_organisation(organisation_id, to_organisation_id)
18 changes: 0 additions & 18 deletions boefjes/katalogus/storage/diskcache.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,21 +55,3 @@ def create(self, repository: Repository) -> None:

def delete_by_id(self, id_: str) -> None:
del self._repositories[id_]


class PluginStatesStorageDisk(PluginEnabledStorage):
def __init__(self, directory: Union[str, Path]):
self._cache = Cache(Path(directory).as_posix())
if "plugins_states" not in self._cache:
self._cache["plugins_states"] = {}

self._plugins_states = self._cache["plugins_states"]

def get_by_id(self, plugin_id: str, repository_id: str, organisation_id: str) -> bool:
return self._plugins_states[plugin_id]

def create(self, plugin_id: str, repository_id: str, enabled: bool, organisation_id: str) -> None:
self._plugins_states[plugin_id] = enabled

def update_or_create_by_id(self, plugin_id: str, repository_id: str, enabled: bool, organisation_id: str) -> None:
self._plugins_states[plugin_id] = enabled
7 changes: 5 additions & 2 deletions boefjes/katalogus/storage/interfaces.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from abc import ABC
from typing import Dict, Type
from typing import Dict, Type, List

from boefjes.katalogus.models import Organisation, Repository, Plugin
from boefjes.katalogus.models import Organisation, Repository


class StorageError(Exception):
Expand Down Expand Up @@ -121,6 +121,9 @@ def __exit__(self, exc_type: Type[Exception], exc_value: str, exc_traceback: str
def get_by_id(self, plugin_id: str, repository_id: str, organisation_id: str) -> bool:
raise NotImplementedError

def get_all_enabled(self, organisation_id: str) -> Dict[str, List[str]]:
raise NotImplementedError

def create(self, plugin_id: str, repository_id: str, enabled: bool, organisation_id: str) -> None:
raise NotImplementedError

Expand Down
39 changes: 26 additions & 13 deletions boefjes/katalogus/storage/memory.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from typing import Dict
from typing import Dict, List

from boefjes.katalogus.local_repository import LocalPluginRepository
from boefjes.katalogus.models import Organisation, Repository
from boefjes.katalogus.storage.interfaces import (
OrganisationStorage,
Expand Down Expand Up @@ -58,22 +59,25 @@ def delete_by_id(self, id_: str) -> None:


class SettingsStorageMemory(SettingsStorage):
def __init__(
self,
organisation: str,
defaults: Dict[str, str] = None,
):
defaults = defaults or {}
self._data = {organisation: defaults}
self._organisation = organisation
def __init__(self):
self._data = {}

def get_by_key(self, key: str, organisation_id: str, plugin_id: str) -> str:
return self._data[organisation_id][f"{plugin_id}.{key}"]

def get_all(self, organisation_id: str, plugin_id: str) -> Dict[str, str]:
return {k.split(".", maxsplit=1)[1]: v for k, v in self._data[organisation_id].items() if plugin_id in k}
if organisation_id not in self._data:
return {}

org_data = self._data[organisation_id].items()
org_data_for_plugin = {k: v for k, v in org_data if plugin_id == k.split(".", maxsplit=1)[0]}

return {k.split(".", maxsplit=1)[1]: v for k, v in org_data_for_plugin.items()}

def create(self, key: str, value: str, organisation_id: str, plugin_id: str) -> None:
if organisation_id not in self._data:
self._data[organisation_id] = {}

self._data[organisation_id][f"{plugin_id}.{key}"] = str(value)

def update_by_key(self, key: str, value: str, organisation_id: str, plugin_id: str) -> None:
Expand All @@ -93,10 +97,19 @@ def __init__(
self._organisation = organisation

def get_by_id(self, plugin_id: str, repository_id: str, organisation_id: str) -> bool:
return self._data[plugin_id]
return self._data[f"{organisation_id}.{plugin_id}"]

def get_all_enabled(self, organisation_id: str) -> Dict[str, List[str]]:
return {
LocalPluginRepository.RESERVED_ID: [
key.split(".", maxsplit=1)[1]
for key, value in self._data.items()
if value and key.split(".", maxsplit=1)[0] == organisation_id
]
}

def create(self, plugin_id: str, repository_id: str, enabled: bool, organisation_id: str) -> None:
self._data[plugin_id] = enabled
self._data[f"{organisation_id}.{plugin_id}"] = enabled

def update_or_create_by_id(self, plugin_id: str, repository_id: str, enabled: bool, organisation_id: str) -> None:
self._data[plugin_id] = enabled
self._data[f"{organisation_id}.{plugin_id}"] = enabled
100 changes: 92 additions & 8 deletions boefjes/katalogus/tests/integration/test_api.py
Original file line number Diff line number Diff line change
@@ -1,31 +1,115 @@
import os
import time
from unittest import TestCase, skipIf

import requests
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import sessionmaker
from starlette.testclient import TestClient

from boefjes.config import settings
from boefjes.katalogus.models import Organisation
from boefjes.katalogus.api import app
from boefjes.katalogus.dependencies.encryption import IdentityMiddleware
from boefjes.katalogus.models import Organisation, Repository
from boefjes.sql.db import get_engine, SQL_BASE
from boefjes.sql.organisation_storage import SQLOrganisationStorage
from boefjes.sql.plugin_enabled_storage import SQLPluginEnabledStorage
from boefjes.sql.repository_storage import SQLRepositoryStorage
from boefjes.sql.setting_storage import SQLSettingsStorage


@skipIf(os.environ.get("CI") != "1", "Needs a CI database.")
class TestAPI(TestCase):
"""This tests the API when settings.enable_db=False."""

def setUp(self) -> None:
self.engine = get_engine()

# Some retries to handle db startup time in tests
for i in range(3):
try:
SQL_BASE.metadata.create_all(self.engine)
break
except OperationalError as e:
if i == 2:
raise e

time.sleep(1)

session = sessionmaker(bind=self.engine)()
self.organisation_storage = SQLOrganisationStorage(session, settings)
self.repository_storage = SQLRepositoryStorage(session, settings)
self.settings_storage = SQLSettingsStorage(session, IdentityMiddleware())
self.plugin_state_storage = SQLPluginEnabledStorage(session, settings)

with self.repository_storage as store:
store.create(
Repository(
id="LOCAL",
name="Test",
base_url="http://test.url",
)
)

self.org = Organisation(id="test", name="Test Organisation")

response = requests.post(f"{settings.katalogus_api}/v1/organisations/", self.org.json())
self.client = TestClient(app)
response = self.client.post("/v1/organisations/", self.org.json())
self.assertEqual(response.status_code, 201)

def tearDown(self) -> None:
response = requests.delete(f"{settings.katalogus_api}/v1/organisations/{self.org.id}")
self.assertEqual(response.status_code, 200)
session = sessionmaker(bind=get_engine())()

for table in SQL_BASE.metadata.tables.keys():
session.execute(f"DELETE FROM {table} CASCADE")

session.commit()
session.close()

def test_plugin_api(self):
response = requests.get(f"{settings.katalogus_api}/v1/organisations/{self.org.id}/plugins/dns-records")
response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/dns-records")

self.assertEqual(response.status_code, 200)
data = response.json()

self.assertEqual("dns-records", data["id"])
self.assertEqual("LOCAL", data["repository_id"])

def test_clone_settings(self):
plug = "dns-records"

# Set a setting on the first organisation and enable dns-records
self.client.post(f"/v1/organisations/{self.org.id}/{plug}/settings/test_key", json={"value": "test value"})
self.client.post(f"/v1/organisations/{self.org.id}/{plug}/settings/test_key_2", json={"value": "test value 2"})
self.client.patch(f"/v1/organisations/{self.org.id}/repositories/LOCAL/plugins/{plug}", json={"enabled": True})

assert self.client.get(f"/v1/organisations/{self.org.id}/{plug}/settings").json() == {
"test_key": "test value",
"test_key_2": "test value 2",
}
assert self.client.get(f"/v1/organisations/{self.org.id}/plugins/{plug}").json()["enabled"] is True

# Add the second organisation
new_org_id = "org2"
org2 = Organisation(id=new_org_id, name="Second test Organisation")
self.client.post("/v1/organisations/", org2.json())

# Show that the second organisation has no settings and dns-records is not enabled
assert self.client.get(f"/v1/organisations/{new_org_id}/{plug}/settings").json() == {}
assert self.client.get(f"/v1/organisations/{new_org_id}/plugins/{plug}").json()["enabled"] is False

# Enable two boefjes that should get disabled by the cloning
self.client.patch(f"/v1/organisations/{new_org_id}/repositories/LOCAL/plugins/nmap", json={"enabled": True})
assert self.client.get(f"/v1/organisations/{new_org_id}/plugins/nmap").json()["enabled"] is True

# Call the clone endpoint
self.client.post(f"/v1/organisations/{self.org.id}/settings/clone/{new_org_id}")

# Verify that all settings are copied
response = self.client.get(f"/v1/organisations/{new_org_id}/{plug}/settings")
assert response.json() == {"test_key": "test value", "test_key_2": "test value 2"}

# And that the enabled boefje from the original organisation got enabled
response = self.client.get(f"/v1/organisations/{new_org_id}/plugins/{plug}")
assert response.json()["enabled"] is True

# And the originally enabled boefje got disabled
response = self.client.get(f"/v1/organisations/{new_org_id}/plugins/nmap")
assert response.json()["enabled"] is False
52 changes: 51 additions & 1 deletion boefjes/katalogus/tests/test_plugin_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ def get_plugin_seed():


def mock_plugin_service(organisation_id: str) -> PluginService:
storage = SettingsStorageMemory("test")
storage = SettingsStorageMemory()
storage.create("DUMMY_VAR", "123", "test", "test_plugin")

repo_store = RepositoryStorageMemory(organisation_id)
Expand Down Expand Up @@ -226,3 +226,53 @@ def test_adding_integer_settings_with_faulty_value_given_constraints(self):

plugin = self.service.by_plugin_id(plugin_id, self.organisation)
self.assertFalse(plugin.enabled)

def test_clone_one_setting(self):
new_org_id = "org2"

plugin_id = "kat_test"
self.service.create_setting("api_key", "24", self.organisation, plugin_id)
assert self.service.get_setting_by_key("api_key", self.organisation, plugin_id) == "24"

self.service.update_by_id(LocalPluginRepository.RESERVED_ID, plugin_id, self.organisation, True)

self.service.update_by_id(LocalPluginRepository.RESERVED_ID, "test-boefje-1", new_org_id, True)
self.service.update_by_id(LocalPluginRepository.RESERVED_ID, "test-boefje-2", new_org_id, True)

with self.assertRaises(KeyError):
self.service.get_setting_by_key("api_key", new_org_id, plugin_id)

new_org_plugins = self.service.get_all(new_org_id)
assert len(new_org_plugins) == 8
assert len([x for x in new_org_plugins if x.enabled]) == 6 # 4 Normalizers plus two boefjes enabled above
assert plugin_id not in [x.id for x in new_org_plugins if x.enabled]
assert "test-boefje-1" in [x.id for x in new_org_plugins if x.enabled]

self.service.clone_settings_to_organisation(self.organisation, new_org_id)

assert self.service.get_setting_by_key("api_key", self.organisation, plugin_id) == "24"
assert self.service.get_setting_by_key("api_key", new_org_id, plugin_id) == "24"

new_org_plugins = self.service.get_all(new_org_id)
assert len(new_org_plugins) == 8
assert len([x for x in new_org_plugins if x.enabled]) == 5
assert plugin_id in [x.id for x in new_org_plugins if x.enabled]
assert "test-boefje-1" not in [x.id for x in new_org_plugins if x.enabled]

def test_clone_many_settings(self):
plugin_ids = ["test-boefje-1", "test-boefje-2"] * 4
all_settings = {str(x): str(x + 1) for x in range(8)}

for plugin_id, (key, value) in zip(plugin_ids, all_settings.items()):
self.service.create_setting(key, value, self.organisation, plugin_id)

self.service.clone_settings_to_organisation(self.organisation, "org2")

all_settings_for_new_org = self.service.get_all_settings("org2", "test-boefje-1")
assert len(all_settings_for_new_org) == 4
assert all_settings_for_new_org == {
"0": "1",
"2": "3",
"4": "5",
"6": "7",
}
Empty file.
Empty file.
Loading

0 comments on commit 2470c29

Please sign in to comment.