From 9d3cb2e5469b225356e80099f3aa252e96580634 Mon Sep 17 00:00:00 2001 From: Gagan Deep Date: Mon, 26 Jun 2023 20:34:16 +0530 Subject: [PATCH 01/20] [chores] Updated drf-yasg dependency to also install coreapi #341 - Added test for swagger docs Related to #341 --- setup.py | 8 +++++++- tests/test_project/tests/test_integrations.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 tests/test_project/tests/test_integrations.py diff --git a/setup.py b/setup.py index 68235957..ce144390 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,13 @@ 'rest': [ 'djangorestframework~=3.14.0', 'django-filter~=23.2', # django-filter uses CalVer - 'drf-yasg~=1.21.0', + # The coreapi package is archived and all packages + # are moving away from coreapi (e.g. DRF, django-filter, drf-yasg). + # There's already an open PR in drf-yasg + # https://github.com/axnsan12/drf-yasg/pull/857. + # TODO: Before releasing, check if newer version + # of drf-yasg is available. + 'drf-yasg[coreapi]~=1.21.0', ], 'celery': ['celery~=5.3.0'], 'selenium': ['selenium~=4.10.0'], diff --git a/tests/test_project/tests/test_integrations.py b/tests/test_project/tests/test_integrations.py new file mode 100644 index 00000000..a67dc5b8 --- /dev/null +++ b/tests/test_project/tests/test_integrations.py @@ -0,0 +1,10 @@ +from django.test import TestCase +from django.urls import reverse + +from . import AdminTestMixin + + +class TestIntegrations(AdminTestMixin, TestCase): + def test_swagger_api_docs(self): + response = self.client.get(reverse('schema-swagger-ui'), {'format': 'openapi'}) + self.assertEqual(response.status_code, 200) From 3a99c392bfab207e89bb4dba0d4de4e78bb84f15 Mon Sep 17 00:00:00 2001 From: Gagan Deep Date: Wed, 28 Jun 2023 20:56:20 +0530 Subject: [PATCH 02/20] [feature] Added fallback fields #333 Closes #333 --- README.rst | 134 ++++++++++++++- openwisp_utils/base.py | 37 ++-- openwisp_utils/fields.py | 161 ++++++++++++++++++ tests/test_project/admin.py | 14 +- .../0005_organizationradiussettings.py | 71 ++++++++ tests/test_project/models.py | 52 +++++- tests/test_project/tests/__init__.py | 6 + tests/test_project/tests/test_admin.py | 87 +++++++++- tests/test_project/tests/test_model.py | 88 +++++++++- 9 files changed, 616 insertions(+), 34 deletions(-) create mode 100644 openwisp_utils/fields.py create mode 100644 tests/test_project/migrations/0005_organizationradiussettings.py diff --git a/README.rst b/README.rst index cb943349..36c763bf 100644 --- a/README.rst +++ b/README.rst @@ -897,10 +897,138 @@ Model class inheriting ``UUIDModel`` which provides two additional fields: Which use respectively ``AutoCreatedField``, ``AutoLastModifiedField`` from ``model_utils.fields`` (self-updating fields providing the creation date-time and the last modified date-time). -``openwisp_utils.base.KeyField`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +``openwisp_utils.base.FallBackModelMixin`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Model mixin that implements ``get_field_value`` method which can be used +to get value of fallback fields. + +Custom Fields +------------- + +This section describes custom fields defined in ``openwisp_utils.fields`` +that can be used in Django models: + +``openwisp_utils.fields.KeyField`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A model field which provides a random key or token, widely used across openwisp modules. + +``openwisp_utils.fields.FallbackBooleanChoiceField`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This field extends Django's `BooleanField `_ +and provides additional functionality for handling choices with a fallback value. +The field will use the **fallback value** whenever the field is set to ``None``. + +This field is particularly useful when you want to present a choice between enabled +and disabled options, with an additional "Default" option that reflects the fallback value. + +.. code-block:: python + + from django.db import models + from openwisp_utils.fields import FallbackBooleanChoiceField + from myapp import settings as app_settings + + class MyModel(models.Model): + is_active = FallbackBooleanChoiceField( + null=True, + blank=True, + default=None, + fallback=app_settings.IS_ACTIVE_FALLBACK, + ) + +``openwisp_utils.fields.FallbackCharChoiceField`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This field extends Django's `CharField `_ +and provides additional functionality for handling choices with a fallback value. +The field will use the **fallback value** whenever the field is set to ``None``. + +.. code-block:: python + + from django.db import models + from openwisp_utils.fields import FallbackCharChoiceField + from myapp import settings as app_settings + + class MyModel(models.Model): + is_first_name_required = FallbackCharChoiceField( + null=True, + blank=True, + max_length=32, + choices=( + ('disabled', _('Disabled')), + ('allowed', _('Allowed')), + ('mandatory', _('Mandatory')), + ), + fallback=app_settings.IS_FIRST_NAME_REQUIRED, + ) + +``openwisp_utils.fields.FallbackCharField`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This field extends Django's `CharField `_ +and provides additional functionality for handling text fields with a fallback value. -A model field whic provides a random key or token, widely used across openwisp modules. +It allows populating the form with the fallback value when the actual value is set to ``null`` in the database. + +.. code-block:: python + + from django.db import models + from openwisp_utils.fields import FallbackCharField + from myapp import settings as app_settings + + class MyModel(models.Model): + greeting_text = FallbackCharField( + null=True, + blank=True, + max_length=200, + fallback=app_settings.GREETING_TEXT, + ) + +``openwisp_utils.fields.FallbackURLField`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This field extends Django's `URLField `_ +and provides additional functionality for handling URL fields with a fallback value. + +It allows populating the form with the fallback value when the actual value is set to ``null`` in the database. + +.. code-block:: python + + from django.db import models + from openwisp_utils.fields import FallbackURLField + from myapp import settings as app_settings + + class MyModel(models.Model): + password_reset_url = FallbackURLField( + null=True, + blank=True, + max_length=200, + fallback=app_settings.DEFAULT_PASSWORD_RESET_URL, + ) + +``openwisp_utils.fields.FallbackTextField`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This extends Django's `TextField `_ +and provides additional functionality for handling text fields with a fallback value. + +It allows populating the form with the fallback value when the actual value is set to ``null`` in the database. + +.. code-block:: python + + from django.db import models + from openwisp_utils.fields import FallbackTextField + from myapp import settings as app_settings + + class MyModel(models.Model): + extra_config = FallbackTextField( + null=True, + blank=True, + max_length=200, + fallback=app_settings.EXTRA_CONFIG, + ) Admin utilities --------------- diff --git a/openwisp_utils/base.py b/openwisp_utils/base.py index 5daf993f..a949ab8b 100644 --- a/openwisp_utils/base.py +++ b/openwisp_utils/base.py @@ -3,8 +3,9 @@ from django.db import models from django.utils.translation import gettext_lazy as _ from model_utils.fields import AutoCreatedField, AutoLastModifiedField -from openwisp_utils.utils import get_random_key -from openwisp_utils.validators import key_validator + +# For backward compatibility +from .fields import KeyField # noqa class UUIDModel(models.Model): @@ -27,28 +28,10 @@ class Meta: abstract = True -class KeyField(models.CharField): - default_callable = get_random_key - default_validators = [key_validator] - - def __init__( - self, - max_length: int = 64, - unique: bool = False, - db_index: bool = False, - help_text: str = None, - default: [str, callable, None] = default_callable, - validators: list = default_validators, - *args, - **kwargs - ): - super().__init__( - max_length=max_length, - unique=unique, - db_index=db_index, - help_text=help_text, - default=default, - validators=validators, - *args, - **kwargs - ) +class FallbackModelMixin(object): + def get_field_value(self, field_name): + value = getattr(self, field_name) + field = self._meta.get_field(field_name) + if value is None and hasattr(field, 'fallback'): + return field.fallback + return value diff --git a/openwisp_utils/fields.py b/openwisp_utils/fields.py new file mode 100644 index 00000000..6e13dd35 --- /dev/null +++ b/openwisp_utils/fields.py @@ -0,0 +1,161 @@ +from django import forms +from django.db.models.fields import BooleanField, CharField, TextField, URLField +from django.utils.translation import gettext_lazy as _ +from openwisp_utils.utils import get_random_key +from openwisp_utils.validators import key_validator + + +class KeyField(CharField): + default_callable = get_random_key + default_validators = [key_validator] + + def __init__( + self, + max_length: int = 64, + unique: bool = False, + db_index: bool = False, + help_text: str = None, + default: [str, callable, None] = default_callable, + validators: list = default_validators, + *args, + **kwargs, + ): + super().__init__( + max_length=max_length, + unique=unique, + db_index=db_index, + help_text=help_text, + default=default, + validators=validators, + *args, + **kwargs, + ) + + +class FallbackMixin(object): + def __init__(self, *args, **kwargs): + self.fallback = kwargs.pop('fallback', None) + super().__init__(*args, **kwargs) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + kwargs['fallback'] = self.fallback + return (name, path, args, kwargs) + + +class FallbackFromDbValueMixin: + """ + Returns the fallback value when the value of the field + is falsy (None or ''). + + It does not set the field's value to "None" when the value + is equal to the fallback value. This allows overriding of + the value when a user knows that the default will get changed. + """ + + def from_db_value(self, value, expression, connection): + if value is None: + return self.fallback + return value + + +class FalsyValueNoneMixin: + """ + If the field contains an empty string, then + stores "None" in the database if the field is + nullable. + """ + + # Django convention is to use the empty string, not NULL + # for representing "no data" in the database. + # https://docs.djangoproject.com/en/dev/ref/models/fields/#null + # We need to use NULL for fallback field here to keep + # the fallback logic simple. Hence, we allow only "None" (NULL) + # as empty value here. + empty_values = [None] + + def clean(self, value, model_instance): + if not value and self.null is True: + return None + return super().clean(value, model_instance) + + +class FallbackBooleanChoiceField(FallbackMixin, BooleanField): + def formfield(self, **kwargs): + default_value = _('Enabled') if self.fallback else _('Disabled') + kwargs.update( + { + "form_class": forms.NullBooleanField, + 'widget': forms.Select( + choices=[ + ( + '', + _('Default') + f' ({default_value})', + ), + (True, _('Enabled')), + (False, _('Disabled')), + ] + ), + } + ) + return super().formfield(**kwargs) + + +class FallbackCharChoiceField(FallbackMixin, CharField): + def get_choices(self, **kwargs): + for choice, value in self.choices: + if choice == self.fallback: + default = value + break + kwargs.update({'blank_choice': [('', _('Default') + f' ({default})')]}) + return super().get_choices(**kwargs) + + def formfield(self, **kwargs): + kwargs.update( + { + "choices_form_class": forms.TypedChoiceField, + } + ) + return super().formfield(**kwargs) + + +class FallbackCharField( + FallbackMixin, FalsyValueNoneMixin, FallbackFromDbValueMixin, CharField +): + """ + Populates the form with the fallback value + if the value is set to null in the database. + """ + + pass + + +class FallbackURLField( + FallbackMixin, FalsyValueNoneMixin, FallbackFromDbValueMixin, URLField +): + """ + Populates the form with the fallback value + if the value is set to null in the database. + """ + + pass + + +class FallbackTextField( + FallbackMixin, FalsyValueNoneMixin, FallbackFromDbValueMixin, TextField +): + """ + Populates the form with the fallback value + if the value is set to null in the database. + """ + + def formfield(self, **kwargs): + kwargs.update({'form_class': FallbackTextFormField}) + return super().formfield(**kwargs) + + +class FallbackTextFormField(forms.CharField): + def widget_attrs(self, widget): + attrs = super().widget_attrs(widget) + attrs.update({'rows': 2, 'cols': 34, 'style': 'width:auto'}) + return attrs diff --git a/tests/test_project/admin.py b/tests/test_project/admin.py index c57ae36e..1b819465 100644 --- a/tests/test_project/admin.py +++ b/tests/test_project/admin.py @@ -16,7 +16,14 @@ SimpleInputFilter, ) -from .models import Book, Operator, Project, RadiusAccounting, Shelf +from .models import ( + Book, + Operator, + OrganizationRadiusSettings, + Project, + RadiusAccounting, + Shelf, +) admin.site.unregister(User) @@ -114,3 +121,8 @@ class ShelfAdmin(TimeReadonlyAdminMixin, admin.ModelAdmin): ReverseBookFilter, ] search_fields = ['name'] + + +@admin.register(OrganizationRadiusSettings) +class OrganizationRadiusSettingsAdmin(admin.ModelAdmin): + pass diff --git a/tests/test_project/migrations/0005_organizationradiussettings.py b/tests/test_project/migrations/0005_organizationradiussettings.py new file mode 100644 index 00000000..ec1bf6a6 --- /dev/null +++ b/tests/test_project/migrations/0005_organizationradiussettings.py @@ -0,0 +1,71 @@ +# Generated by Django 3.2.19 on 2023-06-24 15:15 + +from django.db import migrations, models +import openwisp_utils.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('test_project', '0004_sheft_data'), + ] + + operations = [ + migrations.CreateModel( + name='OrganizationRadiusSettings', + fields=[ + ( + 'id', + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ( + 'is_active', + openwisp_utils.fields.FallbackBooleanChoiceField( + blank=True, default=None, fallback=False, null=True + ), + ), + ( + 'is_first_name_required', + openwisp_utils.fields.FallbackCharChoiceField( + blank=True, + choices=[ + ('disabled', 'Disabled'), + ('allowed', 'Allowed'), + ('mandatory', 'Mandatory'), + ], + fallback='disabled', + max_length=32, + null=True, + ), + ), + ( + 'greeting_text', + openwisp_utils.fields.FallbackCharField( + blank=True, + fallback='Welcome to OpenWISP!', + max_length=200, + null=True, + ), + ), + ( + 'password_reset_url', + openwisp_utils.fields.FallbackURLField( + blank=True, + fallback='http://localhost:8000/admin/password_change/', + null=True, + ), + ), + ( + 'extra_config', + openwisp_utils.fields.FallbackTextField( + blank=True, fallback='no data', max_length=200, null=True + ), + ), + ], + ), + ] diff --git a/tests/test_project/models.py b/tests/test_project/models.py index b9590934..4e91c788 100644 --- a/tests/test_project/models.py +++ b/tests/test_project/models.py @@ -1,7 +1,19 @@ from django.core.exceptions import ValidationError from django.db import models from django.utils.translation import gettext_lazy as _ -from openwisp_utils.base import KeyField, TimeStampedEditableModel, UUIDModel +from openwisp_utils.base import ( + FallbackModelMixin, + KeyField, + TimeStampedEditableModel, + UUIDModel, +) +from openwisp_utils.fields import ( + FallbackBooleanChoiceField, + FallbackCharChoiceField, + FallbackCharField, + FallbackTextField, + FallbackURLField, +) class Shelf(TimeStampedEditableModel): @@ -72,6 +84,44 @@ class RadiusAccounting(models.Model): ) +class OrganizationRadiusSettings(FallbackModelMixin, models.Model): + is_active = FallbackBooleanChoiceField( + null=True, + blank=True, + default=None, + fallback=False, + ) + is_first_name_required = FallbackCharChoiceField( + null=True, + blank=True, + max_length=32, + choices=( + ('disabled', _('Disabled')), + ('allowed', _('Allowed')), + ('mandatory', _('Mandatory')), + ), + fallback='disabled', + ) + greeting_text = FallbackCharField( + null=True, + blank=True, + max_length=200, + fallback='Welcome to OpenWISP!', + ) + password_reset_url = FallbackURLField( + null=True, + blank=True, + max_length=200, + fallback='http://localhost:8000/admin/password_change/', + ) + extra_config = FallbackTextField( + null=True, + blank=True, + max_length=200, + fallback='no data', + ) + + class Project(UUIDModel): name = models.CharField(max_length=64, null=True, blank=True) key = KeyField(unique=True, db_index=True, help_text=_('unique project key')) diff --git a/tests/test_project/tests/__init__.py b/tests/test_project/tests/__init__.py index 95f1cafe..8c48fd08 100644 --- a/tests/test_project/tests/__init__.py +++ b/tests/test_project/tests/__init__.py @@ -40,3 +40,9 @@ def _create_radius_accounting(self, **kwargs): ra.full_clean() ra.save() return ra + + def _create_org_radius_settings(self, **kwargs): + org_rad_settings = self.org_radius_settings_model(**kwargs) + org_rad_settings.full_clean() + org_rad_settings.save() + return org_rad_settings diff --git a/tests/test_project/tests/test_admin.py b/tests/test_project/tests/test_admin.py index 76c89a50..5d84a89e 100644 --- a/tests/test_project/tests/test_admin.py +++ b/tests/test_project/tests/test_admin.py @@ -13,7 +13,13 @@ from openwisp_utils.admin_theme.filters import InputFilter, SimpleInputFilter from ..admin import ProjectAdmin, ShelfAdmin -from ..models import Operator, Project, RadiusAccounting, Shelf +from ..models import ( + Operator, + OrganizationRadiusSettings, + Project, + RadiusAccounting, + Shelf, +) from . import AdminTestMixin, CreateMixin User = get_user_model() @@ -22,6 +28,7 @@ class TestAdmin(AdminTestMixin, CreateMixin, TestCase): TEST_KEY = 'w1gwJxKaHcamUw62TQIPgYchwLKn3AA0' accounting_model = RadiusAccounting + org_radius_settings_model = OrganizationRadiusSettings def test_radiusaccounting_change(self): options = dict(username='bobby', session_id='1') @@ -478,3 +485,81 @@ def test_ow_autocomplete_filter_uuid_exception(self): response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertContains(response, '“invalid” is not a valid UUID.') + + def test_organization_radius_settings_admin(self): + org_rad_settings = self._create_org_radius_settings( + is_active=True, + is_first_name_required=None, + greeting_text=None, + password_reset_url='http://localhost:8000/reset-password/', + ) + url = reverse( + 'admin:test_project_organizationradiussettings_change', + args=[org_rad_settings.pk], + ) + + with self.subTest('Test default values are rendered'): + response = self.client.get(url) + # Overridden value is selected for BooleanChoiceField + self.assertContains( + response, + '', + html=True, + ) + # Default value is selected for FallbackCharChoiceField + self.assertContains( + response, + '', + html=True, + ) + # Default value is used for FallbackCharField + self.assertContains( + response, + '', + ) + # Overridden value is used for the FallbackURLField + self.assertContains( + response, + '', + ) + + with self.subTest('Test overriding default values from admin'): + payload = { + # Setting the default value for FallbackBooleanChoiceField + 'is_active': '', + # Overriding the default value for FallbackCharChoiceField + 'is_first_name_required': 'allowed', + # Overriding the default value for FallbackCharField + 'greeting_text': 'Greeting text', + # Setting the default value for FallbackURLField + 'password_reset_url': '', + # Setting the default value for FallbackTextField + 'extra_config': '', + } + response = self.client.post(url, payload, follow=True) + self.assertEqual(response.status_code, 200) + org_rad_settings.refresh_from_db() + self.assertEqual(org_rad_settings.get_field_value('is_active'), False) + self.assertEqual( + org_rad_settings.get_field_value('is_first_name_required'), 'allowed' + ) + self.assertEqual( + org_rad_settings.get_field_value('greeting_text'), 'Greeting text' + ) + self.assertEqual( + org_rad_settings.get_field_value('password_reset_url'), + 'http://localhost:8000/admin/password_change/', + ) + self.assertEqual( + org_rad_settings.get_field_value('extra_config'), 'no data' + ) diff --git a/tests/test_project/tests/test_model.py b/tests/test_project/tests/test_model.py index 40252c84..926006b0 100644 --- a/tests/test_project/tests/test_model.py +++ b/tests/test_project/tests/test_model.py @@ -1,7 +1,11 @@ +from unittest.mock import patch + from django.core.exceptions import ValidationError +from django.db import connection from django.test import TestCase -from ..models import Project +from ..models import OrganizationRadiusSettings, Project +from . import CreateMixin class TestModel(TestCase): @@ -20,3 +24,85 @@ def test_key_validator(self): p.full_clean() p.key = self.TEST_KEY p.full_clean() + + +class TestFallbackFields(CreateMixin, TestCase): + org_radius_settings_model = OrganizationRadiusSettings + + def test_fallback_field_falsy_values(self): + org_rad_settings = self._create_org_radius_settings() + + def _verify_none_database_value(field_name): + setattr(org_rad_settings, field_name, '') + org_rad_settings.full_clean() + org_rad_settings.save() + with connection.cursor() as cursor: + cursor.execute( + f'SELECT {field_name} FROM' + f' {org_rad_settings._meta.app_label}_{org_rad_settings._meta.model_name}' + f' WHERE id = \'{org_rad_settings.id}\';', + ) + row = cursor.fetchone() + self.assertEqual(row[0], None) + + with self.subTest('Test "greeting_text" field'): + _verify_none_database_value('greeting_text') + + with self.subTest('Test "password_reset_url" field'): + _verify_none_database_value('password_reset_url') + + with self.subTest('Test "extra_config" field'): + _verify_none_database_value('extra_config') + + def test_fallback_boolean_choice_field(self): + org_rad_settings = self._create_org_radius_settings() + + with self.subTest('Test is_active set to None'): + org_rad_settings.is_active = None + # Ensure fallback value is returned + self.assertEqual(org_rad_settings.get_field_value('is_active'), False) + + with self.subTest('Test fallback value changed'): + with patch.object( + # The fallback value is set on project startup, hence + # it also requires mocking. + OrganizationRadiusSettings._meta.get_field('is_active'), + 'fallback', + True, + ): + org_rad_settings.is_active = None + self.assertEqual(org_rad_settings.get_field_value('is_active'), True) + + with self.subTest('Test overriding default value'): + org_rad_settings.is_active = True + self.assertEqual(org_rad_settings.get_field_value('is_active'), True) + + def test_fallback_char_choice_field(self): + org_rad_settings = self._create_org_radius_settings() + + with self.subTest('Test is_first_name_required set to None'): + org_rad_settings.is_first_name_required = None + # Ensure fallback value is returned + self.assertEqual( + org_rad_settings.get_field_value('is_first_name_required'), 'disabled' + ) + + with self.subTest('Test fallback value changed'): + with patch.object( + # The fallback value is set on project startup, hence + # it also requires mocking. + OrganizationRadiusSettings._meta.get_field('is_first_name_required'), + 'fallback', + 'mandatory', + ): + org_rad_settings.is_first_name_required = None + self.assertEqual( + org_rad_settings.get_field_value('is_first_name_required'), + 'mandatory', + ) + + with self.subTest('Test overriding default value'): + org_rad_settings.is_first_name_required = 'allowed' + self.assertEqual( + org_rad_settings.get_field_value('is_first_name_required'), 'allowed' + ) From f7829677dfb8d5271fa780bf4f0c761b66f66c0a Mon Sep 17 00:00:00 2001 From: Gagan Deep Date: Thu, 6 Jul 2023 19:51:35 +0530 Subject: [PATCH 03/20] [feature] Added "openwisp_utils.db.backends.spatialite" database backend "openwisp_utils.db.backends.spatialite" extends "django.contrib.gis.db.backends.spatialite" to implement workaround for https://code.djangoproject.com/ticket/32935. --- README.rst | 10 ++++++++++ openwisp_utils/db/__init__.py | 0 openwisp_utils/db/backends/__init__.py | 0 openwisp_utils/db/backends/spatialite/__init__.py | 0 openwisp_utils/db/backends/spatialite/base.py | 11 +++++++++++ 5 files changed, 21 insertions(+) create mode 100644 openwisp_utils/db/__init__.py create mode 100644 openwisp_utils/db/backends/__init__.py create mode 100644 openwisp_utils/db/backends/spatialite/__init__.py create mode 100644 openwisp_utils/db/backends/spatialite/base.py diff --git a/README.rst b/README.rst index 36c763bf..3f95304c 100644 --- a/README.rst +++ b/README.rst @@ -1577,6 +1577,16 @@ Example usage: This mixin provides basic setup for Selenium tests with method to open URL and login and logout a user. +Database backends +----------------- + +``openwisp_utils.db.backends.spatialite`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This backend extends ``django.contrib.gis.db.backends.spatialite`` +database backend to implement a workaround for handling +`issue with sqlite 3.36 and spatialite 5 `_. + Quality Assurance Checks ------------------------ diff --git a/openwisp_utils/db/__init__.py b/openwisp_utils/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openwisp_utils/db/backends/__init__.py b/openwisp_utils/db/backends/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openwisp_utils/db/backends/spatialite/__init__.py b/openwisp_utils/db/backends/spatialite/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openwisp_utils/db/backends/spatialite/base.py b/openwisp_utils/db/backends/spatialite/base.py new file mode 100644 index 00000000..f31ef876 --- /dev/null +++ b/openwisp_utils/db/backends/spatialite/base.py @@ -0,0 +1,11 @@ +from django.contrib.gis.db.backends.spatialite import base + + +class DatabaseWrapper(base.DatabaseWrapper): + def prepare_database(self): + # Workaround for https://code.djangoproject.com/ticket/32935 + with self.cursor() as cursor: + cursor.execute("PRAGMA table_info(geometry_columns);") + if cursor.fetchall() == []: + cursor.execute("SELECT InitSpatialMetaData(1)") + super().prepare_database() From 153c45c270b4178d2c1b1b50eed71135020b9fcd Mon Sep 17 00:00:00 2001 From: Gagan Deep Date: Thu, 6 Jul 2023 22:42:14 +0530 Subject: [PATCH 04/20] [chores] Fixed changes in theme due to Django 4.2.0 - Fixed receive URL field - Fixed form row labels --- openwisp_utils/admin_theme/static/admin/css/openwisp.css | 3 +++ openwisp_utils/static/openwisp-utils/js/receive_url.js | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/openwisp_utils/admin_theme/static/admin/css/openwisp.css b/openwisp_utils/admin_theme/static/admin/css/openwisp.css index 825280ac..58c15c04 100644 --- a/openwisp_utils/admin_theme/static/admin/css/openwisp.css +++ b/openwisp_utils/admin_theme/static/admin/css/openwisp.css @@ -223,6 +223,9 @@ input[type=tel], textarea, padding: 8px 12px; box-sizing: border-box; } +.aligned label { + float: left; +} input.readonly { border: 1px solid rgba(0, 0, 0, 0.05) !important; background-color: rgba(0, 0, 0, 0.07); diff --git a/openwisp_utils/static/openwisp-utils/js/receive_url.js b/openwisp_utils/static/openwisp-utils/js/receive_url.js index 022924c1..c90f7f96 100644 --- a/openwisp_utils/static/openwisp-utils/js/receive_url.js +++ b/openwisp_utils/static/openwisp-utils/js/receive_url.js @@ -1,6 +1,6 @@ 'use strict'; django.jQuery(function ($) { - var p = $('.field-receive_url p, .field-receive_url > div > div'), + var p = $('.field-receive_url p, .field-receive_url div.readonly'), value = p.text(); p.html(''); var field = $('#id_receive_url'); From 242afc8b1aa70c4691fd5b43e9ce1a3ca9eb8700 Mon Sep 17 00:00:00 2001 From: Gagan Deep Date: Wed, 26 Jul 2023 22:51:17 +0530 Subject: [PATCH 05/20] [feature] Added FallbackPositiveIntegerField --- README.rst | 21 ++++++++++++++ openwisp_utils/fields.py | 14 ++++++++- .../0006_alter_shelf_books_count.py | 21 ++++++++++++++ tests/test_project/models.py | 10 +++++-- tests/test_project/tests/test_model.py | 29 ++++++++++++++++++- 5 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 tests/test_project/migrations/0006_alter_shelf_books_count.py diff --git a/README.rst b/README.rst index 3f95304c..09deef45 100644 --- a/README.rst +++ b/README.rst @@ -1030,6 +1030,27 @@ It allows populating the form with the fallback value when the actual value is s fallback=app_settings.EXTRA_CONFIG, ) +``openwisp_utils.fields.FallbackPositiveIntegerField`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This extends Django's `PositiveIntegerField `_ +and provides additional functionality for handling positive integer fields with a fallback value. + +It allows populating the form with the fallback value when the actual value is set to ``null`` in the database. + +.. code-block:: python + + from django.db import models + from openwisp_utils.fields import FallbackPositiveIntegerField + from myapp import settings as app_settings + + class MyModel(models.Model): + count = FallbackPositiveIntegerField( + blank=True, + null=True, + fallback=app_settings.DEFAULT_COUNT, + ) + Admin utilities --------------- diff --git a/openwisp_utils/fields.py b/openwisp_utils/fields.py index 6e13dd35..731224ad 100644 --- a/openwisp_utils/fields.py +++ b/openwisp_utils/fields.py @@ -1,5 +1,11 @@ from django import forms -from django.db.models.fields import BooleanField, CharField, TextField, URLField +from django.db.models.fields import ( + BooleanField, + CharField, + PositiveIntegerField, + TextField, + URLField, +) from django.utils.translation import gettext_lazy as _ from openwisp_utils.utils import get_random_key from openwisp_utils.validators import key_validator @@ -119,6 +125,12 @@ def formfield(self, **kwargs): return super().formfield(**kwargs) +class FallbackPositiveIntegerField( + FallbackMixin, FallbackFromDbValueMixin, PositiveIntegerField +): + pass + + class FallbackCharField( FallbackMixin, FalsyValueNoneMixin, FallbackFromDbValueMixin, CharField ): diff --git a/tests/test_project/migrations/0006_alter_shelf_books_count.py b/tests/test_project/migrations/0006_alter_shelf_books_count.py new file mode 100644 index 00000000..c76af13a --- /dev/null +++ b/tests/test_project/migrations/0006_alter_shelf_books_count.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.3 on 2023-07-25 11:19 + +from django.db import migrations +import openwisp_utils.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("test_project", "0005_organizationradiussettings"), + ] + + operations = [ + migrations.AlterField( + model_name="shelf", + name="books_count", + field=openwisp_utils.fields.FallbackPositiveIntegerField( + blank=True, fallback=21, null=True, verbose_name="Number of books" + ), + ), + ] diff --git a/tests/test_project/models.py b/tests/test_project/models.py index 4e91c788..9170b25c 100644 --- a/tests/test_project/models.py +++ b/tests/test_project/models.py @@ -11,12 +11,13 @@ FallbackBooleanChoiceField, FallbackCharChoiceField, FallbackCharField, + FallbackPositiveIntegerField, FallbackTextField, FallbackURLField, ) -class Shelf(TimeStampedEditableModel): +class Shelf(FallbackModelMixin, TimeStampedEditableModel): TYPES = ( ('HORROR', 'HORROR'), ('FANTASY', 'FANTASY'), @@ -34,7 +35,12 @@ class Shelf(TimeStampedEditableModel): books_type = models.CharField( _("Type of book"), choices=TYPES, null=True, blank=True, max_length=50 ) - books_count = models.PositiveIntegerField(_("Number of books"), default=0) + books_count = FallbackPositiveIntegerField( + blank=True, + null=True, + fallback=21, + verbose_name=_("Number of books"), + ) locked = models.BooleanField(_("Is locked"), default=True) owner = models.ForeignKey( "auth.User", diff --git a/tests/test_project/tests/test_model.py b/tests/test_project/tests/test_model.py index 926006b0..8ec6a4c4 100644 --- a/tests/test_project/tests/test_model.py +++ b/tests/test_project/tests/test_model.py @@ -4,7 +4,7 @@ from django.db import connection from django.test import TestCase -from ..models import OrganizationRadiusSettings, Project +from ..models import OrganizationRadiusSettings, Project, Shelf from . import CreateMixin @@ -28,6 +28,7 @@ def test_key_validator(self): class TestFallbackFields(CreateMixin, TestCase): org_radius_settings_model = OrganizationRadiusSettings + shelf_model = Shelf def test_fallback_field_falsy_values(self): org_rad_settings = self._create_org_radius_settings() @@ -106,3 +107,29 @@ def test_fallback_char_choice_field(self): self.assertEqual( org_rad_settings.get_field_value('is_first_name_required'), 'allowed' ) + + def test_fallback_positive_integer_field(self): + shelf = self._create_shelf() + + with self.subTest('Test books_count set to None'): + shelf.books_count = None + # Ensure fallback value is returned + self.assertEqual(shelf.get_field_value('books_count'), 21) + + with self.subTest('Test fallback value changed'): + with patch.object( + # The fallback value is set on project startup, hence + # it also requires mocking. + Shelf._meta.get_field('books_count'), + 'fallback', + 32, + ): + shelf.books_count = None + self.assertEqual( + shelf.get_field_value('books_count'), + 32, + ) + + with self.subTest('Test overriding default value'): + shelf.books_count = 56 + self.assertEqual(shelf.get_field_value('books_count'), 56) From 62527ad763a445898c1d8f56bf7a354b2b1a8a65 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Tue, 22 Aug 2023 14:26:03 +0200 Subject: [PATCH 06/20] [deps] Updated drf-yasg~=1.21.7 #352 Closes #352 --- setup.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/setup.py b/setup.py index ce144390..4bd5dfbe 100644 --- a/setup.py +++ b/setup.py @@ -59,13 +59,7 @@ 'rest': [ 'djangorestframework~=3.14.0', 'django-filter~=23.2', # django-filter uses CalVer - # The coreapi package is archived and all packages - # are moving away from coreapi (e.g. DRF, django-filter, drf-yasg). - # There's already an open PR in drf-yasg - # https://github.com/axnsan12/drf-yasg/pull/857. - # TODO: Before releasing, check if newer version - # of drf-yasg is available. - 'drf-yasg[coreapi]~=1.21.0', + 'drf-yasg~=1.21.7', ], 'celery': ['celery~=5.3.0'], 'selenium': ['selenium~=4.10.0'], From 569698f2e2b406d9d78c6b7a24dd6528f7e94e9a Mon Sep 17 00:00:00 2001 From: Gagan Deep Date: Tue, 26 Sep 2023 17:21:57 +0530 Subject: [PATCH 07/20] [feature] Added filter option to dashboard chart Related to: https://github.com/openwisp/openwisp-monitoring/issues/528 --- README.rst | 7 +++ openwisp_utils/admin_theme/dashboard.py | 45 +++++++++++++++---- .../static/admin/js/ow-dashboard.js | 12 +++-- tests/test_project/apps.py | 44 ++++++++++++++++++ ...07_radiusaccounting_start_time_and_more.py | 25 +++++++++++ tests/test_project/models.py | 10 +++++ tests/test_project/tests/test_admin.py | 10 ++++- tests/test_project/tests/test_dashboard.py | 40 +++++++++++++++-- 8 files changed, 176 insertions(+), 17 deletions(-) create mode 100644 tests/test_project/migrations/0007_radiusaccounting_start_time_and_more.py diff --git a/README.rst b/README.rst index 09deef45..d7bfd97b 100644 --- a/README.rst +++ b/README.rst @@ -461,6 +461,8 @@ Following properties can be configured for each chart ``config``: | | +------------------------+---------------------------------------------------------------------------+ | | | | ``aggregate`` | Alternative to ``group_by``, ``dict`` used for more complex queries. | | | | +------------------------+---------------------------------------------------------------------------+ | +| | | ``filter`` | ``dict`` used for filtering queryset. | | +| | +------------------------+---------------------------------------------------------------------------+ | | | | ``organization_field`` | (``str``) If the model does not have a direct relation with the | | | | | | ``Organization`` model, then indirect relation can be specified using | | | | | | this property. E.g.: ``device__organization_id``. | | @@ -477,6 +479,11 @@ Following properties can be configured for each chart ``config``: | | ``query_params`` to define the link that will be generated to filter results (pie charts are | | | clickable and clicking on a portion of it will show the filtered results). | +------------------+---------------------------------------------------------------------------------------------------------+ +| ``main_filters`` | An **optional** ``dict`` which can be used to add additional filtering on the target link. | ++------------------+---------------------------------------------------------------------------------------------------------+ +| ``filtering`` | An **optional** ``str`` which can be set to ``'False'`` (str) to disable filtering on target links. | +| | This is useful when clicking on any section of the chart should take user to the same URL. | ++------------------+---------------------------------------------------------------------------------------------------------+ | ``quick_link`` | An **optional** ``dict`` which contains configuration for the quick link button rendered | | | below the chart. | | | | diff --git a/openwisp_utils/admin_theme/dashboard.py b/openwisp_utils/admin_theme/dashboard.py index 43b0cd60..106fe1d1 100644 --- a/openwisp_utils/admin_theme/dashboard.py +++ b/openwisp_utils/admin_theme/dashboard.py @@ -18,7 +18,11 @@ def _validate_chart_config(config): assert 'name' in config assert 'app_label' in query_params assert 'model' in query_params - assert 'group_by' in query_params or 'annotate' in query_params + assert ( + 'filter' in query_params + or 'group_by' in query_params + or 'annotate' in query_params + ) assert not ('group_by' in query_params and 'annotate' in query_params) if 'annotate' in query_params: assert 'filters' in config, 'filters must be defined when using annotate' @@ -135,6 +139,7 @@ def get_dashboard_context(request): query_params = value['query_params'] app_label = query_params['app_label'] model_name = query_params['model'] + qs_filter = query_params.get('filter') group_by = query_params.get('group_by') annotate = query_params.get('annotate') aggregate = query_params.get('aggregate') @@ -150,7 +155,12 @@ def get_dashboard_context(request): f'REASON: {app_label}.{model_name} could not be loaded.' ) - qs = model.objects.all() + qs = model.objects + if qs_filter: + for field, lookup_value in qs_filter.items(): + if callable(lookup_value): + qs_filter[field] = lookup_value() + qs = qs.filter(**qs_filter) # Filter query according to organization of user if not request.user.is_superuser and ( @@ -179,6 +189,20 @@ def get_dashboard_context(request): labels = [] colors = [] filters = [] + main_filters = [] + url_operator = '?' + value['target_link'] = f'/admin/{app_label}/{model_name}/' + if value.get('main_filters'): + for main_filter_key, main_filter_value in value['main_filters'].items(): + if callable(main_filter_value): + main_filter_value = str(main_filter_value()) + main_filters.append(f'{main_filter_key}={main_filter_value}') + + value['target_link'] = '{path}?{main_filters}'.format( + path=value['target_link'], main_filters='&'.join(main_filters) + ) + value.pop('main_filters', None) + url_operator = '&' if group_by: for obj in qs: @@ -201,9 +225,9 @@ def get_dashboard_context(request): if value.get('colors') and qs_key in value['colors']: colors.append(value['colors'][qs_key]) values.append(obj['count']) - value[ - 'target_link' - ] = f'/admin/{app_label}/{model_name}/?{group_by}__exact=' + value['target_link'] = '{path}{url_operator}{group_by}__exact='.format( + path=value['target_link'], url_operator=url_operator, group_by=group_by + ) if aggregate: for qs_key, qs_value in qs.items(): @@ -212,9 +236,14 @@ def get_dashboard_context(request): labels.append(labels_i18n[qs_key]) values.append(qs_value) colors.append(value['colors'][qs_key]) - filters.append(value['filters'][qs_key]) - filter_key = value['filters']['key'] - value['target_link'] = f'/admin/{app_label}/{model_name}/?{filter_key}=' + if value.get('filters'): + filters.append(value['filters'][qs_key]) + if value.get('filters'): + value['target_link'] = '{path}{url_operator}{filter_key}='.format( + url_operator=url_operator, + path=value['target_link'], + filter_key=value['filters']['key'], + ) value['query_params'] = {'values': values, 'labels': labels} value['colors'] = colors diff --git a/openwisp_utils/admin_theme/static/admin/js/ow-dashboard.js b/openwisp_utils/admin_theme/static/admin/js/ow-dashboard.js index 7d78c76e..f62d6240 100644 --- a/openwisp_utils/admin_theme/static/admin/js/ow-dashboard.js +++ b/openwisp_utils/admin_theme/static/admin/js/ow-dashboard.js @@ -84,6 +84,7 @@ data.texttemplate = '%{value}
(%{percent})'; data.targetLink = elementsParam[i].target_link; data.filters = elementsParam[i].filters; + data.filtering = elementsParam[i].filtering; // add total to pie chart for (var c = 0; c < data.values.length; c++) { @@ -109,11 +110,14 @@ element.on('plotly_click', function (data) { var path = data.points[0].data.targetLink, filters = data.points[0].data.filters, + filtering = data.points[0].data.filtering, i = data.points[0].i; - if (filters && typeof(filters[i]) !== 'undefined') { - path += filters[i]; - } else { - path += encodeURIComponent(data.points[0].label); + if (filtering !== 'False'){ + if (filters && typeof(filters[i]) !== 'undefined') { + path += filters[i]; + } else { + path += encodeURIComponent(data.points[0].label); + } } window.location = path; }); diff --git a/tests/test_project/apps.py b/tests/test_project/apps.py index 0f98feee..d4049920 100644 --- a/tests/test_project/apps.py +++ b/tests/test_project/apps.py @@ -1,5 +1,6 @@ from django.db.models import Case, Count, Sum, When from django.urls import reverse_lazy +from django.utils.timezone import localdate from django.utils.translation import gettext_lazy as _ from openwisp_utils.admin_theme import ( register_dashboard_chart, @@ -98,6 +99,49 @@ def register_dashboard_charts(self): }, }, ) + register_dashboard_chart( + position=3, + config={ + 'name': _('Open RADIUS Sessions'), + 'query_params': { + 'app_label': 'test_project', + 'model': 'radiusaccounting', + 'filter': { + 'stop_time__isnull': True, + 'start_time__date': localdate, + }, + 'aggregate': { + 'active__count': Count('id'), + }, + }, + 'colors': { + 'active__count': '#267126', + }, + 'labels': { + 'active__count': _('Active sessions'), + }, + 'filters': { + 'key': 'stop_time__isnull', + 'active__count': 'true', + }, + }, + ) + register_dashboard_chart( + position=4, + config={ + 'name': _('Shelf Books Type'), + 'query_params': { + 'app_label': 'test_project', + 'model': 'shelf', + 'group_by': 'books_type', + }, + 'colors': {'HORROR': 'red', 'FANTASY': 'orange'}, + 'labels': {'HORROR': _('Horror'), 'FANTASY': _('Fantasy')}, + 'main_filters': { + 'created_at__date': localdate, + }, + }, + ) register_dashboard_template( position=0, config={ diff --git a/tests/test_project/migrations/0007_radiusaccounting_start_time_and_more.py b/tests/test_project/migrations/0007_radiusaccounting_start_time_and_more.py new file mode 100644 index 00000000..975507bc --- /dev/null +++ b/tests/test_project/migrations/0007_radiusaccounting_start_time_and_more.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.3 on 2023-09-13 10:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("test_project", "0006_alter_shelf_books_count"), + ] + + operations = [ + migrations.AddField( + model_name="radiusaccounting", + name="start_time", + field=models.DateTimeField( + blank=True, null=True, verbose_name="start time" + ), + ), + migrations.AddField( + model_name="radiusaccounting", + name="stop_time", + field=models.DateTimeField(blank=True, null=True, verbose_name="stop time"), + ), + ] diff --git a/tests/test_project/models.py b/tests/test_project/models.py index 9170b25c..1b0ae101 100644 --- a/tests/test_project/models.py +++ b/tests/test_project/models.py @@ -88,6 +88,16 @@ class RadiusAccounting(models.Model): username = models.CharField( verbose_name=_('username'), max_length=64, db_index=True, null=True, blank=True ) + start_time = models.DateTimeField( + verbose_name=_('start time'), + null=True, + blank=True, + ) + stop_time = models.DateTimeField( + verbose_name=_('stop time'), + null=True, + blank=True, + ) class OrganizationRadiusSettings(FallbackModelMixin, models.Model): diff --git a/tests/test_project/tests/test_admin.py b/tests/test_project/tests/test_admin.py index 5d84a89e..50e057d4 100644 --- a/tests/test_project/tests/test_admin.py +++ b/tests/test_project/tests/test_admin.py @@ -104,14 +104,20 @@ class TestReadOnlyAdmin(ReadOnlyAdmin): exclude = ['id'] modeladmin = TestReadOnlyAdmin(RadiusAccounting, AdminSite) - self.assertEqual(modeladmin.readonly_fields, ['session_id', 'username']) + self.assertEqual( + modeladmin.readonly_fields, + ['session_id', 'username', 'start_time', 'stop_time'], + ) def test_readonlyadmin_fields(self): class TestReadOnlyAdmin(ReadOnlyAdmin): pass modeladmin = TestReadOnlyAdmin(RadiusAccounting, AdminSite) - self.assertEqual(modeladmin.readonly_fields, ['id', 'session_id', 'username']) + self.assertEqual( + modeladmin.readonly_fields, + ['id', 'session_id', 'username', 'start_time', 'stop_time'], + ) def test_context_processor(self): url = reverse('admin:index') diff --git a/tests/test_project/tests/test_dashboard.py b/tests/test_project/tests/test_dashboard.py index ef227162..3d9f1c70 100644 --- a/tests/test_project/tests/test_dashboard.py +++ b/tests/test_project/tests/test_dashboard.py @@ -6,6 +6,7 @@ from django.core.exceptions import ImproperlyConfigured from django.test import TestCase as DjangoTestCase from django.urls import reverse +from django.utils.timezone import localdate, now, timedelta from openwisp_utils.admin_theme import ( register_dashboard_chart, register_dashboard_template, @@ -14,8 +15,8 @@ ) from openwisp_utils.admin_theme.dashboard import get_dashboard_context -from ..models import Project -from . import AdminTestMixin +from ..models import Project, RadiusAccounting +from . import AdminTestMixin, CreateMixin from .utils import MockRequest, MockUser @@ -177,8 +178,25 @@ def test_dashboard_after_charts(self): ) -class TestAdminDashboard(AdminTestMixin, DjangoTestCase): +class TestAdminDashboard(AdminTestMixin, CreateMixin, DjangoTestCase): + accounting_model = RadiusAccounting + def test_index_content(self): + self._create_radius_accounting( + session_id='1', + start_time=now(), + stop_time=now(), + ) + self._create_radius_accounting( + session_id='2', + start_time=now() - timedelta(days=1), + stop_time=None, + ) + self._create_radius_accounting( + session_id='2', + start_time=now(), + stop_time=None, + ) response = self.client.get(reverse('admin:index')) self.assertContains(response, 'Operator Project Distribution') self.assertContains(response, '\'values\': [1, 1]') @@ -205,6 +223,22 @@ def test_index_content(self): self.assertContains(response, 'with_operator') self.assertContains(response, 'without_operator') self.assertContains(response, 'project__name__exact') + # Assertion for main_filter + # "created_at__date=" is the main filter + self.assertContains( + response, + '/admin/test_project/shelf/?' + f'created_at__date={localdate()}' + '&books_type__exact=', + ) + # Assertion for queryset filtering + self.assertContains( + response, + '{' + '\'name\': \'Open RADIUS Sessions\', ' + '\'query_params\': {\'values\': [1], ' + '\'labels\': [\'Active sessions\']}', + ) with self.subTest('Test no data'): Project.objects.all().delete() From 59853689aee8eed36dd927b59069e6956237f32e Mon Sep 17 00:00:00 2001 From: shubham yadav <68185027+shubham-yadavv@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:29:29 +0530 Subject: [PATCH 08/20] [docs] Updated CONTRIBUTING.rst file with link to guidelines --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 81509155..6caad925 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,5 +1,5 @@ Contributing ============ -Please read the Contributing section in the README of this project. +Please refer to the `OpenWISP contributing guidelines `_. From d3fce3889a6eb932178ba3cd1a2d81dc9a9edf41 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Fri, 10 Nov 2023 13:56:02 -0300 Subject: [PATCH 09/20] [deps] Updated tblib --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4bd5dfbe..dd057324 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ 'isort~=5.0', 'readme-renderer~=28.0', 'coveralls~=3.0.0', # depends on coverage as well - 'tblib~=1.7', + 'tblib~=3.0.0', ], 'rest': [ 'djangorestframework~=3.14.0', From 746b14618dc1df922224f8cb3f258bc3f788e0e1 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Thu, 23 Nov 2023 12:35:32 -0300 Subject: [PATCH 10/20] [fix] Make sure to HTML escape all dashboard labels --- openwisp_utils/admin_theme/dashboard.py | 12 ++++++++++- tests/test_project/apps.py | 4 +++- tests/test_project/tests/test_dashboard.py | 25 +++++++++++++++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/openwisp_utils/admin_theme/dashboard.py b/openwisp_utils/admin_theme/dashboard.py index 106fe1d1..1f3ccf6f 100644 --- a/openwisp_utils/admin_theme/dashboard.py +++ b/openwisp_utils/admin_theme/dashboard.py @@ -1,4 +1,5 @@ import copy +import html from django.core.exceptions import ImproperlyConfigured from django.db.models import Count @@ -145,7 +146,12 @@ def get_dashboard_context(request): aggregate = query_params.get('aggregate') org_field = query_params.get('organization_field') default_org_field = 'organization_id' + labels_i18n = value.get('labels') + # HTML escape labels defined in configuration to prevent breaking the JS + if labels_i18n: + for label_key, label_value in labels_i18n.items(): + labels_i18n[label_key] = html.escape(label_value) try: model = load_model(app_label, model_name) @@ -216,9 +222,13 @@ def get_dashboard_context(request): if labels_i18n and qs_key in labels_i18n: # store original label as filter, but only # if we have more than the empty default label defined - # if len(labels_i18n.keys()) > 1 filters.append(label) label = labels_i18n[qs_key] + else: + # HTML escape labels coming from values in the DB + # to avoid possible XSS attacks caused by + # malicious DB values set by users + label = html.escape(label) labels.append(label) # use predefined colors if available, # otherwise the JS lib will choose automatically diff --git a/tests/test_project/apps.py b/tests/test_project/apps.py index d4049920..7e7a4395 100644 --- a/tests/test_project/apps.py +++ b/tests/test_project/apps.py @@ -89,7 +89,9 @@ def register_dashboard_charts(self): 'without_operator__sum': '#353c44', }, 'labels': { - 'with_operator__sum': _('Projects with operators'), + # the is for testing purposes to + # verify it's being HTML escaped correctly + 'with_operator__sum': _('Projects with operators'), 'without_operator__sum': _('Projects without operators'), }, 'filters': { diff --git a/tests/test_project/tests/test_dashboard.py b/tests/test_project/tests/test_dashboard.py index 3d9f1c70..97af6f9a 100644 --- a/tests/test_project/tests/test_dashboard.py +++ b/tests/test_project/tests/test_dashboard.py @@ -15,7 +15,7 @@ ) from openwisp_utils.admin_theme.dashboard import get_dashboard_context -from ..models import Project, RadiusAccounting +from ..models import Operator, Project, RadiusAccounting from . import AdminTestMixin, CreateMixin from .utils import MockRequest, MockUser @@ -271,3 +271,26 @@ def test_dashboard_disabled(self): with self.subTest('Test "Dashboard" is absent from menu items'): response = self.client.get(reverse('admin:index')) self.assertNotContains(response, 'Dashboard') + + def test_get_dashboard_context_html_escape(self): + # craft malicious DB value which will be shown in labels + project = Project.objects.create(name='') + Operator.objects.create(project=project, first_name='xss', last_name='xss') + # prepare mock request and get context + mocked_user = MockUser(is_superuser=True) + mocked_request = MockRequest(user=mocked_user) + context = get_dashboard_context(mocked_request) + # ensure DB value is escaped + self.assertEqual( + context['dashboard_charts'][0]['query_params']['labels'][0], + '<script>alert(1)</script>', + ) + # ensure configured labels are escaped + self.assertEqual( + context['dashboard_charts'][1]['labels']['with_operator__sum'], + '<strong>Projects with operators</strong>', + ) + self.assertEqual( + context['dashboard_charts'][1]['query_params']['labels'][0], + '<strong>Projects with operators</strong>', + ) From 61ac1b063ac7e47eeb434c87a61f54e5c480364d Mon Sep 17 00:00:00 2001 From: Gagan Deep Date: Thu, 30 Nov 2023 02:56:54 +0530 Subject: [PATCH 11/20] [feature] Added page to show OpenWISP versions #237 Closes #273 --- openwisp_utils/admin_theme/admin.py | 22 ++++++++ openwisp_utils/admin_theme/apps.py | 8 +++ .../admin_theme/static/admin/css/openwisp.css | 4 ++ .../static/ui/openwisp/images/info.svg | 1 + openwisp_utils/admin_theme/system_info.py | 55 +++++++++++++++++++ .../templates/admin/openwisp_info.html | 18 ++++++ runtests.py | 2 +- tests/__init__.py | 0 tests/manage.py | 2 +- tests/openwisp2/__init__.py | 1 + .../{ => openwisp2}/local_settings.example.py | 0 tests/{ => openwisp2}/settings.py | 2 +- tests/{ => openwisp2}/urls.py | 0 tests/test_project/tests/test_admin.py | 30 ++++++++++ 14 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 openwisp_utils/admin_theme/static/ui/openwisp/images/info.svg create mode 100644 openwisp_utils/admin_theme/system_info.py create mode 100644 openwisp_utils/admin_theme/templates/admin/openwisp_info.html delete mode 100644 tests/__init__.py create mode 100644 tests/openwisp2/__init__.py rename tests/{ => openwisp2}/local_settings.example.py (100%) rename tests/{ => openwisp2}/settings.py (99%) rename tests/{ => openwisp2}/urls.py (100%) diff --git a/openwisp_utils/admin_theme/admin.py b/openwisp_utils/admin_theme/admin.py index 555721c1..dc36759d 100644 --- a/openwisp_utils/admin_theme/admin.py +++ b/openwisp_utils/admin_theme/admin.py @@ -2,12 +2,19 @@ from django.conf import settings from django.contrib import admin +from django.shortcuts import render from django.urls import path from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy +from django.utils.translation import gettext_lazy as _ from . import settings as app_settings from .dashboard import get_dashboard_context +from .system_info import ( + get_enabled_openwisp_modules, + get_openwisp_version, + get_os_details, +) logger = logging.getLogger(__name__) @@ -30,6 +37,16 @@ def index(self, request, extra_context=None): context = {'dashboard_enabled': False} return super().index(request, extra_context=context) + def openwisp_info(self, request, *args, **kwargs): + context = { + 'enabled_openwisp_modules': get_enabled_openwisp_modules(), + 'system_info': get_os_details(), + 'openwisp_version': get_openwisp_version(), + 'title': _('System Information'), + 'site_title': self.site_title, + } + return render(request, 'admin/openwisp_info.html', context) + def get_urls(self): autocomplete_view = import_string(app_settings.AUTOCOMPLETE_FILTER_VIEW) return [ @@ -38,6 +55,11 @@ def get_urls(self): self.admin_view(autocomplete_view.as_view(admin_site=self)), name='ow-auto-filter', ), + path( + 'openwisp-system-info/', + self.admin_view(self.openwisp_info), + name='ow-info', + ), ] + super().get_urls() diff --git a/openwisp_utils/admin_theme/apps.py b/openwisp_utils/admin_theme/apps.py index 584bb0d0..9793a006 100644 --- a/openwisp_utils/admin_theme/apps.py +++ b/openwisp_utils/admin_theme/apps.py @@ -51,6 +51,14 @@ def register_menu_groups(self): position=10, config={'label': _('Home'), 'url': '/admin', 'icon': 'ow-dashboard-icon'}, ) + register_menu_group( + position=899, + config={ + 'label': _('System info'), + 'url': '/admin/ow-info', + 'icon': 'ow-info-icon', + }, + ) def modify_admin_theme_settings_links(self): link_files = [] diff --git a/openwisp_utils/admin_theme/static/admin/css/openwisp.css b/openwisp_utils/admin_theme/static/admin/css/openwisp.css index 58c15c04..dcf71a07 100644 --- a/openwisp_utils/admin_theme/static/admin/css/openwisp.css +++ b/openwisp_utils/admin_theme/static/admin/css/openwisp.css @@ -1511,6 +1511,10 @@ body::-webkit-scrollbar { mask-image: url(../../ui/openwisp/images/dashboard.svg); -webkit-mask-image: url(../../ui/openwisp/images/dashboard.svg); } +.ow-info-icon { + mask-image: url(../../ui/openwisp/images/info.svg); + -webkit-mask-image: url(../../ui/openwisp/images/info.svg); +} .password { mask-image: url(../../ui/openwisp/images/password.svg); -webkit-mask-image: url(../../ui/openwisp/images/password.svg); diff --git a/openwisp_utils/admin_theme/static/ui/openwisp/images/info.svg b/openwisp_utils/admin_theme/static/ui/openwisp/images/info.svg new file mode 100644 index 00000000..1244ae88 --- /dev/null +++ b/openwisp_utils/admin_theme/static/ui/openwisp/images/info.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/openwisp_utils/admin_theme/system_info.py b/openwisp_utils/admin_theme/system_info.py new file mode 100644 index 00000000..e0a0dc24 --- /dev/null +++ b/openwisp_utils/admin_theme/system_info.py @@ -0,0 +1,55 @@ +import platform +from collections import OrderedDict + +import pkg_resources +from django.conf import settings +from django.utils.module_loading import import_string + +EXTRA_OPENWISP_PACKAGES = ['netdiff', 'netjsonconfig'] + + +def get_installed_openwisp_packages(): + dists = pkg_resources.working_set + return { + dist.key: dist.version + for dist in dists + if dist.key.startswith('openwisp') or dist.key in EXTRA_OPENWISP_PACKAGES + } + + +def get_openwisp_version(): + try: + return import_string('openwisp2.__openwisp_version__') + except ImportError: + return None + + +def get_enabled_openwisp_modules(): + enabled_packages = {} + installed_packages = get_installed_openwisp_packages() + extra_packages = {} + for package, version in installed_packages.items(): + if package in EXTRA_OPENWISP_PACKAGES: + extra_packages[package] = version + continue + package_name = package.replace('-', '_') + if package_name in settings.INSTALLED_APPS: + enabled_packages[package] = version + else: + # check for sub-apps + for app in settings.INSTALLED_APPS: + if app.startswith(package_name + '.'): + enabled_packages[package] = version + break + enabled_packages = OrderedDict(sorted(enabled_packages.items())) + enabled_packages.update(OrderedDict(sorted(extra_packages.items()))) + return enabled_packages + + +def get_os_details(): + uname = platform.uname() + return { + 'os_version': uname.version, + 'kernel_version': uname.release, + 'hardware_platform': uname.machine, + } diff --git a/openwisp_utils/admin_theme/templates/admin/openwisp_info.html b/openwisp_utils/admin_theme/templates/admin/openwisp_info.html new file mode 100644 index 00000000..b4ad589c --- /dev/null +++ b/openwisp_utils/admin_theme/templates/admin/openwisp_info.html @@ -0,0 +1,18 @@ +{% extends "admin/base_site.html" %} +{% load i18n %} + +{% block content %} +{% if openwisp_version %} +

{% trans "OpenWISP Version" %}: {{ openwisp_version }}

+{% endif %} +

{% trans "Installed OpenWISP Modules" %}

+
    +{% for name, version in enabled_openwisp_modules.items %} +
  • {{ name }}: {{ version }}
    • +{% endfor %} +
+

{% trans "OS Information" %}

+

{% trans "OS version" %}: {{ system_info.os_version }}

+

{% trans "Kernel version" %}: {{ system_info.kernel_version }}

+

{% trans "Hardware platform" %}: {{ system_info.hardware_platform }}

+{% endblock content %} diff --git a/runtests.py b/runtests.py index 97bde978..5427a446 100755 --- a/runtests.py +++ b/runtests.py @@ -5,7 +5,7 @@ import sys sys.path.insert(0, "tests") -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "openwisp2.settings") if __name__ == "__main__": from django.core.management import execute_from_command_line diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/manage.py b/tests/manage.py index f9726f9e..33876c7b 100755 --- a/tests/manage.py +++ b/tests/manage.py @@ -3,7 +3,7 @@ import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "openwisp2.settings") from django.core.management import execute_from_command_line diff --git a/tests/openwisp2/__init__.py b/tests/openwisp2/__init__.py new file mode 100644 index 00000000..62bcec8d --- /dev/null +++ b/tests/openwisp2/__init__.py @@ -0,0 +1 @@ +__openwisp_version__ = '23.0.0a' diff --git a/tests/local_settings.example.py b/tests/openwisp2/local_settings.example.py similarity index 100% rename from tests/local_settings.example.py rename to tests/openwisp2/local_settings.example.py diff --git a/tests/settings.py b/tests/openwisp2/settings.py similarity index 99% rename from tests/settings.py rename to tests/openwisp2/settings.py index dffaeec0..7e3493d6 100644 --- a/tests/settings.py +++ b/tests/openwisp2/settings.py @@ -46,7 +46,7 @@ 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] -ROOT_URLCONF = 'urls' +ROOT_URLCONF = 'openwisp2.urls' LANGUAGE_CODE = 'en-us' TIME_ZONE = 'UTC' diff --git a/tests/urls.py b/tests/openwisp2/urls.py similarity index 100% rename from tests/urls.py rename to tests/openwisp2/urls.py diff --git a/tests/test_project/tests/test_admin.py b/tests/test_project/tests/test_admin.py index 50e057d4..74eb4e94 100644 --- a/tests/test_project/tests/test_admin.py +++ b/tests/test_project/tests/test_admin.py @@ -569,3 +569,33 @@ def test_organization_radius_settings_admin(self): self.assertEqual( org_rad_settings.get_field_value('extra_config'), 'no data' ) + + @patch( + 'openwisp_utils.admin_theme.system_info.settings.INSTALLED_APPS', + ['openwisp_users', 'openwisp_utils.admin_theme'], + ) + def test_system_information(self, *args): + def _assert_system_information(response): + self.assertContains(response, '
  • openwisp-utils:') + self.assertContains(response, '
  • netjsonconfig:') + self.assertContains(response, '

    OS Information

    ') + self.assertContains(response, 'OS version:') + self.assertContains(response, 'Kernel version:') + self.assertContains(response, 'Hardware platform:') + + with self.subTest('Test openwisp version is defined'): + with patch('openwisp2.__openwisp_version__', '23.0.0'): + response = self.client.get(reverse('admin:ow-info')) + self.assertEqual(response.status_code, 200) + self.assertContains(response, '

    OpenWISP Version: 23.0.0

    ') + _assert_system_information(response) + + with self.subTest('Test openwisp version is not defined'): + with patch( + 'openwisp_utils.admin_theme.system_info.import_string', + side_effect=ImportError, + ): + response = self.client.get(reverse('admin:ow-info')) + self.assertEqual(response.status_code, 200) + self.assertNotContains(response, '

    OpenWISP Version') + _assert_system_information(response) From 828ced9037faf066b34cb8a8605535c9faaa2644 Mon Sep 17 00:00:00 2001 From: Gagan Deep Date: Tue, 5 Dec 2023 23:09:12 +0530 Subject: [PATCH 12/20] [fix] Fixed menu URL for system info #237 Related to #237 --- openwisp_utils/admin_theme/apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwisp_utils/admin_theme/apps.py b/openwisp_utils/admin_theme/apps.py index 9793a006..261aed6f 100644 --- a/openwisp_utils/admin_theme/apps.py +++ b/openwisp_utils/admin_theme/apps.py @@ -55,7 +55,7 @@ def register_menu_groups(self): position=899, config={ 'label': _('System info'), - 'url': '/admin/ow-info', + 'url': '/admin/openwisp-system-info/', 'icon': 'ow-info-icon', }, ) From e177b7d0fc4e16038baeff5a6dd2cb8105cb9c5c Mon Sep 17 00:00:00 2001 From: Prapti Sharma Date: Mon, 11 Dec 2023 18:25:33 +0530 Subject: [PATCH 13/20] [chores] Removed travis-ci skip commit logic #163 Closes #163 --- openwisp-qa-check | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/openwisp-qa-check b/openwisp-qa-check index c4388554..93fd3d8b 100755 --- a/openwisp-qa-check +++ b/openwisp-qa-check @@ -149,18 +149,14 @@ runblack() { runcheckcommit() { if [ -z "$COMMIT_MESSAGE" ]; then COMMIT_MESSAGE=$(git log -1 --pretty=%B); fi - if [ "$TRAVIS" = true ] && [ "$TRAVIS_PULL_REQUEST" = false ]; then - echo "SKIPPED: Commit message check skipped!" - else - checkcommit --message "$COMMIT_MESSAGE" && - echo "SUCCESS: Commit message check successful!" || - { - echo -e "Checked commit message:\n\n" - echo -e "$COMMIT_MESSAGE\n\n" - echoerr "ERROR: Commit message check failed!" - FAILURE=1 - } - fi + + checkcommit --message "$COMMIT_MESSAGE" && + echo "SUCCESS: Commit message check successful!" || + { + echo -e "Checked commit message:\n\n$COMMIT_MESSAGE\n\n" + echoerr "ERROR: Commit message check failed!" + FAILURE=1 + } } runcheckpendingmigrations() { From 2196a37a1116dc7a84633152a4c6d2f41e871480 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Fri, 16 Feb 2024 18:17:30 -0300 Subject: [PATCH 14/20] [chores] Fixed stylelint issues and ran prettier --- .stylelintrc.json | 6 - .../admin_theme/static/admin/css/openwisp.css | 244 +++++++++++------- .../static/admin/css/ow-auto-filter.css | 12 +- .../static/admin/css/ow-dashboard.css | 11 +- .../static/admin/css/ow-filters.css | 19 +- .../static/drf-yasg/ow-drf-yasg.css | 71 +++-- .../rest_framework/css/browsable-api.css | 52 ++-- 7 files changed, 251 insertions(+), 164 deletions(-) diff --git a/.stylelintrc.json b/.stylelintrc.json index 517d1d88..54c467e9 100644 --- a/.stylelintrc.json +++ b/.stylelintrc.json @@ -5,17 +5,11 @@ "comment-empty-line-before": ["always", { "ignore": ["stylelint-commands", "after-comment"] }], - "declaration-colon-space-after": "always", - "indentation": [2, { - "except": ["value"] - }], - "max-empty-lines": 4, "rule-empty-line-before": ["never-multi-line", { "except": ["first-nested"], "ignore": ["after-comment", "inside-block"] }], "unit-allowed-list": ["em", "rem", "%", "s", "px", "vh", "deg", "dpi", "fr"], - "declaration-block-trailing-semicolon": "always", "property-no-unknown": true } } diff --git a/openwisp_utils/admin_theme/static/admin/css/openwisp.css b/openwisp_utils/admin_theme/static/admin/css/openwisp.css index dcf71a07..b9558d3b 100644 --- a/openwisp_utils/admin_theme/static/admin/css/openwisp.css +++ b/openwisp_utils/admin_theme/static/admin/css/openwisp.css @@ -2,22 +2,22 @@ See https://code.djangoproject.com/ticket/33878 */ @font-face { - font-family: 'Roboto'; - src: url('../fonts/Roboto-Bold-webfont.woff'); + font-family: "Roboto"; + src: url("../fonts/Roboto-Bold-webfont.woff"); font-weight: 700; font-style: normal; } @font-face { - font-family: 'Roboto'; - src: url('../fonts/Roboto-Regular-webfont.woff'); + font-family: "Roboto"; + src: url("../fonts/Roboto-Regular-webfont.woff"); font-weight: 400; font-style: normal; } @font-face { - font-family: 'Roboto'; - src: url('../fonts/Roboto-Light-webfont.woff'); + font-family: "Roboto"; + src: url("../fonts/Roboto-Light-webfont.woff"); font-weight: 300; font-style: normal; } @@ -73,7 +73,8 @@ See https://code.djangoproject.com/ticket/33878 */ --object-tools-fg: var(--button-fg) !important; --object-tools-bg: var(--close-button-bg) !important; --object-tools-hover-bg: var(--close-button-hover-bg) !important; - --font-family-primary: "Roboto","Lucida Grande","DejaVu Sans","Bitstream Vera Sans",Verdana,Arial,sans-serif; + --font-family-primary: "Roboto", "Lucida Grande", "DejaVu Sans", + "Bitstream Vera Sans", Verdana, Arial, sans-serif; } body { min-width: 320px; @@ -84,7 +85,7 @@ body { width: 100%; top: 0; left: 0; - background: rgba(0, 0, 0, 0.10); + background: rgba(0, 0, 0, 0.1); z-index: 9999; display: none; } @@ -94,16 +95,20 @@ body { } #main-content .breadcrumbs, .module caption, -div[class^="app-"].module th, div[class^="app-"].module td { +div[class^="app-"].module th, +div[class^="app-"].module td { padding: 1rem 1.3rem; } -div[class^="app-"].module td { padding-left: 0; } +div[class^="app-"].module td { + padding-left: 0; +} #main-content .results thead th .text a, -#main-content thead th { +#main-content thead th { padding: 15px 10px; font-size: 13.5px; } -#main-content .results tbody td, #main-content .results tbody th, +#main-content .results tbody td, +#main-content .results tbody th, div#main-content table tbody th, div#main-content table tbody td { padding: 1rem 10px; @@ -111,15 +116,16 @@ div#main-content table tbody td { font-size: 14px; } -#main-content .breadcrumbs { font-size: 15px; } +#main-content .breadcrumbs { + font-size: 15px; +} #main-content .breadcrumbs a, a.section:link, a.section:visited, .module th a, #main .help a, fieldset.collapsed a.collapse-toggle, -fieldset a.collapse-toggle - { +fieldset a.collapse-toggle { color: #df5d43; text-decoration: none; } @@ -182,17 +188,22 @@ a:focus { line-height: 20px; color: rgba(0, 0, 0, 0.45); } -.addlink, .changelink { +.addlink, +.changelink { font-weight: bold; } -.changelink { padding-left: 17px; } -.addlink:hover, .changelink:hover { +.changelink { + padding-left: 17px; +} +.addlink:hover, +.changelink:hover { color: #000; } /* increase font size and spacing */ -.module caption, .inline-group h2, +.module caption, +.inline-group h2, #content-related h3, fieldset.module h2 { font-size: 16px; @@ -202,23 +213,38 @@ fieldset.module h2 { font-size: 15px; background: #f9f9f9; } -td, th, li { +td, +th, +li { font-size: 14px; } -.mini { font-size: 12px; } +.mini { + font-size: 12px; +} #main .actionlist li { line-height: 20px; background-position: 0px 4px; } #main .aligned label, -input, textarea, select, .form-row p, form .button, +input, +textarea, +select, +.form-row p, +form .button, .readonly, -#main .form-row { font-size: 15px; } -select:not([disabled]):not(.readonly) { background: #fff; } -input[type=text], -input[type=password], input[type=email], -input[type=url], input[type=number], -input[type=tel], textarea, +#main .form-row { + font-size: 15px; +} +select:not([disabled]):not(.readonly) { + background: #fff; +} +input[type="text"], +input[type="password"], +input[type="email"], +input[type="url"], +input[type="number"], +input[type="tel"], +textarea, #main .vTextField { padding: 8px 12px; box-sizing: border-box; @@ -230,7 +256,10 @@ input.readonly { border: 1px solid rgba(0, 0, 0, 0.05) !important; background-color: rgba(0, 0, 0, 0.07); } -select { height: 36px; padding: 0 12px; } +select { + height: 36px; + padding: 0 12px; +} #main span.select2-selection { min-height: 36px; min-width: 320px; @@ -264,22 +293,35 @@ select { height: 36px; padding: 0 12px; } .select2-selection__clear { left: -27px; } -.related-widget-wrapper .change-related { margin-left: 7px; } -#main .form-row { padding: 15px; box-sizing: border-box; } -#main .inline-group .tabular td.original p { height: auto; } +.related-widget-wrapper .change-related { + margin-left: 7px; +} +#main .form-row { + padding: 15px; + box-sizing: border-box; +} +#main .inline-group .tabular td.original p { + height: auto; +} #main .inline-group thead th { background: transparent; } #main .inline-group thead th:not(.original), -#main .add-row td , +#main .add-row td, #main div.add-row { padding-top: 15px; padding-bottom: 15px; } #main .add-row td a, -#main div.add-row a { font-size: 15px !important; } -#main .delete input { vertical-align: -1px; } -#main .delete label { font-size: 14px; } +#main div.add-row a { + font-size: 15px !important; +} +#main .delete input { + vertical-align: -1px; +} +#main .delete label { + font-size: 14px; +} ul.messagelist li { padding: 15px 10px 15px 65px; background-position: 40px 17px; @@ -287,7 +329,7 @@ ul.messagelist li { /* hide first delete link, hide save and add anoher */ #main .submit-row:first-child .deletelink-box, -.submit-row input[name='_addanother'] { +.submit-row input[name="_addanother"] { display: none; } @@ -435,7 +477,7 @@ span.datetimeshortcuts .clock-icon:hover { /* For danger,error, red button [ class="danger-btn" ] */ .danger-btn, div.submit-row a.deletelink, -.delete-confirmation form input[type='submit'] { +.delete-confirmation form input[type="submit"] { display: inline-block; text-align: center; border-radius: 4px; @@ -455,25 +497,27 @@ div.submit-row a.deletelink { div.submit-row a.deletelink:hover, div.submit-row a.deletelink:focus, div.submit-row a.deletelink:active, -.delete-confirmation form input[type='submit']:hover, -.delete-confirmation form input[type='submit']:active, -.delete-confirmation form input[type='submit']:focus { +.delete-confirmation form input[type="submit"]:hover, +.delete-confirmation form input[type="submit"]:active, +.delete-confirmation form input[type="submit"]:focus { background: #b31e00; outline: 0 none; } -#content .submit-row p.deletelink-box { margin: 0; } +#content .submit-row p.deletelink-box { + margin: 0; +} /* For any normal, default, dark button [ class="default-btn" ] */ .default-btn, .button.default, -input[type='submit'].default, +input[type="submit"].default, .object-tools a, .object-tools a:link, .object-tools a:visited, .button, -input[type='submit'], -input[type='button'], +input[type="submit"], +input[type="button"], .submit-row input, .delete-confirmation form .cancel-link, a.button, @@ -492,7 +536,7 @@ a.button, div#toolbar { padding: 0.8rem 1.2rem; } -div#toolbar form input[type='submit'] { +div#toolbar form input[type="submit"] { margin: 0 10px 0 3px; padding: 0.47rem 1rem; background: #333; @@ -520,7 +564,7 @@ div#changelist .actions .button { height: unset; } div#changelist .actions .button:hover, -div#toolbar form input[type='submit']:hover { +div#toolbar form input[type="submit"]:hover { opacity: 0.7; border-color: #333; } @@ -545,9 +589,9 @@ div#toolbar form input[type='submit']:hover { .button.default:hover, .button.default:active, .button.default:focus, -input[type='submit'].default:hover, -input[type='submit'].default:active, -input[type='submit'].default:focus, +input[type="submit"].default:hover, +input[type="submit"].default:active, +input[type="submit"].default:focus, .default-btn:focus, .object-tools a:hover, .object-tools a:active, @@ -555,26 +599,31 @@ input[type='submit'].default:focus, .object-tools button:hover, .object-tools button:focus, .button:active, -input[type='submit']:active, -input[type='button']:active, +input[type="submit"]:active, +input[type="button"]:active, .button:focus, -input[type='submit']:focus, -input[type='button']:focus, +input[type="submit"]:focus, +input[type="button"]:focus, .button:hover, -input[type='submit']:hover, -input[type='button']:hover, +input[type="submit"]:hover, +input[type="button"]:hover, .delete-confirmation form .cancel-link:hover { outline: 0 none; background: rgba(51, 51, 51, 0.7); } -#main .related-lookup { padding-left: 20px; width: auto; } +#main .related-lookup { + padding-left: 20px; + width: auto; +} .form-row .item-label { position: relative; bottom: -2px; margin-left: 2px; font-weight: bold; } -#main label.vCheckboxLabel { width: auto; } +#main label.vCheckboxLabel { + width: auto; +} /* For success, green button [ class="success-btn" ] */ .success-btn, @@ -758,7 +807,7 @@ input[type='button']:hover, width: 150px; height: 86px; background: url(../../../static/ui/openwisp/images/openwisp-logo-black.svg) - no-repeat scroll 0 50%/100%; + no-repeat scroll 0 50%/100%; } /* Fix for object tools */ @@ -768,11 +817,12 @@ input[type='button']:hover, #content-main .form-row a:link { text-decoration: underline; } -#content-main .form-row a:link:hover, #content-main a:link:focus { +#content-main .form-row a:link:hover, +#content-main a:link:focus { text-decoration: none; } .title-wrapper h1, -.title-wrapper h2{ +.title-wrapper h2 { display: inline-block; margin: 0px; margin-bottom: 20px; @@ -787,14 +837,14 @@ input[type='button']:hover, .title-wrapper h2::after { content: ")"; } -.title-wrapper .object-tools{ +.title-wrapper .object-tools { margin-top: -4px; margin-bottom: 20px; } .title-wrapper .object-tools li { height: auto; } -#changelist .paginator{ +#changelist .paginator { box-sizing: border-box; font-size: 14px; } @@ -868,51 +918,53 @@ p.calendar-cancel { font-size: 14px; line-height: 20px; } -div.calendar caption, div.calendarbox h2{ +div.calendar caption, +div.calendarbox h2 { background: #df5d43; color: #fff; } -div.calendar caption{ +div.calendar caption { padding: 0.9rem 10px; } .calendarbox a.calendarnav-previous, -.calendarbox a.calendarnav-next{ +.calendarbox a.calendarnav-next { background: rgb(255 255 255 / 75%); mask-size: contain !important; -webkit-mask-size: contain !important; top: 15px; } -div.calendarbox, div.clockbox{ +div.calendarbox, +div.clockbox { width: auto; background-color: #efefef; border-radius: 8px; box-shadow: 0px 1px 3px #00000040; } -div.calendar th{ +div.calendar th { padding: 10px; } -div.calendar td a{ +div.calendar td a { padding: 10px; } -ul.timelist a{ +ul.timelist a { padding: 8px 0px; } div.calendar td a:hover, div.calendar td.selected a, -ul.timelist a:hover{ +ul.timelist a:hover { background: #ffe5e5; color: #df5d43; } -.calendarbox a.calendarnav-previous{ +.calendarbox a.calendarnav-previous { mask: url(../../ui/openwisp/images/calender-prev.svg) no-repeat center; -webkit-mask: url(../../ui/openwisp/images/calender-prev.svg) no-repeat center; } -.calendarbox a.calendarnav-next{ +.calendarbox a.calendarnav-next { mask: url(../../ui/openwisp/images/calender-next.svg) no-repeat center; -webkit-mask: url(../../ui/openwisp/images/calender-next.svg) no-repeat center; } .calendarbox a.calendarnav-previous, -.calendarbox a.calendarnav-next{ +.calendarbox a.calendarnav-next { padding: 4px; top: 13px; margin: 0 3px; @@ -923,16 +975,16 @@ ul.timelist, div.calendarbox .calendarnav-next:focus, div.calendarbox .calendarnav-next:hover, .calendarbox .calendarnav-previous:focus, -.calendarbox .calendarnav-previous:hover{ +.calendarbox .calendarnav-previous:hover { background: #fff; } -div.calendar-shortcuts{ +div.calendar-shortcuts { padding: 10px; } p.calendar-cancel { padding: 10px; } -div.calendar-shortcuts a:hover{ +div.calendar-shortcuts a:hover { color: #df5d43; } @@ -974,7 +1026,7 @@ div.calendar-shortcuts a:hover{ text-align: center; } .title-wrapper h1, - .title-wrapper h2{ + .title-wrapper h2 { margin-bottom: 15px; display: block; } @@ -989,13 +1041,16 @@ div.calendar-shortcuts a:hover{ } @media (max-width: 1226px) and (min-width: 1024px) { - /*********** Selector fix ***********/ .related-widget-wrapper > a { order: 2; } - div.selector{display: flex;flex-direction: column;align-items: center;} - div.related-widget-wrapper{ + div.selector { + display: flex; + flex-direction: column; + align-items: center; + } + div.related-widget-wrapper { display: flex; align-items: flex-start; width: 100%; @@ -1010,7 +1065,8 @@ div.calendar-shortcuts a:hover{ #container.toggle-menu.no-auth { flex-direction: column; } -div#container.no-auth, .popup div#container{ +div#container.no-auth, +.popup div#container { display: flex; } #container.no-auth #menu { @@ -1021,7 +1077,6 @@ div#container.no-auth, .popup div#container{ min-height: unset; } #container.no-auth #main-content { - /* width: auto; */ margin-left: 0; } @@ -1079,7 +1134,7 @@ a.menu-item, .mg-head:focus, #menu .mg-link:focus, .select2-results__option--highlighted[aria-selected], -.select2-results__option[aria-selected='true'] { +.select2-results__option[aria-selected="true"] { background: #ffe5e5 !important; color: #df5d43 !important; } @@ -1178,7 +1233,7 @@ h1#site-name { height: 95px; margin: auto; background: url(../../../static/ui/openwisp/images/openwisp-logo-black.svg) - no-repeat scroll 0 50%/100%; + no-repeat scroll 0 50%/100%; text-indent: -2000px; } #main-content.m-0 { @@ -1339,7 +1394,6 @@ h1#site-name { pointer-events: all; transition: top 0.3s ease; } - } /****************** MENU(screen-size > 1024px) ******************/ @@ -1391,7 +1445,7 @@ h1#site-name { width: 2.5rem; height: 2.5rem; background: url(../../../static/ui/openwisp/images/openwisp-logo-small.svg) - no-repeat center; + no-repeat center; } .hamburger { padding: 0.813rem 1rem; @@ -1424,7 +1478,7 @@ h1#site-name { .toggle-menu #menu { height: 3rem; } - .toggle-menu .nav{ + .toggle-menu .nav { display: none; } } @@ -1460,7 +1514,7 @@ body.login { } .login #site-name a { background: url(../../../static/ui/openwisp/images/openwisp-logo-black.svg) - no-repeat scroll 0 50%/100%; + no-repeat scroll 0 50%/100%; text-indent: -2000px; width: 150px; height: 86px; @@ -1527,7 +1581,8 @@ body::-webkit-scrollbar { mask-image: url(../../ui/openwisp/images/help.svg); -webkit-mask-image: url(../../ui/openwisp/images/help.svg); } -#menu .icon.None, #menu .icon.None:hover { +#menu .icon.None, +#menu .icon.None:hover { background: transparent; } @@ -1702,23 +1757,22 @@ body::-webkit-scrollbar { mask-image: url(../../ui/openwisp/images/network-topology.svg); -webkit-mask-image: url(../../ui/openwisp/images/network-topology.svg); } -.ow-link{ +.ow-link { mask-image: url(../../ui/openwisp/images/link.svg); -webkit-mask-image: url(../../ui/openwisp/images/link.svg); } -.ow-topology{ +.ow-topology { mask-image: url(../../ui/openwisp/images/topology.svg); -webkit-mask-image: url(../../ui/openwisp/images/topology.svg); } -.ow-node{ +.ow-node { mask-image: url(../../ui/openwisp/images/node.svg); -webkit-mask-image: url(../../ui/openwisp/images/node.svg); } /* LOGIN SCREEN */ @media only screen and (max-width: 576px) { - - .login #container { + .login #container { width: auto !important; } } diff --git a/openwisp_utils/admin_theme/static/admin/css/ow-auto-filter.css b/openwisp_utils/admin_theme/static/admin/css/ow-auto-filter.css index 2b82b99c..96d1d65e 100644 --- a/openwisp_utils/admin_theme/static/admin/css/ow-auto-filter.css +++ b/openwisp_utils/admin_theme/static/admin/css/ow-auto-filter.css @@ -45,10 +45,12 @@ border-radius: 0.2rem !important; padding: 0.4rem 1.25rem 0.3125rem 1rem; } -.auto-filter .select2-selection[aria-expanded='true'] { +.auto-filter .select2-selection[aria-expanded="true"] { border-color: #df5d43 !important; } -.auto-filter .select2-selection[aria-expanded='true'] .select2-selection__rendered { +.auto-filter + .select2-selection[aria-expanded="true"] + .select2-selection__rendered { color: #df5d43 !important; } .auto-filter .select2-selection__placeholder { @@ -88,9 +90,11 @@ font-size: 14px; border-radius: 4px; font-weight: 500; - transition: color 0.15s, background 0.15s; + transition: + color 0.15s, + background 0.15s; } -.auto-filter-choices .select2-results__option[aria-selected='true'] { +.auto-filter-choices .select2-results__option[aria-selected="true"] { background: #df5d43 !important; color: #ffff !important; } diff --git a/openwisp_utils/admin_theme/static/admin/css/ow-dashboard.css b/openwisp_utils/admin_theme/static/admin/css/ow-dashboard.css index 72d825f3..a36fbef7 100644 --- a/openwisp_utils/admin_theme/static/admin/css/ow-dashboard.css +++ b/openwisp_utils/admin_theme/static/admin/css/ow-dashboard.css @@ -25,29 +25,28 @@ .quick-link-container .button.quick-link { padding: 0.3rem 1rem; } -.dashboard:not([class*='app-']) #container #content { +.dashboard:not([class*="app-"]) #container #content { margin: 0 auto; width: auto; } -.dashboard:not([class*='app-']) #container #content-main { +.dashboard:not([class*="app-"]) #container #content-main { width: 63%; } -.dashboard:not([class*='app-']) #container #content-related { +.dashboard:not([class*="app-"]) #container #content-related { width: 35%; margin-left: 2%; margin-right: 0; } @media (max-width: 767px) { - .dashboard:not([class*='app-']) #container #content-main, - .dashboard:not([class*='app-']) #container #content-related { + .dashboard:not([class*="app-"]) #container #content-main, + .dashboard:not([class*="app-"]) #container #content-related { width: 100% !important; margin-left: 0; } } @media only screen and (max-width: 576px) { - #plot-container { display: flex; align-items: center; diff --git a/openwisp_utils/admin_theme/static/admin/css/ow-filters.css b/openwisp_utils/admin_theme/static/admin/css/ow-filters.css index 8abd0841..4bf0eafa 100644 --- a/openwisp_utils/admin_theme/static/admin/css/ow-filters.css +++ b/openwisp_utils/admin_theme/static/admin/css/ow-filters.css @@ -41,17 +41,17 @@ .filter { margin-right: 0.625rem; } -.ow-input-filter .filter-options{ +.ow-input-filter .filter-options { display: none !important; } -.ow-input-filter .filter-title{ +.ow-input-filter .filter-title { padding: 0.4rem 0rem 0.3125rem 0rem; min-width: 13.5rem; } -.ow-input-filter .filter-title h3{ +.ow-input-filter .filter-title h3 { margin-left: 8px; } -.ow-input-filter .ow-input-filter-field{ +.ow-input-filter .ow-input-filter-field { padding: 0.4rem 2.25rem 0.3125rem 1rem; position: absolute; height: 100%; @@ -87,11 +87,11 @@ display: none; overflow-y: auto; } -.filter-options ul{ +.filter-options ul { margin: 0px; padding: 0px; } -.filter-options ul li{ +.filter-options ul li { list-style: none; margin: 0px; padding: 0px; @@ -158,7 +158,8 @@ opacity: 1; margin-top: 3px; } -.filter-options .selected,.filter-options .selected:hover, +.filter-options .selected, +.filter-options .selected:hover, .filter-options .selected:focus { background: #df5d43; color: #ffff; @@ -219,7 +220,7 @@ .right-arrow.down { background: rgb(185 185 185 / 20%) !important; } -.disabled-click{ +.disabled-click { pointer-events: none; } a.field-clear { @@ -235,7 +236,7 @@ a.field-clear { #ow-changelist-filter { padding: 1.25rem 1.875rem; } - .filters-bottom{ + .filters-bottom { margin: auto 30px; margin-top: 1.25rem; } diff --git a/openwisp_utils/admin_theme/static/drf-yasg/ow-drf-yasg.css b/openwisp_utils/admin_theme/static/drf-yasg/ow-drf-yasg.css index 971f6190..49896df6 100644 --- a/openwisp_utils/admin_theme/static/drf-yasg/ow-drf-yasg.css +++ b/openwisp_utils/admin_theme/static/drf-yasg/ow-drf-yasg.css @@ -20,9 +20,8 @@ body::-webkit-scrollbar { background: #333; } - /* LOGO */ -#swagger-ui > section > div.topbar > div > div > a > img{ +#swagger-ui > section > div.topbar > div > div > a > img { content: url(../ui/openwisp/images/openwisp-logo-black.svg); width: 165px; height: 95px; @@ -34,34 +33,34 @@ body::-webkit-scrollbar { background-color: #f9f9f9; border-bottom: 1px solid #eee; } -.swagger-ui .topbar .download-url-wrapper input[type=text] { +.swagger-ui .topbar .download-url-wrapper input[type="text"] { border-color: rgb(0, 0, 0); } -.swagger-ui .topbar .download-url-wrapper input[type=text]:focus{ +.swagger-ui .topbar .download-url-wrapper input[type="text"]:focus { border: 3px solid rgb(0, 0, 0); } /* EXPLORE BTN */ -.swagger-ui .topbar .download-url-wrapper .download-url-button{ +.swagger-ui .topbar .download-url-wrapper .download-url-button { background: #333; font-size: 15px; font-weight: normal; text-transform: uppercase; } -.swagger-ui .topbar .download-url-wrapper .download-url-button:hover{ +.swagger-ui .topbar .download-url-wrapper .download-url-button:hover { background: #999; } /* SCHEMES */ -.swagger-ui select{ +.swagger-ui select { border-color: #ccc; } .swagger-ui .info a { color: #df5d43; } .swagger-ui .info a:hover, -.swagger-ui .info a:visited{ - color: #D32F2F; +.swagger-ui .info a:visited { + color: #d32f2f; } /* AUTH LABEL */ @@ -77,11 +76,11 @@ body::-webkit-scrollbar { } /* LOGOUT BTN */ -#django-session-auth > div.btn.authorize{ +#django-session-auth > div.btn.authorize { background-color: #333; border: 0; } -#django-session-auth > div.btn.authorize:hover{ +#django-session-auth > div.btn.authorize:hover { background-color: #999; } .swagger-ui .btn.authorize svg { @@ -102,7 +101,7 @@ body::-webkit-scrollbar { } /* LOGOUT & CLOSE BTN */ -.swagger-ui .btn{ +.swagger-ui .btn { color: #fff; border: 0; background-color: #333; @@ -111,27 +110,28 @@ body::-webkit-scrollbar { text-transform: uppercase; font-weight: normal; margin: 0 5px; - font-family: "Roboto","Lucida Grande","DejaVu Sans","Bitstream Vera Sans",Verdana,Arial,sans-seri; + font-family: "Roboto", "Lucida Grande", "DejaVu Sans", "Bitstream Vera Sans", + Verdana, Arial, sans-seri; } -.swagger-ui .btn:hover{ +.swagger-ui .btn:hover { background-color: #999; } /* OPBLOCK */ -.swagger-ui .opblock-tag.no-desc{ +.swagger-ui .opblock-tag.no-desc { border: 1px solid rgba(0, 0, 0, 0.1); padding-left: 26px; } -.swagger-ui .opblock-tag.no-desc span:hover{ +.swagger-ui .opblock-tag.no-desc span:hover { background: #fdf2f2f2; color: #df5d43; } -.swagger-ui .opblock-tag:hover .nostyle > span{ +.swagger-ui .opblock-tag:hover .nostyle > span { background-color: #fdf2f2f2; color: #df5d43; } -.swagger-ui .opblock-tag:hover{ +.swagger-ui .opblock-tag:hover { background-color: #fdf2f2f2; } .swagger-ui .download-contents { @@ -151,7 +151,7 @@ body::-webkit-scrollbar { color: #fff; background-color: #333; } -.swagger-ui .btn.cancel:hover{ +.swagger-ui .btn.cancel:hover { background-color: #999; } .swagger-ui .btn.execute { @@ -159,7 +159,7 @@ body::-webkit-scrollbar { color: #fff; background-color: #333; } -.swagger-ui .btn.execute:hover{ +.swagger-ui .btn.execute:hover { background-color: #999; } @@ -168,27 +168,46 @@ body::-webkit-scrollbar { .swagger-ui section.models .model-container { background: #f6f6f9; } -.swagger-ui section.models .model-container:hover{ +.swagger-ui section.models .model-container:hover { background: #fdf2f2f2; color: #df5d43; } -.swagger-ui .model-title:hover{ +.swagger-ui .model-title:hover { color: #df5d43; } -.swagger-ui section.models .model-container.is-open:hover{ +.swagger-ui section.models .model-container.is-open:hover { background: #fdf2f2f2; color: #df5d43; } .swagger-ui .prop-type { color: #df5d43; } -.swagger-ui section.models .model-container:hover .model-box>.pointer>.model-box>.model-title{ +.swagger-ui + section.models + .model-container:hover + .model-box + > .pointer + > .model-box + > .model-title { color: #df5d43; } -.swagger-ui section.models .model-container:hover .model-box>.pointer>.model-toggle .collapsed{ +.swagger-ui + section.models + .model-container:hover + .model-box + > .pointer + > .model-toggle + .collapsed { color: #df5d43; } -#swagger-ui > section > div.swagger-ui > div > div > section > section > h4:hover{ +#swagger-ui + > section + > div.swagger-ui + > div + > div + > section + > section + > h4:hover { background: #fdf2f2f2; color: #df5d43; } diff --git a/openwisp_utils/admin_theme/static/rest_framework/css/browsable-api.css b/openwisp_utils/admin_theme/static/rest_framework/css/browsable-api.css index 74f33d64..76d64c4a 100644 --- a/openwisp_utils/admin_theme/static/rest_framework/css/browsable-api.css +++ b/openwisp_utils/admin_theme/static/rest_framework/css/browsable-api.css @@ -35,7 +35,7 @@ body::-webkit-scrollbar { ::-webkit-scrollbar-thumb:hover { background: #333; } -body{ +body { background: none; background-attachment: initial; } @@ -48,7 +48,8 @@ body{ position: static; border-bottom: 1px solid #eee; } -.navbar .nav li, .navbar .nav li a{ +.navbar .nav li, +.navbar .nav li a { margin-top: 30px; color: #000; background-color: rgba(0, 0, 0, 0.1); @@ -57,13 +58,13 @@ body{ text-align: center; border-radius: 0.25em; } -.navbar-brand>img{ +.navbar-brand > img { width: 165px; height: 95px; margin-left: 15px; margin-bottom: 100px; } -.navbar-brand{ +.navbar-brand { width: 185px; height: 105px; margin: 0; @@ -72,16 +73,20 @@ body{ ul.breadcrumb { margin: 90px 0 0 0; } -ul.breadcrumb{ +ul.breadcrumb { display: none; } -.kwd, .prettyprint .tag { +.kwd, +.prettyprint .tag { color: #333; } .lit { color: #000; } -.pun, .opn, .clo, .pln { +.pun, +.opn, +.clo, +.pln { color: #333; } .prettyprint { @@ -98,16 +103,19 @@ ul.breadcrumb{ background-color: #999; border: 0; } -.btn-primary.active, .btn-primary:active, .open>.dropdown-toggle.btn-primary { +.btn-primary.active, +.btn-primary:active, +.open > .dropdown-toggle.btn-primary { color: #fff; background-color: #333 !important; background-image: none; border: 0; } -.dropdown-menu>li>a { +.dropdown-menu > li > a { color: #777; } -.dropdown-menu>li:hover, .dropdown-menu>li>a:hover{ +.dropdown-menu > li:hover, +.dropdown-menu > li > a:hover { color: #df5d43; background: #fdf2f2f2; } @@ -116,14 +124,16 @@ ul.breadcrumb{ color: #777; font-weight: bold; } -.nav-tabs li a:visited{ +.nav-tabs li a:visited { color: #777; } -.nav-tabs li a:hover{ +.nav-tabs li a:hover { color: #df5d43; background-color: #f9f9f9; } -.nav-tabs>li.active>a, .nav-tabs>li.active>a:focus, .nav-tabs>li.active>a:hover { +.nav-tabs > li.active > a, +.nav-tabs > li.active > a:focus, +.nav-tabs > li.active > a:hover { color: #df5d43; cursor: default; font-weight: bold; @@ -135,20 +145,26 @@ ul.breadcrumb{ border: 2px solid rgb(0, 0, 0); box-shadow: none; } -.form-group label{ +.form-group label { font-weight: normal; color: #000; } -.pagination>li>a, .pagination>li>span{ +.pagination > li > a, +.pagination > li > span { color: #777; } -.pagination>li>a:hover, .pagination>li>span:hover{ +.pagination > li > a:hover, +.pagination > li > span:hover { color: #df5d43; background-color: #ffe5e5; border-color: #ffd2c9; } -.pagination>.active>a, .pagination>.active>a:focus, .pagination>.active>a:hover, -.pagination>.active>span, .pagination>.active>span:focus, .pagination>.active>span:hover { +.pagination > .active > a, +.pagination > .active > a:focus, +.pagination > .active > a:hover, +.pagination > .active > span, +.pagination > .active > span:focus, +.pagination > .active > span:hover { color: #df5d43; background-color: #ffe5e5; border-color: #ffd2c9; From 7cb292978121b7b7c4d607a2e5961e5978b7903a Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Fri, 16 Feb 2024 18:23:28 -0300 Subject: [PATCH 15/20] [qa] Minor qa fixes in admin css --- openwisp_utils/admin_theme/static/admin/css/openwisp.css | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openwisp_utils/admin_theme/static/admin/css/openwisp.css b/openwisp_utils/admin_theme/static/admin/css/openwisp.css index b9558d3b..a8a275bb 100644 --- a/openwisp_utils/admin_theme/static/admin/css/openwisp.css +++ b/openwisp_utils/admin_theme/static/admin/css/openwisp.css @@ -115,7 +115,6 @@ div#main-content table tbody td { vertical-align: middle; font-size: 14px; } - #main-content .breadcrumbs { font-size: 15px; } @@ -1040,8 +1039,8 @@ div.calendar-shortcuts a:hover { } } +/*********** Selector fix ***********/ @media (max-width: 1226px) and (min-width: 1024px) { - /*********** Selector fix ***********/ .related-widget-wrapper > a { order: 2; } @@ -1077,7 +1076,6 @@ div#container.no-auth, min-height: unset; } #container.no-auth #main-content { - /* width: auto; */ margin-left: 0; } #container.no-auth #site-name { From 8216bb7df6320f8150036c77a7989e718f9f9c1c Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Mon, 19 Feb 2024 16:30:30 -0300 Subject: [PATCH 16/20] [chores:ui] Avoid resizing of cancel button in confirmation pages --- openwisp_utils/admin_theme/static/admin/css/openwisp.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openwisp_utils/admin_theme/static/admin/css/openwisp.css b/openwisp_utils/admin_theme/static/admin/css/openwisp.css index a8a275bb..6edd44b6 100644 --- a/openwisp_utils/admin_theme/static/admin/css/openwisp.css +++ b/openwisp_utils/admin_theme/static/admin/css/openwisp.css @@ -532,6 +532,11 @@ a.button, font-weight: 500; font-size: 15px; } +.delete-confirmation form .cancel-link, +.delete-confirmation input[type="submit"] { + height: auto !important; + line-height: inherit !important; +} div#toolbar { padding: 0.8rem 1.2rem; } From 7460f2a1586001c34caed561c99f01009e7d11d6 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Mon, 19 Feb 2024 18:09:53 -0300 Subject: [PATCH 17/20] [chores:ui] Fixed messagelist padding --- openwisp_utils/admin_theme/static/admin/css/openwisp.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwisp_utils/admin_theme/static/admin/css/openwisp.css b/openwisp_utils/admin_theme/static/admin/css/openwisp.css index 6edd44b6..0ff27812 100644 --- a/openwisp_utils/admin_theme/static/admin/css/openwisp.css +++ b/openwisp_utils/admin_theme/static/admin/css/openwisp.css @@ -321,7 +321,7 @@ select { #main .delete label { font-size: 14px; } -ul.messagelist li { +#main ul.messagelist li { padding: 15px 10px 15px 65px; background-position: 40px 17px; } From 66cce4613c95ee6ab35796aa2b824b2b5ebe230f Mon Sep 17 00:00:00 2001 From: Gagan Deep Date: Tue, 20 Feb 2024 22:34:21 +0530 Subject: [PATCH 18/20] [docs] Fixed errors in README --- README.rst | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/README.rst b/README.rst index d7bfd97b..282546b1 100644 --- a/README.rst +++ b/README.rst @@ -218,7 +218,7 @@ Extend admin theme programmatically ``openwisp_utils.admin_theme.theme.register_theme_link`` """""""""""""""""""""""""""""""""""""""""""""""""""""""" -Allows adding items to `OPENWISP_ADMIN_THEME_LINKS <#openwisp_admin_theme_links>`_. +Allows adding items to `OPENWISP_ADMIN_THEME_LINKS <#openwisp_admin_theme_links>`__. This function is meant to be used by third party apps or OpenWISP modules which aim to extend the core look and feel of the OpenWISP theme (eg: add new menu icons). @@ -229,17 +229,17 @@ aim to extend the core look and feel of the OpenWISP theme (eg: add new menu ico register_theme_link(links) -+--------------------+-------------------------------------------------------------+ -| **Parameter** | **Description** | -+--------------------+-------------------------------------------------------------+ -| ``links`` | (``list``) List of *link* items to be added to | -| | `OPENWISP_ADMIN_THEME_LINKS <#openwisp_admin_theme_links>`_ | -+--------------------+-------------------------------------------------------------+ ++--------------------+--------------------------------------------------------------+ +| **Parameter** | **Description** | ++--------------------+--------------------------------------------------------------+ +| ``links`` | (``list``) List of *link* items to be added to | +| | `OPENWISP_ADMIN_THEME_LINKS <#openwisp_admin_theme_links>`__ | ++--------------------+--------------------------------------------------------------+ ``openwisp_utils.admin_theme.theme.unregister_theme_link`` """""""""""""""""""""""""""""""""""""""""""""""""""""""""" -Allows removing items from `OPENWISP_ADMIN_THEME_LINKS <#openwisp_admin_theme_links>`_. +Allows removing items from `OPENWISP_ADMIN_THEME_LINKS <#openwisp_admin_theme_links>`__. This function is meant to be used by third party apps or OpenWISP modules which aim additional functionalities to UI of OpenWISP (eg: adding a support chatbot). @@ -250,12 +250,12 @@ aim additional functionalities to UI of OpenWISP (eg: adding a support chatbot). unregister_theme_link(links) -+--------------------+-------------------------------------------------------------+ -| **Parameter** | **Description** | -+--------------------+-------------------------------------------------------------+ -| ``links`` | (``list``) List of *link* items to be removed from | -| | `OPENWISP_ADMIN_THEME_LINKS <#openwisp_admin_theme_links>`_ | -+--------------------+-------------------------------------------------------------+ ++--------------------+--------------------------------------------------------------+ +| **Parameter** | **Description** | ++--------------------+--------------------------------------------------------------+ +| ``links`` | (``list``) List of *link* items to be removed from | +| | `OPENWISP_ADMIN_THEME_LINKS <#openwisp_admin_theme_links>`__ | ++--------------------+--------------------------------------------------------------+ ``openwisp_utils.admin_theme.theme.register_theme_js`` """""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -506,7 +506,7 @@ Code example: .. code-block:: python - from openwisp_utils.admin_theme import register_dashboard_chart + from openwisp_utils.admin_theme import register_dashboard_chart register_dashboard_chart( position=1, @@ -1143,7 +1143,7 @@ inline object. Following is an example: # (optional) You can provide a link to documentation for user reference 'documentation_url': ( 'https://github.com/openwisp/openwisp-utils' - ) + ), # (optional) Icon to be shown along with help text. By default it uses # "/static/admin/img/icon-alert.svg" 'image_url': '/static/admin/img/icon-alert.svg' @@ -1167,8 +1167,8 @@ Code example: class MyModelAdmin(admin.ModelAdmin): list_filter = [ ('my_field', InputFilter), - 'other_field' - ... + 'other_field', + # ... ] By default ``InputFilter`` use exact lookup to filter items which matches to the value being @@ -1188,8 +1188,8 @@ searched by the user. But this behavior can be changed by modifying ``InputFilte class MyModelAdmin(admin.ModelAdmin): list_filter = [ ('my_field', MyInputFilter), - 'other_field' - ... + 'other_field', + # ... ] To know about other lookups that can be used please check @@ -1223,8 +1223,8 @@ The derived filter class should define the ``queryset`` method as shown in follo class MyModelAdmin(admin.ModelAdmin): list_filter = [ MyInputFilter, - 'other_field' - ... + 'other_field', + # ... ] ``openwisp_utils.admin_theme.filters.AutocompleteFilter`` From 8270d9242572d787966f200551da344ae0552bcb Mon Sep 17 00:00:00 2001 From: Gagan Deep Date: Sat, 24 Feb 2024 23:29:23 +0530 Subject: [PATCH 19/20] [feature] Added optional cleaninsights metric collection #360 Closes #360 --- README.rst | 78 +++++ openwisp_utils/measurements/__init__.py | 0 openwisp_utils/measurements/apps.py | 42 +++ .../measurements/migrations/0001_initial.py | 42 +++ .../measurements/migrations/__init__.py | 0 openwisp_utils/measurements/models.py | 46 +++ openwisp_utils/measurements/tasks.py | 56 ++++ openwisp_utils/measurements/tests/__init__.py | 206 +++++++++++++ openwisp_utils/measurements/tests/runner.py | 19 ++ .../measurements/tests/test_models.py | 281 ++++++++++++++++++ openwisp_utils/measurements/utils.py | 36 +++ openwisp_utils/utils.py | 42 +++ requirements-test.txt | 2 +- runtests.py | 1 + tests/openwisp2/__init__.py | 4 + tests/openwisp2/celery.py | 9 + tests/openwisp2/settings.py | 7 +- tests/test_project/tests/test_test_utils.py | 66 +++- 18 files changed, 930 insertions(+), 7 deletions(-) create mode 100644 openwisp_utils/measurements/__init__.py create mode 100644 openwisp_utils/measurements/apps.py create mode 100644 openwisp_utils/measurements/migrations/0001_initial.py create mode 100644 openwisp_utils/measurements/migrations/__init__.py create mode 100644 openwisp_utils/measurements/models.py create mode 100644 openwisp_utils/measurements/tasks.py create mode 100644 openwisp_utils/measurements/tests/__init__.py create mode 100644 openwisp_utils/measurements/tests/runner.py create mode 100644 openwisp_utils/measurements/tests/test_models.py create mode 100644 openwisp_utils/measurements/utils.py create mode 100644 tests/openwisp2/celery.py diff --git a/README.rst b/README.rst index 282546b1..c66174e6 100644 --- a/README.rst +++ b/README.rst @@ -54,6 +54,7 @@ Current features * `Admin Theme utilities <#admin-theme-utilities>`_ * `REST API utilities <#rest-api-utilities>`_ * `Test utilities <#test-utilities>`_ +* `Collection of Usage Metrics <#collection-of-usage-metrics>`_ * `Quality assurance checks <#quality-assurance-checks>`_ ------------ @@ -1347,6 +1348,60 @@ Usage: but not for complex background tasks which can take a long time to execute (eg: firmware upgrades, network operations with retry mechanisms). +``openwisp_utils.tasks.retryable_requests`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A utility function for making HTTP requests with built-in retry logic. +This function is useful for handling transient errors encountered during HTTP +requests by automatically retrying failed requests with exponential backoff. +It provides flexibility in configuring various retry parameters to suit +different use cases. + +Usage: + +.. code-block:: python + + from your_module import retryable_request + + response = retryable_request( + method='GET', + url='https://openwisp.org', + timeout=(4, 8), + max_retries=3, + backoff_factor=1, + backoff_jitter=0.0, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=('HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST'), + retry_kwargs=None, + headers={'Authorization': 'Bearer token'} + ) + +**Paramters:** + +- ``method`` (str): The HTTP method to be used for the request in lower + case (e.g., 'get', 'post', etc.). +- ``timeout`` (tuple): A tuple containing two elements: connection timeout + and read timeout in seconds (default: (4, 8)). +- ``max_retries`` (int): The maximum number of retry attempts in case of + request failure (default: 3). +- ``backoff_factor`` (float): A factor by which the retry delay increases + after each retry (default: 1). +- ``backoff_jitter`` (float): A jitter to apply to the backoff factor to prevent + retry storms (default: 0.0). +- ``status_forcelist`` (tuple): A tuple of HTTP status codes for which retries + should be attempted (default: (429, 500, 502, 503, 504)). +- ``allowed_methods`` (tuple): A tuple of HTTP methods that are allowed for + the request (default: ('HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST')). +- ``retry_kwargs`` (dict): Additional keyword arguments to be passed to the + retry mechanism (default: None). +- ``**kwargs``: Additional keyword arguments to be passed to the underlying request + method (e.g. 'headers', etc.). + +Note: This method will raise a requests.exceptions.RetryError if the request +remains unsuccessful even after all retry attempts have been exhausted. +This exception indicates that the operation could not be completed successfully +despite the retry mechanism. + Storage utilities ----------------- @@ -1615,6 +1670,29 @@ This backend extends ``django.contrib.gis.db.backends.spatialite`` database backend to implement a workaround for handling `issue with sqlite 3.36 and spatialite 5 `_. +Collection of Usage Metrics +--------------------------- + +The openwisp-utils module includes an optional sub-app ``openwisp_utils.measurements``. +This sub-app enables collection of following measurements: + +- Installed OpenWISP Version +- Enabled OpenWISP modules: A list of the enabled OpenWISP modules + along with their respective versions +- OS details: Information on the operating system, including its + version, kernel version, and platform +- Whether the event is related to a new installation or an upgrade + +We collect data on OpenWISP usage to gauge user engagement, satisfaction, +and upgrade patterns. This informs our development decisions, ensuring +continuous improvement aligned with user needs. + +To enhance our understanding and management of this data, we have +integrated `Clean Insights `_, a privacy-preserving +analytics tool. Clean Insights allows us to responsibly gather and analyze +usage metrics without compromising user privacy. It provides us with the +means to make data-driven decisions while respecting our users' rights and trust. + Quality Assurance Checks ------------------------ diff --git a/openwisp_utils/measurements/__init__.py b/openwisp_utils/measurements/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openwisp_utils/measurements/apps.py b/openwisp_utils/measurements/apps.py new file mode 100644 index 00000000..222bd59f --- /dev/null +++ b/openwisp_utils/measurements/apps.py @@ -0,0 +1,42 @@ +from django.apps import AppConfig +from django.conf import settings +from django.db.models.signals import post_migrate + + +class MeasurementsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'openwisp_utils.measurements' + app_label = 'openwisp_measurements' + + def ready(self): + super().ready() + self.connect_post_migrate_signal() + + def connect_post_migrate_signal(self): + post_migrate.connect(self.post_migrate_receiver, sender=self) + + @classmethod + def post_migrate_receiver(cls, **kwargs): + if getattr(settings, 'DEBUG', False): + # Do not send usage metrics in debug mode + # i.e. when running tests. + return + + from .tasks import send_usage_metrics + + is_new_install = False + if kwargs.get('plan'): + migration, migration_rolled_back = kwargs['plan'][0] + is_new_install = ( + migration_rolled_back is False + and str(migration) == 'contenttypes.0001_initial' + ) + + # If the migration plan includes creating table + # for the ContentType model, then the installation is + # treated as a new installation. + if is_new_install: + # This is a new installation + send_usage_metrics.delay() + else: + send_usage_metrics.delay(upgrade_only=True) diff --git a/openwisp_utils/measurements/migrations/0001_initial.py b/openwisp_utils/measurements/migrations/0001_initial.py new file mode 100644 index 00000000..b3eff0a9 --- /dev/null +++ b/openwisp_utils/measurements/migrations/0001_initial.py @@ -0,0 +1,42 @@ +# Generated by Django 4.2.7 on 2023-12-06 15:30 + +from django.db import migrations, models +import django.utils.timezone +import model_utils.fields +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="OpenwispVersion", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "created", + model_utils.fields.AutoCreatedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="created", + ), + ), + ("module_version", models.JSONField(blank=True, default=dict)), + ], + options={ + "ordering": ("-created",), + }, + ), + ] diff --git a/openwisp_utils/measurements/migrations/__init__.py b/openwisp_utils/measurements/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openwisp_utils/measurements/models.py b/openwisp_utils/measurements/models.py new file mode 100644 index 00000000..563d7e93 --- /dev/null +++ b/openwisp_utils/measurements/models.py @@ -0,0 +1,46 @@ +from django.db import models +from openwisp_utils.base import TimeStampedEditableModel +from packaging.version import parse as parse_version + + +class OpenwispVersion(TimeStampedEditableModel): + modified = None + module_version = models.JSONField(default=dict, blank=True) + + class Meta: + ordering = ('-created',) + + @classmethod + def is_new_installation(cls): + return not cls.objects.exists() + + @classmethod + def get_upgraded_modules(cls, current_versions): + """ + Retrieves a dictionary of upgraded modules based on current versions. + Also updates the OpenwispVersion object with the new versions. + + Args: + current_versions (dict): A dictionary containing the current versions of modules. + + Returns: + dict: A dictionary containing the upgraded modules and their versions. + """ + openwisp_version = cls.objects.first() + if not openwisp_version: + cls.objects.create(module_version=current_versions) + return {} + old_versions = openwisp_version.module_version + upgraded_modules = {} + for module, version in current_versions.items(): + if module in old_versions and parse_version( + old_versions[module] + ) < parse_version(version): + upgraded_modules[module] = version + openwisp_version.module_version[module] = version + if upgraded_modules: + # Save the new versions in a new object + OpenwispVersion.objects.create( + module_version=openwisp_version.module_version + ) + return upgraded_modules diff --git a/openwisp_utils/measurements/tasks.py b/openwisp_utils/measurements/tasks.py new file mode 100644 index 00000000..7eabbf09 --- /dev/null +++ b/openwisp_utils/measurements/tasks.py @@ -0,0 +1,56 @@ +import logging + +from celery import shared_task +from openwisp_utils.admin_theme.system_info import ( + get_enabled_openwisp_modules, + get_openwisp_version, + get_os_details, +) + +from ..tasks import OpenwispCeleryTask +from ..utils import retryable_request +from .models import OpenwispVersion +from .utils import _get_events, get_openwisp_module_metrics, get_os_detail_metrics + +USER_METRIC_COLLECTION_URL = 'https://analytics.openwisp.io/cleaninsights.php' + +logger = logging.getLogger(__name__) + + +def post_usage_metrics(events): + try: + response = retryable_request( + 'post', + url=USER_METRIC_COLLECTION_URL, + json={ + 'idsite': 5, + 'events': events, + }, + max_retries=10, + ) + assert response.status_code == 204 + except Exception as error: + if isinstance(error, AssertionError): + message = f'HTTP {response.status_code} Response' + else: + message = str(error) + logger.error( + f'Collection of usage metrics failed, max retries exceeded. Error: {message}' + ) + + +@shared_task(base=OpenwispCeleryTask) +def send_usage_metrics(upgrade_only=False): + current_versions = get_enabled_openwisp_modules() + current_versions.update({'OpenWISP Version': get_openwisp_version()}) + metrics = [] + metrics.extend(get_os_detail_metrics(get_os_details())) + if OpenwispVersion.is_new_installation(): + metrics.extend(_get_events('Install', current_versions)) + OpenwispVersion.objects.create(module_version=current_versions) + else: + upgraded_modules = OpenwispVersion.get_upgraded_modules(current_versions) + metrics.extend(_get_events('Upgrade', upgraded_modules)) + if not upgrade_only: + metrics.extend(get_openwisp_module_metrics(current_versions)) + post_usage_metrics(metrics) diff --git a/openwisp_utils/measurements/tests/__init__.py b/openwisp_utils/measurements/tests/__init__.py new file mode 100644 index 00000000..5a964e0f --- /dev/null +++ b/openwisp_utils/measurements/tests/__init__.py @@ -0,0 +1,206 @@ +_ENABLED_OPENWISP_MODULES_RETURN_VALUE = { + 'openwisp-utils': '1.1.0a', + 'openwisp-users': '1.1.0a', +} +_OS_DETAILS_RETURN_VALUE = { + 'kernel_version': '5.13.0-52-generic', + 'os_version': '#59~20.04.1-Ubuntu SMP Thu Jun 16 21:21:28 UTC 2022', + 'hardware_platform': 'x86_64', +} + +_MODULES_UPGRADE_EXPECTED_METRICS = [ + { + 'category': 'OS Detail', + 'action': 'kernel_version', + 'name': '5.13.0-52-generic', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'OS Detail', + 'action': 'os_version', + 'name': '#59~20.04.1-Ubuntu SMP Thu Jun 16 21:21:28 UTC 2022', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'OS Detail', + 'action': 'hardware_platform', + 'name': 'x86_64', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'Upgrade', + 'action': 'openwisp-utils', + 'name': '1.1.0a', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'Upgrade', + 'action': 'openwisp-users', + 'name': '1.1.0a', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'Upgrade', + 'action': 'OpenWISP Version', + 'name': '23.0.0a', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'Heartbeat', + 'action': 'openwisp-utils', + 'name': '1.1.0a', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'Heartbeat', + 'action': 'openwisp-users', + 'name': '1.1.0a', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'Heartbeat', + 'action': 'OpenWISP Version', + 'name': '23.0.0a', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, +] + +_HEARTBEAT_METRICS = [ + { + 'category': 'OS Detail', + 'action': 'kernel_version', + 'name': '5.13.0-52-generic', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'OS Detail', + 'action': 'os_version', + 'name': '#59~20.04.1-Ubuntu SMP Thu Jun 16 21:21:28 UTC 2022', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'OS Detail', + 'action': 'hardware_platform', + 'name': 'x86_64', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'Heartbeat', + 'action': 'openwisp-utils', + 'name': '1.1.0a', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'Heartbeat', + 'action': 'openwisp-users', + 'name': '1.1.0a', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'Heartbeat', + 'action': 'OpenWISP Version', + 'name': '23.0.0a', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, +] +_NEW_INSTALLATION_METRICS = [ + { + 'category': 'OS Detail', + 'action': 'kernel_version', + 'name': '5.13.0-52-generic', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'OS Detail', + 'action': 'os_version', + 'name': '#59~20.04.1-Ubuntu SMP Thu Jun 16 21:21:28 UTC 2022', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'OS Detail', + 'action': 'hardware_platform', + 'name': 'x86_64', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'Install', + 'action': 'openwisp-utils', + 'name': '1.1.0a', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'Install', + 'action': 'openwisp-users', + 'name': '1.1.0a', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, + { + 'category': 'Install', + 'action': 'OpenWISP Version', + 'name': '23.0.0a', + 'value': 1, + 'times': 1, + 'period_start': 1701388800, + 'period_end': 1701388800, + }, +] diff --git a/openwisp_utils/measurements/tests/runner.py b/openwisp_utils/measurements/tests/runner.py new file mode 100644 index 00000000..ce7378e7 --- /dev/null +++ b/openwisp_utils/measurements/tests/runner.py @@ -0,0 +1,19 @@ +from unittest.mock import MagicMock + +import requests +from django.test.runner import DiscoverRunner +from openwisp_utils import utils + +success_response = requests.Response() +success_response.status_code = 204 + + +class MockRequestPostRunner(DiscoverRunner): + """ + This runner ensures that usage metrics are + not sent in development when running tests. + """ + + def setup_databases(self, **kwargs): + utils.requests.post = MagicMock(return_value=success_response) + return super().setup_databases(**kwargs) diff --git a/openwisp_utils/measurements/tests/test_models.py b/openwisp_utils/measurements/tests/test_models.py new file mode 100644 index 00000000..1a5b1c3a --- /dev/null +++ b/openwisp_utils/measurements/tests/test_models.py @@ -0,0 +1,281 @@ +from datetime import datetime, timezone +from unittest.mock import patch + +import requests +from django.apps import apps +from django.db import migrations +from django.test import TestCase, override_settings +from freezegun import freeze_time +from urllib3.response import HTTPResponse + +from .. import tasks +from ..models import OpenwispVersion +from . import ( + _ENABLED_OPENWISP_MODULES_RETURN_VALUE, + _HEARTBEAT_METRICS, + _MODULES_UPGRADE_EXPECTED_METRICS, + _NEW_INSTALLATION_METRICS, + _OS_DETAILS_RETURN_VALUE, +) + + +class TestOpenwispVersion(TestCase): + def setUp(self): + # The post_migrate signal creates the first OpenwispVersion object + # and uses the actual modules installed in the Python environment. + # This would cause tests to fail when other modules are also installed. + # import ipdb; ipdb.set_trace() + OpenwispVersion.objects.update( + module_version={ + 'OpenWISP Version': '23.0.0a', + **_ENABLED_OPENWISP_MODULES_RETURN_VALUE, + }, + created=datetime.strptime( + '2023-11-01 00:00:00', '%Y-%m-%d %H:%M:%S' + ).replace(tzinfo=timezone.utc), + ) + + def test_get_upgraded_modules_when_openwispversion_object_does_not_exist(self): + OpenwispVersion.objects.all().delete() + self.assertEqual( + OpenwispVersion.get_upgraded_modules(tasks.get_enabled_openwisp_modules()), + {}, + ) + + def test_get_upgraded_modules_on_new_installation(self): + self.assertEqual( + OpenwispVersion.get_upgraded_modules(tasks.get_enabled_openwisp_modules()), + {}, + ) + + @patch.object(tasks, 'get_openwisp_version', return_value='23.0.0a') + @patch.object( + tasks, + 'get_enabled_openwisp_modules', + return_value=_ENABLED_OPENWISP_MODULES_RETURN_VALUE, + ) + @patch.object( + tasks, + 'get_os_details', + return_value=_OS_DETAILS_RETURN_VALUE, + ) + @patch.object(tasks, 'post_usage_metrics') + @freeze_time('2023-12-01 00:00:00') + def test_new_installation(self, mocked_post, *args): + OpenwispVersion.objects.all().delete() + tasks.send_usage_metrics.delay() + mocked_post.assert_called_with(_NEW_INSTALLATION_METRICS) + self.assertEqual(OpenwispVersion.objects.count(), 1) + version = OpenwispVersion.objects.first() + expected_module_version = { + 'OpenWISP Version': '23.0.0a', + **_ENABLED_OPENWISP_MODULES_RETURN_VALUE, + } + self.assertEqual(version.module_version, expected_module_version) + + @patch.object(tasks, 'get_openwisp_version', return_value='23.0.0a') + @patch.object( + tasks, + 'get_enabled_openwisp_modules', + return_value=_ENABLED_OPENWISP_MODULES_RETURN_VALUE, + ) + @patch.object( + tasks, + 'get_os_details', + return_value=_OS_DETAILS_RETURN_VALUE, + ) + @patch.object(tasks, 'post_usage_metrics') + @freeze_time('2023-12-01 00:00:00') + def test_heartbeat(self, mocked_post, *args): + expected_module_version = { + 'OpenWISP Version': '23.0.0a', + **_ENABLED_OPENWISP_MODULES_RETURN_VALUE, + } + self.assertEqual(OpenwispVersion.objects.count(), 1) + tasks.send_usage_metrics.delay() + mocked_post.assert_called_with(_HEARTBEAT_METRICS) + self.assertEqual(OpenwispVersion.objects.count(), 1) + version = OpenwispVersion.objects.first() + self.assertEqual(version.module_version, expected_module_version) + + @patch.object(tasks, 'get_openwisp_version', return_value='23.0.0a') + @patch.object( + tasks, + 'get_enabled_openwisp_modules', + return_value=_ENABLED_OPENWISP_MODULES_RETURN_VALUE, + ) + @patch.object( + tasks, + 'get_os_details', + return_value=_OS_DETAILS_RETURN_VALUE, + ) + @patch.object(tasks, 'post_usage_metrics') + @freeze_time('2023-12-01 00:00:00') + def test_modules_upgraded(self, mocked_post, *args): + self.assertEqual(OpenwispVersion.objects.count(), 1) + OpenwispVersion.objects.update( + module_version={ + 'OpenWISP Version': '22.10.0', + 'openwisp-utils': '1.0.5', + 'openwisp-users': '1.0.2', + } + ) + tasks.send_usage_metrics.delay() + mocked_post.assert_called_with(_MODULES_UPGRADE_EXPECTED_METRICS) + + self.assertEqual(OpenwispVersion.objects.count(), 2) + version = OpenwispVersion.objects.first() + expected_module_version = { + 'OpenWISP Version': '23.0.0a', + **_ENABLED_OPENWISP_MODULES_RETURN_VALUE, + } + self.assertEqual(version.module_version, expected_module_version) + + @freeze_time('2023-12-01 00:00:00') + @patch.object(tasks, 'get_openwisp_version', return_value='23.0.0a') + @patch.object( + tasks, + 'get_enabled_openwisp_modules', + return_value=_ENABLED_OPENWISP_MODULES_RETURN_VALUE, + ) + @patch.object( + tasks, + 'get_os_details', + return_value=_OS_DETAILS_RETURN_VALUE, + ) + @patch.object(tasks, 'post_usage_metrics') + @patch.object(tasks, 'get_openwisp_module_metrics') + def test_send_usage_metrics_upgrade_only_flag( + self, mocked_get_openwisp_module_metrics, *args + ): + self.assertEqual(OpenwispVersion.objects.count(), 1) + # Store old versions of OpenWISP modules in OpenwispVersion object + OpenwispVersion.objects.update( + module_version={ + 'OpenWISP Version': '22.10.0', + 'openwisp-utils': '1.0.5', + 'openwisp-users': '1.0.2', + } + ) + tasks.send_usage_metrics.delay(upgrade_only=True) + mocked_get_openwisp_module_metrics.assert_not_called() + self.assertEqual(OpenwispVersion.objects.count(), 2) + version = OpenwispVersion.objects.first() + expected_module_version = { + 'OpenWISP Version': '23.0.0a', + **_ENABLED_OPENWISP_MODULES_RETURN_VALUE, + } + self.assertEqual(version.module_version, expected_module_version) + + @patch('time.sleep') + @patch('logging.Logger.warning') + @patch('logging.Logger.error') + def test_post_usage_metrics_400_response(self, mocked_error, mocked_warning, *args): + bad_response = requests.Response() + bad_response.status_code = 400 + with patch.object( + requests.Session, 'post', return_value=bad_response + ) as mocked_post: + tasks.send_usage_metrics.delay() + mocked_post.assert_called_once() + mocked_warning.assert_not_called() + mocked_error.assert_called_with( + 'Collection of usage metrics failed, max retries exceeded.' + ' Error: HTTP 400 Response' + ) + + @patch('urllib3.util.retry.Retry.sleep') + @patch( + 'urllib3.connectionpool.HTTPConnection.getresponse', + return_value=HTTPResponse(status=500, version='1.1'), + ) + @patch('logging.Logger.error') + def test_post_usage_metrics_500_response( + self, mocked_error, mocked_getResponse, *args + ): + tasks.send_usage_metrics.delay() + self.assertEqual(len(mocked_getResponse.mock_calls), 11) + mocked_error.assert_called_with( + 'Collection of usage metrics failed, max retries exceeded.' + ' Error: HTTPSConnectionPool(host=\'analytics.openwisp.io\', port=443):' + ' Max retries exceeded with url: /cleaninsights.php (Caused by ResponseError' + '(\'too many 500 error responses\'))' + ) + + @patch('time.sleep') + @patch('logging.Logger.warning') + @patch('logging.Logger.error') + def test_post_usage_metrics_204_response(self, mocked_error, mocked_warning, *args): + bad_response = requests.Response() + bad_response.status_code = 204 + with patch.object( + requests.Session, 'post', return_value=bad_response + ) as mocked_post: + tasks.send_usage_metrics.delay() + self.assertEqual(len(mocked_post.mock_calls), 1) + mocked_warning.assert_not_called() + mocked_error.assert_not_called() + + @patch('urllib3.util.retry.Retry.sleep') + @patch( + 'urllib3.connectionpool.HTTPConnectionPool._get_conn', + side_effect=OSError, + ) + @patch('logging.Logger.error') + def test_post_usage_metrics_connection_error( + self, mocked_error, mocked_get_conn, *args + ): + tasks.send_usage_metrics.delay() + mocked_error.assert_called_with( + 'Collection of usage metrics failed, max retries exceeded.' + ' Error: HTTPSConnectionPool(host=\'analytics.openwisp.io\', port=443):' + ' Max retries exceeded with url: /cleaninsights.php' + ' (Caused by ProtocolError(\'Connection aborted.\', OSError()))' + ) + self.assertEqual(mocked_get_conn.call_count, 11) + + @patch.object(tasks.send_usage_metrics, 'delay') + def test_post_migrate_receiver(self, mocked_task, *args): + app = apps.get_app_config('measurements') + + with self.subTest( + 'Test task is called for checking upgrades when plan is empty' + ): + app.post_migrate_receiver(plan=[]) + mocked_task.assert_called_with(upgrade_only=True) + mocked_task.reset_mock() + + with self.subTest( + 'Test task is called for checking upgrades ' + 'when first migration in plan is not for ContentTypes' + ): + app.post_migrate_receiver( + plan=[ + ( + migrations.Migration( + name='0001_initial', app_label='openwisp_users' + ), + False, + ) + ] + ) + mocked_task.assert_called_with(upgrade_only=True) + mocked_task.reset_mock() + plan = [ + ( + migrations.Migration(name='0001_initial', app_label='contenttypes'), + False, + ) + ] + + with self.subTest( + 'Test task called when first migration in plan is for ContentTypes' + ): + app.post_migrate_receiver(plan=plan) + mocked_task.assert_called_with() + mocked_task.reset_mock() + + with self.subTest('Test task not called in development'): + with override_settings(DEBUG=True): + app.post_migrate_receiver(plan=plan) + mocked_task.assert_not_called() diff --git a/openwisp_utils/measurements/utils.py b/openwisp_utils/measurements/utils.py new file mode 100644 index 00000000..c816c3a0 --- /dev/null +++ b/openwisp_utils/measurements/utils.py @@ -0,0 +1,36 @@ +from django.utils.html import escape +from django.utils.timezone import now + + +def _get_events(category, data): + """ + This function takes a category and data representing usage metrics, + and returns a list of events in a format accepted by the + Clean Insights Matomo Proxy (CIMP) API. + + Read the "Event Measurement Schema" in the CIMP documentation: + https://cutt.ly/SwBkC40A + """ + events = [] + unix_time = int(now().timestamp()) + for key, value in data.items(): + events.append( + { + 'category': category, + 'action': escape(key), + 'name': escape(value), + 'value': 1, + 'times': 1, + 'period_start': unix_time, + 'period_end': unix_time, + } + ) + return events + + +def get_openwisp_module_metrics(module_versions): + return _get_events('Heartbeat', module_versions) + + +def get_os_detail_metrics(os_detail): + return _get_events('OS Detail', os_detail) diff --git a/openwisp_utils/utils.py b/openwisp_utils/utils.py index d4190249..1193cddb 100644 --- a/openwisp_utils/utils.py +++ b/openwisp_utils/utils.py @@ -1,8 +1,11 @@ from collections import OrderedDict from copy import deepcopy +import requests from django.conf import settings from django.utils.crypto import get_random_string +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry class SortedOrderedDict(OrderedDict): @@ -62,3 +65,42 @@ def print_color(string, color_name, end='\n'): } color = color_dict.get(color_name, '0') print(f'\033[{color}m{string}\033[0m', end=end) + + +def retryable_request( + method, + timeout=(4, 8), + max_retries=3, + backoff_factor=1, + backoff_jitter=0.0, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=('HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST'), + retry_kwargs=None, + **kwargs, +): + retry_kwargs = retry_kwargs or {} + retry_kwargs.update( + dict( + total=max_retries, + backoff_factor=backoff_factor, + status_forcelist=status_forcelist, + allowed_methods=allowed_methods, + backoff_jitter=backoff_jitter, + ) + ) + retry_kwargs = retry_kwargs or {} + retry_kwargs.update( + dict( + total=max_retries, + backoff_factor=backoff_factor, + backoff_jitter=backoff_jitter, + status_forcelist=status_forcelist, + allowed_methods=allowed_methods, + ) + ) + request_session = requests.Session() + retries = Retry(**retry_kwargs) + request_session.mount('https://', HTTPAdapter(max_retries=retries)) + request_session.mount('http://', HTTPAdapter(max_retries=retries)) + request_method = getattr(request_session, method) + return request_method(timeout=timeout, **kwargs) diff --git a/requirements-test.txt b/requirements-test.txt index a4fd17a5..35090b90 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,3 @@ # For testing Dependency loaders openwisp_controller @ https://github.com/openwisp/openwisp-controller/tarball/master - +freezegun diff --git a/runtests.py b/runtests.py index 5427a446..4dbbe8a1 100755 --- a/runtests.py +++ b/runtests.py @@ -13,4 +13,5 @@ args = sys.argv args.insert(1, "test") args.insert(2, "test_project") + args.insert(3, "openwisp_utils.measurements") execute_from_command_line(args) diff --git a/tests/openwisp2/__init__.py b/tests/openwisp2/__init__.py index 62bcec8d..e61ef422 100644 --- a/tests/openwisp2/__init__.py +++ b/tests/openwisp2/__init__.py @@ -1 +1,5 @@ +from .celery import app as celery_app + +__all__ = ['celery_app'] + __openwisp_version__ = '23.0.0a' diff --git a/tests/openwisp2/celery.py b/tests/openwisp2/celery.py new file mode 100644 index 00000000..e3c1425d --- /dev/null +++ b/tests/openwisp2/celery.py @@ -0,0 +1,9 @@ +import os + +from celery import Celery + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'openwisp2.settings') + +app = Celery('openwisp2') +app.config_from_object('django.conf:settings', namespace='CELERY') +app.autodiscover_tasks() diff --git a/tests/openwisp2/settings.py b/tests/openwisp2/settings.py index 7e3493d6..737f4327 100644 --- a/tests/openwisp2/settings.py +++ b/tests/openwisp2/settings.py @@ -19,6 +19,7 @@ # test project 'test_project', 'openwisp_utils.admin_theme', + 'openwisp_utils.measurements', 'django.contrib.sites', # admin 'django.contrib.admin', @@ -81,7 +82,7 @@ DATABASES = { 'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': 'openwisp_utils.db'} } - +TEST_RUNNER = 'openwisp_utils.measurements.tests.runner.MockRequestPostRunner' OPENWISP_ADMIN_SITE_CLASS = 'test_project.site.CustomAdminSite' SITE_ID = 1 @@ -124,6 +125,10 @@ ] OPENWISP_ADMIN_THEME_JS = ['dummy.js'] +CELERY_TASK_ALWAYS_EAGER = True +CELERY_TASK_EAGER_PROPAGATES = True +CELERY_BROKER_URL = 'memory://' + # local settings must be imported before test runner otherwise they'll be ignored try: from local_settings import * diff --git a/tests/test_project/tests/test_test_utils.py b/tests/test_project/tests/test_test_utils.py index f228eb71..417d286a 100644 --- a/tests/test_project/tests/test_test_utils.py +++ b/tests/test_project/tests/test_test_utils.py @@ -1,5 +1,5 @@ import sys -import unittest +from unittest.mock import patch from django.dispatch import Signal from django.test import TestCase, override_settings @@ -11,7 +11,9 @@ capture_stdout, catch_signal, ) -from openwisp_utils.utils import deep_merge_dicts, print_color +from openwisp_utils.utils import deep_merge_dicts, print_color, retryable_request +from requests.exceptions import ConnectionError, RetryError +from urllib3.response import HTTPResponse from ..models import Shelf @@ -87,12 +89,66 @@ def test_capture_stderr(self, captured_error): print('Testing capture_stderr', file=sys.stderr, end='') self.assertEqual(captured_error.getvalue(), 'Testing capture_stderr') + @patch('urllib3.util.retry.Retry.sleep') + def test_retryable_request(self, *args): + with self.subTest('Test failure to connect to server'): + with patch( + 'urllib3.connectionpool.HTTPConnectionPool._get_conn', + side_effect=OSError, + ) as mocked__get_conn: + with self.assertRaises(ConnectionError) as error: + retryable_request('get', url='https://openwisp.org') + self.assertEqual(len(mocked__get_conn.mock_calls), 4) + self.assertIn( + 'OSError', + str(error.exception), + ) + + with self.subTest('Test retry on server error'): + # Simulates the test never recovered + with patch( + 'urllib3.connectionpool.HTTPConnection.getresponse', + return_value=HTTPResponse(status=500, version='1.1'), + ) as mocked_getResponse: + with self.assertRaises(RetryError) as error: + retryable_request('get', url='https://openwisp.org') + self.assertEqual(len(mocked_getResponse.mock_calls), 4) + self.assertIn( + 'too many 500 error responses', + str(error.exception), + ) + + with self.subTest('Test customization with retry_kwargs'): + with patch( + 'openwisp_utils.utils.Retry', + ) as mocked_retry, patch('openwisp_utils.utils.requests.Session'): + retryable_request( + method='get', + url='https://openwisp.org', + max_retries=10, + backoff_factor=2, + backoff_jitter=0.2, + status_forcelist=(429, 500), + allowed_methods=('HEAD',), + retry_kwargs={'raise_on_redirect': False}, + ) + self.assertDictEqual( + mocked_retry.call_args.kwargs, + { + 'total': 10, + 'backoff_factor': 2, + 'status_forcelist': (429, 500), + 'allowed_methods': ('HEAD',), + 'backoff_jitter': 0.2, + 'raise_on_redirect': False, + }, + ) + self.assertEqual(mocked_retry.call_args[1]['total'], 10) + class TestAssertNumQueriesSubTest(AssertNumQueriesSubTestMixin, TestCase): def test_assert_num_queries(self): - with unittest.mock.patch.object( - self, 'subTest', wraps=self.subTest - ) as patched_subtest: + with patch.object(self, 'subTest', wraps=self.subTest) as patched_subtest: with self.assertNumQueries(1): Shelf.objects.count() patched_subtest.assert_called_once() From 2440f962f1f2952d6d1b298443175090cab5856a Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Tue, 27 Feb 2024 13:53:54 -0300 Subject: [PATCH 20/20] [chores] Added comments to matomo clean insights events #360 Related to #360 --- openwisp_utils/measurements/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openwisp_utils/measurements/utils.py b/openwisp_utils/measurements/utils.py index c816c3a0..718f8add 100644 --- a/openwisp_utils/measurements/utils.py +++ b/openwisp_utils/measurements/utils.py @@ -16,10 +16,15 @@ def _get_events(category, data): for key, value in data.items(): events.append( { + # OS Details, Install, Hearthbeat, Upgrade 'category': category, + # Name of OW module or OS parameter 'action': escape(key), + # Actual version of OW module, OS or general OW version 'name': escape(value), + # Value is always 1 'value': 1, + # Event happened only 1 time, we do not aggregate 'times': 1, 'period_start': unix_time, 'period_end': unix_time,