[ch29016] Added custom json Validator based on jsonschema lib

django_api/Pipfile

@@ -41,6 +41,7 @@ djangorestframework-gis = "<=0.17"
 djangorestframework-simplejwt = "<=4.8"
 drfpasswordless = "<=1.5.7"
 greenlet = "<=1.1.1"
+jsonschema = "<=4.4.0"
 newrelic = "<=6.8.0.163"
 openpyxl = "<=3.0.7"
 psycopg2-binary = "<=2.9.1"

django_api/etools_prp/apps/core/validators.py

@@ -1,14 +1,17 @@
 from django.contrib.contenttypes.models import ContentType
+from django.core import exceptions
+from django.utils.deconstruct import deconstructible
+from django.utils.translation import gettext_lazy as _
 
+from jsonschema import exceptions as jsonschema_exceptions, validate
 from rest_framework.exceptions import ValidationError
 
-from etools_prp.apps.cluster.models import ClusterActivity, ClusterObjective
-from etools_prp.apps.partner.models import PartnerActivity, PartnerActivityProjectContext, PartnerProject
-
 
 class AddIndicatorObjectTypeValidator:
 
     def __call__(self, value):
+        from etools_prp.apps.cluster.models import ClusterActivity, ClusterObjective
+        from etools_prp.apps.partner.models import PartnerActivity, PartnerActivityProjectContext, PartnerProject
         model_choices = {
             ClusterObjective,
             ClusterActivity,
@@ -26,3 +29,26 @@ def __call__(self, value):
 
 
 add_indicator_object_type_validator = AddIndicatorObjectTypeValidator()
+
+
+@deconstructible
+class JSONSchemaValidator:
+    message = _("Invalid JSON: %(value)s")
+    code = 'invalid_json'
+
+    def __init__(self, json_schema, message=None):
+        self.json_schema = json_schema
+        if message:
+            self.message = message
+
+    def __call__(self, value):
+        try:
+            validate(value, self.json_schema)
+        except jsonschema_exceptions.ValidationError as e:
+            raise exceptions.ValidationError(self.message, code=self.code, params={'value': e.message})
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and \
+            self.json_schema == other.json_schema and \
+            self.message == other.message and \
+            self.code == other.code

django_api/etools_prp/apps/indicator/json_schemas.py

@@ -0,0 +1,22 @@
+
+
+indicator_schema = {
+    "title": "Json schema for total, target, baseline and in_need fields",
+    "type": "object",
+    "additionalProperties": False,
+    "properties": {
+        "c": {"type": "number"},
+        "d": {"type": "number"},
+        "v": {"type": "number"}
+    },
+    "required": ["d", "v"]
+}
+
+disaggregation_schema = {
+    "title": "Disaggregation json schema",
+    "type": "object",
+    "additionalProperties": False,
+    "patternProperties": {
+        "^\((\d*,\s*)*\d*\)$": indicator_schema  # noqa W605
+    }
+}

django_api/etools_prp/apps/indicator/migrations/0009_auto_20220418_1714.py

@@ -0,0 +1,60 @@
+# Generated by Django 3.2.6 on 2022-04-18 17:14
+
+from django.db import migrations, models
+import etools_prp.apps.core.validators
+import etools_prp.apps.indicator.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('indicator', '0008_alter_indicatorlocationdata_disaggregation'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='indicatorlocationdata',
+            name='disaggregation',
+            field=models.JSONField(default=etools_prp.apps.indicator.models.default_disaggregation, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'patternProperties': {'^\\((\\d*,\\s*)*\\d*\\)$': {'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'}}, 'title': 'Disaggregation json schema', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='indicatorreport',
+            name='total',
+            field=models.JSONField(default=etools_prp.apps.indicator.models.default_total, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='reportable',
+            name='baseline',
+            field=models.JSONField(default=etools_prp.apps.indicator.models.default_value, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='reportable',
+            name='in_need',
+            field=models.JSONField(blank=True, null=True, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='reportable',
+            name='target',
+            field=models.JSONField(default=etools_prp.apps.indicator.models.default_value, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='reportable',
+            name='total',
+            field=models.JSONField(default=etools_prp.apps.indicator.models.default_total, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='reportablelocationgoal',
+            name='baseline',
+            field=models.JSONField(default=etools_prp.apps.indicator.models.default_value, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='reportablelocationgoal',
+            name='in_need',
+            field=models.JSONField(blank=True, null=True, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='reportablelocationgoal',
+            name='target',
+            field=models.JSONField(default=etools_prp.apps.indicator.models.default_value, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+    ]

django_api/etools_prp/apps/indicator/models.py

@@ -25,8 +25,10 @@
     REPORTING_TYPES,
 )
 from etools_prp.apps.core.models import TimeStampedExternalSourceModel
+from etools_prp.apps.core.validators import JSONSchemaValidator
 from etools_prp.apps.indicator.constants import ValueType
 from etools_prp.apps.indicator.disaggregators import QuantityIndicatorDisaggregator, RatioIndicatorDisaggregator
+from etools_prp.apps.indicator.json_schemas import disaggregation_schema, indicator_schema
 from etools_prp.apps.indicator.utilities import convert_string_number_to_float
 from etools_prp.apps.partner.models import PartnerActivity
 from etools_prp.apps.utils.emails import send_email_from_template
@@ -241,9 +243,18 @@ class Reportable(TimeStampedExternalSourceModel):
         cluster.ClusterObjective (ForeignKey): "content_object"
         self (ForeignKey): "parent_indicator"
     """
-    target = models.JSONField(default=default_value)
-    baseline = models.JSONField(default=default_value)
-    in_need = models.JSONField(blank=True, null=True)
+    target = models.JSONField(
+        default=default_value,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
+    baseline = models.JSONField(
+        default=default_value,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
+    in_need = models.JSONField(
+        blank=True, null=True,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
     assumptions = models.TextField(null=True, blank=True)
     means_of_verification = models.CharField(max_length=255, null=True, blank=True)
     comments = models.TextField(max_length=4048, blank=True, null=True)
@@ -260,7 +271,7 @@ class Reportable(TimeStampedExternalSourceModel):
 
     # Current total, transactional and dynamically calculated based on
     # IndicatorReports
-    total = models.JSONField(default=default_total)
+    total = models.JSONField(default=default_total, validators=[JSONSchemaValidator(json_schema=indicator_schema)])
 
     # unique code for this indicator within the current context
     # eg: (1.1) result code 1 - indicator code 1
@@ -633,9 +644,18 @@ def clone_ca_reportable_to_pa_signal(sender, instance, created, **kwargs):
 class ReportableLocationGoal(TimeStampedModel):
     reportable = models.ForeignKey(Reportable, on_delete=models.CASCADE)
     location = models.ForeignKey("core.Location", on_delete=models.CASCADE)
-    target = models.JSONField(default=default_value)
-    baseline = models.JSONField(default=default_value)
-    in_need = models.JSONField(blank=True, null=True)
+    target = models.JSONField(
+        default=default_value,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
+    baseline = models.JSONField(
+        default=default_value,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
+    in_need = models.JSONField(
+        blank=True, null=True,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
     is_active = models.BooleanField(default=True)
 
     class Meta:
@@ -685,7 +705,10 @@ class IndicatorReport(TimeStampedModel):
         verbose_name='Frequency of reporting'
     )
 
-    total = models.JSONField(default=default_total)
+    total = models.JSONField(
+        default=default_total,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
 
     remarks = models.TextField(blank=True, null=True)
     report_status = models.CharField(
@@ -1068,7 +1091,10 @@ class IndicatorLocationData(TimeStampedModel):
         on_delete=models.CASCADE,
     )
 
-    disaggregation = models.JSONField(default=default_disaggregation)
+    disaggregation = models.JSONField(
+        default=default_disaggregation,
+        validators=[JSONSchemaValidator(json_schema=disaggregation_schema)]
+    )
     num_disaggregation = models.IntegerField()
     level_reported = models.IntegerField()
     disaggregation_reported_on = ArrayField(models.IntegerField(), default=list)

django_api/etools_prp/apps/indicator/serializers.py

@@ -25,12 +25,13 @@
 )
 from etools_prp.apps.core.models import Location
 from etools_prp.apps.core.serializers import IdLocationSerializer, LocationSerializer
-from etools_prp.apps.core.validators import add_indicator_object_type_validator
+from etools_prp.apps.core.validators import add_indicator_object_type_validator, JSONSchemaValidator
 from etools_prp.apps.ocha.imports.serializers import DiscardUniqueTogetherValidationMixin
 from etools_prp.apps.partner.models import Partner, PartnerActivity, PartnerActivityProjectContext, PartnerProject
 from etools_prp.apps.unicef.models import LowerLevelOutput, ProgressReport
 
 from .fields import SortedDateArrayField
+from .json_schemas import disaggregation_schema, indicator_schema
 from .models import (
     create_pa_reportables_for_new_ca_reportable,
     Disaggregation,
@@ -227,9 +228,16 @@ def update(self, instance, validated_data):
 
 class ReportableLocationGoalBaselineInNeedSerializer(serializers.ModelSerializer):
     id = serializers.IntegerField()
-    baseline = serializers.JSONField()
-    in_need = serializers.JSONField(required=False, allow_null=True)
-    target = serializers.JSONField()
+    baseline = serializers.JSONField(
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
+    in_need = serializers.JSONField(
+        required=False, allow_null=True,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
+    target = serializers.JSONField(
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
     location = LocationSerializer(read_only=True)
 
     def validate(self, data):
@@ -594,7 +602,9 @@ class IndicatorLocationDataUpdateSerializer(serializers.ModelSerializer):
     disaggregation_reported_on = serializers.ListField(
         child=serializers.IntegerField()
     )
-    disaggregation = serializers.JSONField()
+    disaggregation = serializers.JSONField(
+        validators=[JSONSchemaValidator(json_schema=disaggregation_schema)]
+    )
     reporting_entity_percentage_map = serializers.JSONField(
         required=False,
     )
@@ -1126,8 +1136,8 @@ class ClusterObjectiveIndicatorAdoptSerializer(serializers.Serializer):
     cluster_objective_id = serializers.IntegerField()
     reportable_id = serializers.IntegerField()
     locations = ReportableLocationGoalSerializer(many=True, write_only=True)
-    target = serializers.JSONField()
-    baseline = serializers.JSONField()
+    target = serializers.JSONField(validators=[JSONSchemaValidator(json_schema=indicator_schema)])
+    baseline = serializers.JSONField(validators=[JSONSchemaValidator(json_schema=indicator_schema)])
 
     def validate(self, data):
         """
@@ -1245,12 +1255,17 @@ class ClusterIndicatorSerializer(serializers.ModelSerializer):
 
     disaggregations = IdDisaggregationSerializer(many=True, read_only=True)
     object_type = serializers.CharField(
-        validators=[add_indicator_object_type_validator], write_only=True)
+        validators=[add_indicator_object_type_validator], write_only=True
+    )
     blueprint = IndicatorBlueprintSerializer()
     locations = ReportableLocationGoalSerializer(many=True, write_only=True)
-    target = serializers.JSONField()
-    baseline = serializers.JSONField()
-    in_need = serializers.JSONField(required=False, allow_null=True)
+    target = serializers.JSONField(validators=[JSONSchemaValidator(json_schema=indicator_schema)])
+    baseline = serializers.JSONField(validators=[JSONSchemaValidator(json_schema=indicator_schema)])
+    in_need = serializers.JSONField(
+        required=False,
+        allow_null=True,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
     project_context_id = serializers.IntegerField(write_only=True, required=False, allow_null=True)
     cs_dates = SortedDateArrayField(child=serializers.DateField(), required=False)