Merge branch 'develop' into 28750-add-404-page

.circleci/config.yml

@@ -255,6 +255,8 @@ workflows:
                 - master
                 - staging
                 - develop
+                - submodule-ref-2
+
       - build_and_deploy_fe_cluster:
           filters:
             branches:

db/Dockerfile

@@ -3,7 +3,7 @@ FROM postgis/postgis:12-3.1
 RUN apt-get update && apt-get install -y --no-install-recommends bzip2
 
 # Use this if there is a db dump file
-COPY ./db1.bz2 /tmp/psql_data/
+
 COPY load_db_data.sh /docker-entrypoint-initdb.d/20_load_db_data.sh
 
 EXPOSE 5432

db/load_db_data.sh

@@ -2,10 +2,10 @@
 
 set -e
 
-export DB_DUMP_LOCATION=/tmp/psql_data/db1.bz2
+export DB_DUMP_LOCATION=/tmp/psql_data/db_dump.bz2
 
 echo "*** RESTORING DATABASE $POSTGRES_DB ***"
-bzcat $DB_DUMP_LOCATION | nice pg_restore --verbose  -U $POSTGRES_USER -F t -d $POSTGRES_DB
-# bzcat $DB_DUMP_LOCATION | nice pg_restore --verbose  -U $POSTGRES_USER -F c -d $POSTGRES_DB
+#bzcat $DB_DUMP_LOCATION | nice pg_restore --verbose  -U $POSTGRES_USER -F t -d $POSTGRES_DB
+bzcat $DB_DUMP_LOCATION | nice pg_restore --verbose  -U $POSTGRES_USER -F c -d $POSTGRES_DB
 
 echo "*** DATABASE CREATED ***"

django_api/Dockerfile-base

@@ -2,6 +2,10 @@
 FROM python:3.9.6-alpine3.14
 
 RUN apk update
+
+RUN apk add \
+    --update alpine-sdk
+
 RUN apk add --upgrade apk-tools \
     openssl \
     ca-certificates \
@@ -26,9 +30,6 @@ RUN apk add --no-cache --virtual .build-deps --update \
     gcc \
     g++
 
-
-# PYTHON
 RUN pip install --no-cache-dir --upgrade \
-    setuptools \
     pip \
     pipenv

django_api/Pipfile

@@ -41,6 +41,7 @@ djangorestframework-gis = "<=0.17"
 djangorestframework-simplejwt = "<=4.8"
 drfpasswordless = "<=1.5.7"
 greenlet = "<=1.1.1"
+jsonschema = "<=4.4.0"
 newrelic = "<=6.8.0.163"
 openpyxl = "<=3.0.7"
 psycopg2-binary = "<=2.9.1"

django_api/etools_prp/apps/core/models.py

@@ -13,7 +13,8 @@
 import mptt
 import pycountry
 from model_utils.models import TimeStampedModel
-from unicef_locations.models import AbstractLocation
+from mptt.managers import TreeManager
+from unicef_locations.models import AbstractLocation, LocationsManager
 
 from etools_prp.apps.utils.emails import send_email_from_template
 
@@ -457,6 +458,18 @@ def partner_activities(self, partner, clusters=None, limit=None):
         return qset
 
 
+class PRPLocationsManager(TreeManager):
+
+    def get_queryset(self):
+        return super().get_queryset().select_related('parent').defer('geom', 'point')
+
+    def active(self):
+        return self.get_queryset().filter(is_active=True)
+
+    def archived_locations(self):
+        return self.get_queryset().filter(is_active=False)
+
+
 class Location(AbstractLocation):
     external_id = models.CharField(
         help_text='An ID representing this instance in an external system',
@@ -469,6 +482,9 @@ class Location(AbstractLocation):
 
     workspaces = models.ManyToManyField(Workspace, related_name='locations')
 
+    objects = PRPLocationsManager()
+    super_objects = LocationsManager()
+
 
 mptt.register(Location, order_insertion_by=['name'])
 

django_api/etools_prp/apps/core/validators.py

@@ -1,14 +1,17 @@
 from django.contrib.contenttypes.models import ContentType
+from django.core import exceptions
+from django.utils.deconstruct import deconstructible
+from django.utils.translation import gettext_lazy as _
 
+from jsonschema import exceptions as jsonschema_exceptions, validate
 from rest_framework.exceptions import ValidationError
 
-from etools_prp.apps.cluster.models import ClusterActivity, ClusterObjective
-from etools_prp.apps.partner.models import PartnerActivity, PartnerActivityProjectContext, PartnerProject
-
 
 class AddIndicatorObjectTypeValidator:
 
     def __call__(self, value):
+        from etools_prp.apps.cluster.models import ClusterActivity, ClusterObjective
+        from etools_prp.apps.partner.models import PartnerActivity, PartnerActivityProjectContext, PartnerProject
         model_choices = {
             ClusterObjective,
             ClusterActivity,
@@ -26,3 +29,26 @@ def __call__(self, value):
 
 
 add_indicator_object_type_validator = AddIndicatorObjectTypeValidator()
+
+
+@deconstructible
+class JSONSchemaValidator:
+    message = _("Invalid JSON: %(value)s")
+    code = 'invalid_json'
+
+    def __init__(self, json_schema, message=None):
+        self.json_schema = json_schema
+        if message:
+            self.message = message
+
+    def __call__(self, value):
+        try:
+            validate(value, self.json_schema)
+        except jsonschema_exceptions.ValidationError as e:
+            raise exceptions.ValidationError(self.message, code=self.code, params={'value': e.message})
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and \
+            self.json_schema == other.json_schema and \
+            self.message == other.message and \
+            self.code == other.code

django_api/etools_prp/apps/indicator/json_schemas.py

@@ -0,0 +1,22 @@
+
+
+indicator_schema = {
+    "title": "Json schema for total, target, baseline and in_need fields",
+    "type": "object",
+    "additionalProperties": False,
+    "properties": {
+        "c": {"type": "number"},
+        "d": {"type": "number"},
+        "v": {"type": "number"}
+    },
+    "required": ["d", "v"]
+}
+
+disaggregation_schema = {
+    "title": "Disaggregation json schema",
+    "type": "object",
+    "additionalProperties": False,
+    "patternProperties": {
+        "^\((\d*,\s*)*\d*\)$": indicator_schema  # noqa W605
+    }
+}

django_api/etools_prp/apps/indicator/migrations/0009_auto_20220418_1714.py

@@ -0,0 +1,60 @@
+# Generated by Django 3.2.6 on 2022-04-18 17:14
+
+from django.db import migrations, models
+import etools_prp.apps.core.validators
+import etools_prp.apps.indicator.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('indicator', '0008_alter_indicatorlocationdata_disaggregation'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='indicatorlocationdata',
+            name='disaggregation',
+            field=models.JSONField(default=etools_prp.apps.indicator.models.default_disaggregation, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'patternProperties': {'^\\((\\d*,\\s*)*\\d*\\)$': {'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'}}, 'title': 'Disaggregation json schema', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='indicatorreport',
+            name='total',
+            field=models.JSONField(default=etools_prp.apps.indicator.models.default_total, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='reportable',
+            name='baseline',
+            field=models.JSONField(default=etools_prp.apps.indicator.models.default_value, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='reportable',
+            name='in_need',
+            field=models.JSONField(blank=True, null=True, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='reportable',
+            name='target',
+            field=models.JSONField(default=etools_prp.apps.indicator.models.default_value, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='reportable',
+            name='total',
+            field=models.JSONField(default=etools_prp.apps.indicator.models.default_total, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='reportablelocationgoal',
+            name='baseline',
+            field=models.JSONField(default=etools_prp.apps.indicator.models.default_value, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='reportablelocationgoal',
+            name='in_need',
+            field=models.JSONField(blank=True, null=True, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+        migrations.AlterField(
+            model_name='reportablelocationgoal',
+            name='target',
+            field=models.JSONField(default=etools_prp.apps.indicator.models.default_value, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]),
+        ),
+    ]

django_api/etools_prp/apps/indicator/models.py

@@ -25,8 +25,10 @@
     REPORTING_TYPES,
 )
 from etools_prp.apps.core.models import TimeStampedExternalSourceModel
+from etools_prp.apps.core.validators import JSONSchemaValidator
 from etools_prp.apps.indicator.constants import ValueType
 from etools_prp.apps.indicator.disaggregators import QuantityIndicatorDisaggregator, RatioIndicatorDisaggregator
+from etools_prp.apps.indicator.json_schemas import disaggregation_schema, indicator_schema
 from etools_prp.apps.indicator.utilities import convert_string_number_to_float
 from etools_prp.apps.partner.models import PartnerActivity
 from etools_prp.apps.utils.emails import send_email_from_template
@@ -241,9 +243,18 @@ class Reportable(TimeStampedExternalSourceModel):
         cluster.ClusterObjective (ForeignKey): "content_object"
         self (ForeignKey): "parent_indicator"
     """
-    target = models.JSONField(default=default_value)
-    baseline = models.JSONField(default=default_value)
-    in_need = models.JSONField(blank=True, null=True)
+    target = models.JSONField(
+        default=default_value,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
+    baseline = models.JSONField(
+        default=default_value,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
+    in_need = models.JSONField(
+        blank=True, null=True,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
     assumptions = models.TextField(null=True, blank=True)
     means_of_verification = models.CharField(max_length=255, null=True, blank=True)
     comments = models.TextField(max_length=4048, blank=True, null=True)
@@ -260,7 +271,7 @@ class Reportable(TimeStampedExternalSourceModel):
 
     # Current total, transactional and dynamically calculated based on
     # IndicatorReports
-    total = models.JSONField(default=default_total)
+    total = models.JSONField(default=default_total, validators=[JSONSchemaValidator(json_schema=indicator_schema)])
 
     # unique code for this indicator within the current context
     # eg: (1.1) result code 1 - indicator code 1
@@ -633,9 +644,18 @@ def clone_ca_reportable_to_pa_signal(sender, instance, created, **kwargs):
 class ReportableLocationGoal(TimeStampedModel):
     reportable = models.ForeignKey(Reportable, on_delete=models.CASCADE)
     location = models.ForeignKey("core.Location", on_delete=models.CASCADE)
-    target = models.JSONField(default=default_value)
-    baseline = models.JSONField(default=default_value)
-    in_need = models.JSONField(blank=True, null=True)
+    target = models.JSONField(
+        default=default_value,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
+    baseline = models.JSONField(
+        default=default_value,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
+    in_need = models.JSONField(
+        blank=True, null=True,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
     is_active = models.BooleanField(default=True)
 
     class Meta:
@@ -685,7 +705,10 @@ class IndicatorReport(TimeStampedModel):
         verbose_name='Frequency of reporting'
     )
 
-    total = models.JSONField(default=default_total)
+    total = models.JSONField(
+        default=default_total,
+        validators=[JSONSchemaValidator(json_schema=indicator_schema)]
+    )
 
     remarks = models.TextField(blank=True, null=True)
     report_status = models.CharField(
@@ -1068,7 +1091,10 @@ class IndicatorLocationData(TimeStampedModel):
         on_delete=models.CASCADE,
     )
 
-    disaggregation = models.JSONField(default=default_disaggregation)
+    disaggregation = models.JSONField(
+        default=default_disaggregation,
+        validators=[JSONSchemaValidator(json_schema=disaggregation_schema)]
+    )
     num_disaggregation = models.IntegerField()
     level_reported = models.IntegerField()
     disaggregation_reported_on = ArrayField(models.IntegerField(), default=list)