diff --git a/.circleci/config.yml b/.circleci/config.yml index 75f59ce20..e1f2b6b03 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -255,6 +255,8 @@ workflows: - master - staging - develop + - submodule-ref-2 + - build_and_deploy_fe_cluster: filters: branches: diff --git a/db/Dockerfile b/db/Dockerfile index d7ad8e4d1..ed5b91d40 100644 --- a/db/Dockerfile +++ b/db/Dockerfile @@ -3,7 +3,7 @@ FROM postgis/postgis:12-3.1 RUN apt-get update && apt-get install -y --no-install-recommends bzip2 # Use this if there is a db dump file -COPY ./db1.bz2 /tmp/psql_data/ + COPY load_db_data.sh /docker-entrypoint-initdb.d/20_load_db_data.sh EXPOSE 5432 diff --git a/db/load_db_data.sh b/db/load_db_data.sh index e89224901..82a88fa6c 100644 --- a/db/load_db_data.sh +++ b/db/load_db_data.sh @@ -2,10 +2,10 @@ set -e -export DB_DUMP_LOCATION=/tmp/psql_data/db1.bz2 +export DB_DUMP_LOCATION=/tmp/psql_data/db_dump.bz2 echo "*** RESTORING DATABASE $POSTGRES_DB ***" -bzcat $DB_DUMP_LOCATION | nice pg_restore --verbose -U $POSTGRES_USER -F t -d $POSTGRES_DB -# bzcat $DB_DUMP_LOCATION | nice pg_restore --verbose -U $POSTGRES_USER -F c -d $POSTGRES_DB +#bzcat $DB_DUMP_LOCATION | nice pg_restore --verbose -U $POSTGRES_USER -F t -d $POSTGRES_DB +bzcat $DB_DUMP_LOCATION | nice pg_restore --verbose -U $POSTGRES_USER -F c -d $POSTGRES_DB echo "*** DATABASE CREATED ***" diff --git a/django_api/Dockerfile-base b/django_api/Dockerfile-base index b87583b1b..1bb943572 100644 --- a/django_api/Dockerfile-base +++ b/django_api/Dockerfile-base @@ -2,6 +2,10 @@ FROM python:3.9.6-alpine3.14 RUN apk update + +RUN apk add \ + --update alpine-sdk + RUN apk add --upgrade apk-tools \ openssl \ ca-certificates \ @@ -26,9 +30,6 @@ RUN apk add --no-cache --virtual .build-deps --update \ gcc \ g++ - -# PYTHON RUN pip install --no-cache-dir --upgrade \ - setuptools \ pip \ pipenv \ No newline at end of file diff --git a/django_api/Pipfile b/django_api/Pipfile index 7050ef42f..c9d241acb 100644 --- a/django_api/Pipfile +++ b/django_api/Pipfile @@ -41,6 +41,7 @@ djangorestframework-gis = "<=0.17" djangorestframework-simplejwt = "<=4.8" drfpasswordless = "<=1.5.7" greenlet = "<=1.1.1" +jsonschema = "<=4.4.0" newrelic = "<=6.8.0.163" openpyxl = "<=3.0.7" psycopg2-binary = "<=2.9.1" diff --git a/django_api/Pipfile.lock b/django_api/Pipfile.lock index 9e7fd0c54..e29969ad3 100644 --- a/django_api/Pipfile.lock +++ b/django_api/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c87c68216102318202334a7e9a5d442cbee7643bb5ae4ecbeb8fe99073a3a181" + "sha256": "d167725f05c9e1d536b268a65ac8b150d14025f8a7bb519deada2d05f422b14d" }, "pipfile-spec": 6, "requires": { @@ -715,6 +715,14 @@ ], "version": "==3.1.0" }, + "jsonschema": { + "hashes": [ + "sha256:636694eb41b3535ed608fe04129f26542b59ed99808b4f688aa32dcf55317a83", + "sha256:77281a1f71684953ee8b3d488371b162419767973789272434bbc3f29d9c8823" + ], + "index": "pypi", + "version": "==4.4.0" + }, "kombu": { "hashes": [ "sha256:37cee3ee725f94ea8bb173eaab7c1760203ea53bbebae226328600f9d2799610", @@ -1016,6 +1024,33 @@ "index": "pypi", "version": "==0.6.11" }, + "pyrsistent": { + "hashes": [ + "sha256:0e3e1fcc45199df76053026a51cc59ab2ea3fc7c094c6627e93b7b44cdae2c8c", + "sha256:1b34eedd6812bf4d33814fca1b66005805d3640ce53140ab8bbb1e2651b0d9bc", + "sha256:4ed6784ceac462a7d6fcb7e9b663e93b9a6fb373b7f43594f9ff68875788e01e", + "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26", + "sha256:636ce2dc235046ccd3d8c56a7ad54e99d5c1cd0ef07d9ae847306c91d11b5fec", + "sha256:6455fc599df93d1f60e1c5c4fe471499f08d190d57eca040c0ea182301321286", + "sha256:6bc66318fb7ee012071b2792024564973ecc80e9522842eb4e17743604b5e045", + "sha256:7bfe2388663fd18bd8ce7db2c91c7400bf3e1a9e8bd7d63bf7e77d39051b85ec", + "sha256:7ec335fc998faa4febe75cc5268a9eac0478b3f681602c1f27befaf2a1abe1d8", + "sha256:914474c9f1d93080338ace89cb2acee74f4f666fb0424896fcfb8d86058bf17c", + "sha256:b568f35ad53a7b07ed9b1b2bae09eb15cdd671a5ba5d2c66caee40dbf91c68ca", + "sha256:cdfd2c361b8a8e5d9499b9082b501c452ade8bbf42aef97ea04854f4a3f43b22", + "sha256:d1b96547410f76078eaf66d282ddca2e4baae8964364abb4f4dcdde855cd123a", + "sha256:d4d61f8b993a7255ba714df3aca52700f8125289f84f704cf80916517c46eb96", + "sha256:d7a096646eab884bf8bed965bad63ea327e0d0c38989fc83c5ea7b8a87037bfc", + "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1", + "sha256:e24a828f57e0c337c8d8bb9f6b12f09dfdf0273da25fda9e314f0b684b415a07", + "sha256:e4f3149fd5eb9b285d6bfb54d2e5173f6a116fe19172686797c056672689daf6", + "sha256:e92a52c166426efbe0d1ec1332ee9119b6d32fc1f0bbfd55d5c1088070e7fc1b", + "sha256:f87cc2863ef33c709e237d4b5f4502a62a00fab450c9e020892e8e2ede5847f5", + "sha256:fd8da6d0124efa2f67d86fa70c851022f87c98e205f0594e1fae044e7119a5a6" + ], + "markers": "python_version >= '3.7'", + "version": "==0.18.1" + }, "python-crontab": { "hashes": [ "sha256:1e35ed7a3cdc3100545b43e196d34754e6551e7f95e4caebbe0e1c0ca41c2f1b" diff --git a/django_api/etools_prp/apps/core/models.py b/django_api/etools_prp/apps/core/models.py index d1f5bfb3b..aa4bcae7e 100644 --- a/django_api/etools_prp/apps/core/models.py +++ b/django_api/etools_prp/apps/core/models.py @@ -13,7 +13,8 @@ import mptt import pycountry from model_utils.models import TimeStampedModel -from unicef_locations.models import AbstractLocation +from mptt.managers import TreeManager +from unicef_locations.models import AbstractLocation, LocationsManager from etools_prp.apps.utils.emails import send_email_from_template @@ -457,6 +458,18 @@ def partner_activities(self, partner, clusters=None, limit=None): return qset +class PRPLocationsManager(TreeManager): + + def get_queryset(self): + return super().get_queryset().select_related('parent').defer('geom', 'point') + + def active(self): + return self.get_queryset().filter(is_active=True) + + def archived_locations(self): + return self.get_queryset().filter(is_active=False) + + class Location(AbstractLocation): external_id = models.CharField( help_text='An ID representing this instance in an external system', @@ -469,6 +482,9 @@ class Location(AbstractLocation): workspaces = models.ManyToManyField(Workspace, related_name='locations') + objects = PRPLocationsManager() + super_objects = LocationsManager() + mptt.register(Location, order_insertion_by=['name']) diff --git a/django_api/etools_prp/apps/core/validators.py b/django_api/etools_prp/apps/core/validators.py index 96fca50fb..d5c034e43 100644 --- a/django_api/etools_prp/apps/core/validators.py +++ b/django_api/etools_prp/apps/core/validators.py @@ -1,14 +1,17 @@ from django.contrib.contenttypes.models import ContentType +from django.core import exceptions +from django.utils.deconstruct import deconstructible +from django.utils.translation import gettext_lazy as _ +from jsonschema import exceptions as jsonschema_exceptions, validate from rest_framework.exceptions import ValidationError -from etools_prp.apps.cluster.models import ClusterActivity, ClusterObjective -from etools_prp.apps.partner.models import PartnerActivity, PartnerActivityProjectContext, PartnerProject - class AddIndicatorObjectTypeValidator: def __call__(self, value): + from etools_prp.apps.cluster.models import ClusterActivity, ClusterObjective + from etools_prp.apps.partner.models import PartnerActivity, PartnerActivityProjectContext, PartnerProject model_choices = { ClusterObjective, ClusterActivity, @@ -26,3 +29,26 @@ def __call__(self, value): add_indicator_object_type_validator = AddIndicatorObjectTypeValidator() + + +@deconstructible +class JSONSchemaValidator: + message = _("Invalid JSON: %(value)s") + code = 'invalid_json' + + def __init__(self, json_schema, message=None): + self.json_schema = json_schema + if message: + self.message = message + + def __call__(self, value): + try: + validate(value, self.json_schema) + except jsonschema_exceptions.ValidationError as e: + raise exceptions.ValidationError(self.message, code=self.code, params={'value': e.message}) + + def __eq__(self, other): + return isinstance(other, self.__class__) and \ + self.json_schema == other.json_schema and \ + self.message == other.message and \ + self.code == other.code diff --git a/django_api/etools_prp/apps/indicator/json_schemas.py b/django_api/etools_prp/apps/indicator/json_schemas.py new file mode 100644 index 000000000..58e91eb53 --- /dev/null +++ b/django_api/etools_prp/apps/indicator/json_schemas.py @@ -0,0 +1,22 @@ + + +indicator_schema = { + "title": "Json schema for total, target, baseline and in_need fields", + "type": "object", + "additionalProperties": False, + "properties": { + "c": {"type": "number"}, + "d": {"type": "number"}, + "v": {"type": "number"} + }, + "required": ["d", "v"] +} + +disaggregation_schema = { + "title": "Disaggregation json schema", + "type": "object", + "additionalProperties": False, + "patternProperties": { + "^\((\d*,\s*)*\d*\)$": indicator_schema # noqa W605 + } +} diff --git a/django_api/etools_prp/apps/indicator/migrations/0009_auto_20220418_1714.py b/django_api/etools_prp/apps/indicator/migrations/0009_auto_20220418_1714.py new file mode 100644 index 000000000..61467ee3f --- /dev/null +++ b/django_api/etools_prp/apps/indicator/migrations/0009_auto_20220418_1714.py @@ -0,0 +1,60 @@ +# Generated by Django 3.2.6 on 2022-04-18 17:14 + +from django.db import migrations, models +import etools_prp.apps.core.validators +import etools_prp.apps.indicator.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('indicator', '0008_alter_indicatorlocationdata_disaggregation'), + ] + + operations = [ + migrations.AlterField( + model_name='indicatorlocationdata', + name='disaggregation', + field=models.JSONField(default=etools_prp.apps.indicator.models.default_disaggregation, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'patternProperties': {'^\\((\\d*,\\s*)*\\d*\\)$': {'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'}}, 'title': 'Disaggregation json schema', 'type': 'object'})]), + ), + migrations.AlterField( + model_name='indicatorreport', + name='total', + field=models.JSONField(default=etools_prp.apps.indicator.models.default_total, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]), + ), + migrations.AlterField( + model_name='reportable', + name='baseline', + field=models.JSONField(default=etools_prp.apps.indicator.models.default_value, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]), + ), + migrations.AlterField( + model_name='reportable', + name='in_need', + field=models.JSONField(blank=True, null=True, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]), + ), + migrations.AlterField( + model_name='reportable', + name='target', + field=models.JSONField(default=etools_prp.apps.indicator.models.default_value, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]), + ), + migrations.AlterField( + model_name='reportable', + name='total', + field=models.JSONField(default=etools_prp.apps.indicator.models.default_total, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]), + ), + migrations.AlterField( + model_name='reportablelocationgoal', + name='baseline', + field=models.JSONField(default=etools_prp.apps.indicator.models.default_value, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]), + ), + migrations.AlterField( + model_name='reportablelocationgoal', + name='in_need', + field=models.JSONField(blank=True, null=True, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]), + ), + migrations.AlterField( + model_name='reportablelocationgoal', + name='target', + field=models.JSONField(default=etools_prp.apps.indicator.models.default_value, validators=[etools_prp.apps.core.validators.JSONSchemaValidator(json_schema={'additionalProperties': False, 'properties': {'c': {'type': 'number'}, 'd': {'type': 'number'}, 'v': {'type': 'number'}}, 'required': ['d', 'v'], 'title': 'Json schema for total, target, baseline and in_need fields', 'type': 'object'})]), + ), + ] diff --git a/django_api/etools_prp/apps/indicator/models.py b/django_api/etools_prp/apps/indicator/models.py index ffdd0a73a..947fdd1f8 100644 --- a/django_api/etools_prp/apps/indicator/models.py +++ b/django_api/etools_prp/apps/indicator/models.py @@ -25,8 +25,10 @@ REPORTING_TYPES, ) from etools_prp.apps.core.models import TimeStampedExternalSourceModel +from etools_prp.apps.core.validators import JSONSchemaValidator from etools_prp.apps.indicator.constants import ValueType from etools_prp.apps.indicator.disaggregators import QuantityIndicatorDisaggregator, RatioIndicatorDisaggregator +from etools_prp.apps.indicator.json_schemas import disaggregation_schema, indicator_schema from etools_prp.apps.indicator.utilities import convert_string_number_to_float from etools_prp.apps.partner.models import PartnerActivity from etools_prp.apps.utils.emails import send_email_from_template @@ -241,9 +243,18 @@ class Reportable(TimeStampedExternalSourceModel): cluster.ClusterObjective (ForeignKey): "content_object" self (ForeignKey): "parent_indicator" """ - target = models.JSONField(default=default_value) - baseline = models.JSONField(default=default_value) - in_need = models.JSONField(blank=True, null=True) + target = models.JSONField( + default=default_value, + validators=[JSONSchemaValidator(json_schema=indicator_schema)] + ) + baseline = models.JSONField( + default=default_value, + validators=[JSONSchemaValidator(json_schema=indicator_schema)] + ) + in_need = models.JSONField( + blank=True, null=True, + validators=[JSONSchemaValidator(json_schema=indicator_schema)] + ) assumptions = models.TextField(null=True, blank=True) means_of_verification = models.CharField(max_length=255, null=True, blank=True) comments = models.TextField(max_length=4048, blank=True, null=True) @@ -260,7 +271,7 @@ class Reportable(TimeStampedExternalSourceModel): # Current total, transactional and dynamically calculated based on # IndicatorReports - total = models.JSONField(default=default_total) + total = models.JSONField(default=default_total, validators=[JSONSchemaValidator(json_schema=indicator_schema)]) # unique code for this indicator within the current context # eg: (1.1) result code 1 - indicator code 1 @@ -633,9 +644,18 @@ def clone_ca_reportable_to_pa_signal(sender, instance, created, **kwargs): class ReportableLocationGoal(TimeStampedModel): reportable = models.ForeignKey(Reportable, on_delete=models.CASCADE) location = models.ForeignKey("core.Location", on_delete=models.CASCADE) - target = models.JSONField(default=default_value) - baseline = models.JSONField(default=default_value) - in_need = models.JSONField(blank=True, null=True) + target = models.JSONField( + default=default_value, + validators=[JSONSchemaValidator(json_schema=indicator_schema)] + ) + baseline = models.JSONField( + default=default_value, + validators=[JSONSchemaValidator(json_schema=indicator_schema)] + ) + in_need = models.JSONField( + blank=True, null=True, + validators=[JSONSchemaValidator(json_schema=indicator_schema)] + ) is_active = models.BooleanField(default=True) class Meta: @@ -685,7 +705,10 @@ class IndicatorReport(TimeStampedModel): verbose_name='Frequency of reporting' ) - total = models.JSONField(default=default_total) + total = models.JSONField( + default=default_total, + validators=[JSONSchemaValidator(json_schema=indicator_schema)] + ) remarks = models.TextField(blank=True, null=True) report_status = models.CharField( @@ -1068,7 +1091,10 @@ class IndicatorLocationData(TimeStampedModel): on_delete=models.CASCADE, ) - disaggregation = models.JSONField(default=default_disaggregation) + disaggregation = models.JSONField( + default=default_disaggregation, + validators=[JSONSchemaValidator(json_schema=disaggregation_schema)] + ) num_disaggregation = models.IntegerField() level_reported = models.IntegerField() disaggregation_reported_on = ArrayField(models.IntegerField(), default=list) diff --git a/django_api/etools_prp/apps/indicator/serializers.py b/django_api/etools_prp/apps/indicator/serializers.py index eb6f9d15c..308b2250a 100644 --- a/django_api/etools_prp/apps/indicator/serializers.py +++ b/django_api/etools_prp/apps/indicator/serializers.py @@ -25,12 +25,13 @@ ) from etools_prp.apps.core.models import Location from etools_prp.apps.core.serializers import IdLocationSerializer, LocationSerializer -from etools_prp.apps.core.validators import add_indicator_object_type_validator +from etools_prp.apps.core.validators import add_indicator_object_type_validator, JSONSchemaValidator from etools_prp.apps.ocha.imports.serializers import DiscardUniqueTogetherValidationMixin from etools_prp.apps.partner.models import Partner, PartnerActivity, PartnerActivityProjectContext, PartnerProject from etools_prp.apps.unicef.models import LowerLevelOutput, ProgressReport from .fields import SortedDateArrayField +from .json_schemas import disaggregation_schema, indicator_schema from .models import ( create_pa_reportables_for_new_ca_reportable, Disaggregation, @@ -227,9 +228,16 @@ def update(self, instance, validated_data): class ReportableLocationGoalBaselineInNeedSerializer(serializers.ModelSerializer): id = serializers.IntegerField() - baseline = serializers.JSONField() - in_need = serializers.JSONField(required=False, allow_null=True) - target = serializers.JSONField() + baseline = serializers.JSONField( + validators=[JSONSchemaValidator(json_schema=indicator_schema)] + ) + in_need = serializers.JSONField( + required=False, allow_null=True, + validators=[JSONSchemaValidator(json_schema=indicator_schema)] + ) + target = serializers.JSONField( + validators=[JSONSchemaValidator(json_schema=indicator_schema)] + ) location = LocationSerializer(read_only=True) def validate(self, data): @@ -594,7 +602,9 @@ class IndicatorLocationDataUpdateSerializer(serializers.ModelSerializer): disaggregation_reported_on = serializers.ListField( child=serializers.IntegerField() ) - disaggregation = serializers.JSONField() + disaggregation = serializers.JSONField( + validators=[JSONSchemaValidator(json_schema=disaggregation_schema)] + ) reporting_entity_percentage_map = serializers.JSONField( required=False, ) @@ -1126,8 +1136,8 @@ class ClusterObjectiveIndicatorAdoptSerializer(serializers.Serializer): cluster_objective_id = serializers.IntegerField() reportable_id = serializers.IntegerField() locations = ReportableLocationGoalSerializer(many=True, write_only=True) - target = serializers.JSONField() - baseline = serializers.JSONField() + target = serializers.JSONField(validators=[JSONSchemaValidator(json_schema=indicator_schema)]) + baseline = serializers.JSONField(validators=[JSONSchemaValidator(json_schema=indicator_schema)]) def validate(self, data): """ @@ -1245,12 +1255,17 @@ class ClusterIndicatorSerializer(serializers.ModelSerializer): disaggregations = IdDisaggregationSerializer(many=True, read_only=True) object_type = serializers.CharField( - validators=[add_indicator_object_type_validator], write_only=True) + validators=[add_indicator_object_type_validator], write_only=True + ) blueprint = IndicatorBlueprintSerializer() locations = ReportableLocationGoalSerializer(many=True, write_only=True) - target = serializers.JSONField() - baseline = serializers.JSONField() - in_need = serializers.JSONField(required=False, allow_null=True) + target = serializers.JSONField(validators=[JSONSchemaValidator(json_schema=indicator_schema)]) + baseline = serializers.JSONField(validators=[JSONSchemaValidator(json_schema=indicator_schema)]) + in_need = serializers.JSONField( + required=False, + allow_null=True, + validators=[JSONSchemaValidator(json_schema=indicator_schema)] + ) project_context_id = serializers.IntegerField(write_only=True, required=False, allow_null=True) cs_dates = SortedDateArrayField(child=serializers.DateField(), required=False) diff --git a/django_api/etools_prp/apps/indicator/tests/test_utilities.py b/django_api/etools_prp/apps/indicator/tests/test_utilities.py index 6a9ec9464..917652df9 100644 --- a/django_api/etools_prp/apps/indicator/tests/test_utilities.py +++ b/django_api/etools_prp/apps/indicator/tests/test_utilities.py @@ -60,3 +60,17 @@ def test_ratio_missing_denominator(self): format_total_value_to_string(total, True, "ratio"), "300/1", ) + + def test_string_denominator_value_percentage(self): + total = {"d": "100", "v": "30"} + self.assertEqual( + format_total_value_to_string(total, True, None), + "30%", + ) + + def test_string_denominator_value_ratio(self): + total = {"d": "100", "v": "300"} + self.assertEqual( + format_total_value_to_string(total, True, "ratio"), + "300/100", + ) diff --git a/django_api/etools_prp/apps/indicator/tests/test_views.py b/django_api/etools_prp/apps/indicator/tests/test_views.py index e1092edce..c5a784ccb 100644 --- a/django_api/etools_prp/apps/indicator/tests/test_views.py +++ b/django_api/etools_prp/apps/indicator/tests/test_views.py @@ -1612,8 +1612,8 @@ def test_update_nonnumeric_data_entry_validation(self): self.assertEquals(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertIn( - "c value is not number", - response.data['non_field_errors'][0] + "'aaaa' is not of type 'number'", + response.data['disaggregation'][0] ) update_data['disaggregation'][str(level_reported_3_key)] = copy.deepcopy(validated_data) @@ -1622,8 +1622,8 @@ def test_update_nonnumeric_data_entry_validation(self): self.assertEquals(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertIn( - "d value is not number", - response.data['non_field_errors'][0] + "'aaaa' is not of type 'number'", + response.data['disaggregation'][0] ) update_data['disaggregation'][str(level_reported_3_key)] = copy.deepcopy(validated_data) @@ -1632,8 +1632,8 @@ def test_update_nonnumeric_data_entry_validation(self): self.assertEquals(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertIn( - "v value is not number", - response.data['non_field_errors'][0] + "'aaaa' is not of type 'number'", + response.data['disaggregation'][0] ) def test_update_zero_division_data_entry_validation_not_on_quantity(self): @@ -1811,7 +1811,7 @@ def test_update_higher_coordinate_space_key_validation(self): del update_data['disaggregation'][str(level_reported_3_key)] level_reported_3_key = list(level_reported_3_key) level_reported_3_key.append(next_disaggregation_value_id) - update_data['disaggregation'][str(tuple(level_reported_3_key))] = {} + update_data['disaggregation'][str(tuple(level_reported_3_key))] = {'d': 0, 'v': 0} url = reverse('indicator-location-data-entries-put-api') response = self.client.put(url, update_data, format='json') @@ -1845,7 +1845,7 @@ def test_update_invalid_coordinate_space_key_validation(self): level_reported_3_key = list(level_reported_3_key[:-1]) level_reported_3_key.append(next_disaggregation_value_id) - update_data['disaggregation'][str(tuple(level_reported_3_key))] = {} + update_data['disaggregation'][str(tuple(level_reported_3_key))] = {'d': 0, 'v': 0} url = reverse('indicator-location-data-entries-put-api') response = self.client.put(url, update_data, format='json') @@ -1881,8 +1881,8 @@ def test_update_invalid_coordinate_space_key_format_validation(self): self.assertEquals(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertIn( - "key is not in tuple format", - response.data['non_field_errors'][0] + "'bad key' does not match", + response.data['disaggregation'][0] ) def test_update_invalid_coordinate_space_value_format_validation(self): @@ -1900,7 +1900,7 @@ def test_update_invalid_coordinate_space_value_format_validation(self): level_reported_3_key = key break - update_data['disaggregation'][str(level_reported_3_key)] = {} + update_data['disaggregation'][str(level_reported_3_key)] = {'d': 0, 'v': 0} url = reverse('indicator-location-data-entries-put-api') response = self.client.put(url, update_data, format='json') @@ -2373,12 +2373,23 @@ def test_invalid_serializer_values(self): response = self.client.post(url, data=data, format='json') self.assertTrue(status.is_client_error(response.status_code)) + # Target value is string + data['target'] = {'d': 1, 'v': '1', 'c': 1} + response = self.client.post(url, data=data, format='json') + self.assertTrue(status.is_client_error(response.status_code)) + # Baseline value type check data['target'] = {'d': 1, 'v': 1, 'c': 1} data['baseline'] = list() response = self.client.post(url, data=data, format='json') self.assertTrue(status.is_client_error(response.status_code)) + # Baseline value is string + data['target'] = {'d': 1, 'v': 1, 'c': 1} + data['baseline'] = {'d': 0, 'v': '1', 'c': 1} + response = self.client.post(url, data=data, format='json') + self.assertTrue(status.is_client_error(response.status_code)) + # zero d value in target data['baseline'] = {'d': 1, 'v': 1, 'c': 1} data['target'] = {'d': 0, 'v': 1, 'c': 1} diff --git a/django_api/etools_prp/apps/indicator/utilities.py b/django_api/etools_prp/apps/indicator/utilities.py index cf67bdaeb..677f798fe 100644 --- a/django_api/etools_prp/apps/indicator/utilities.py +++ b/django_api/etools_prp/apps/indicator/utilities.py @@ -12,11 +12,11 @@ def convert_string_number_to_float(num): def format_total_value_to_string(total, is_percentage=False, percentage_display_type=None): - value = total.get(ValueType.VALUE, 0) + value = int(convert_string_number_to_float(total.get(ValueType.VALUE, 0))) locale = to_locale(get_language()) if is_percentage: - denominator = total.get(ValueType.DENOMINATOR, 1) + denominator = int(convert_string_number_to_float(total.get(ValueType.DENOMINATOR, 1))) if percentage_display_type and percentage_display_type == 'ratio': return f"{value}/{denominator}" else: diff --git a/django_api/etools_prp/apps/unicef/exports/annex_c_excel.py b/django_api/etools_prp/apps/unicef/exports/annex_c_excel.py index cb4b1dc4f..29e91ff43 100644 --- a/django_api/etools_prp/apps/unicef/exports/annex_c_excel.py +++ b/django_api/etools_prp/apps/unicef/exports/annex_c_excel.py @@ -118,10 +118,12 @@ def get_general_info_row(self, progress_report, location_data): if indicator_report.is_percentage: indicator_report_value_format = FORMAT_PERCENTAGE - achievement_in_reporting_period = indicator_report.total.get(ValueType.CALCULATED, 0) - total_cumulative_progress = indicator_report.reportable.achieved.get( + # fixing percent number format: 0.1 is equivalent to 10% in math, openpyxl works the same way + # e.g. a calculated value of 88.72 becomes 8872% on xls export, that's why it needs divided by 100 + achievement_in_reporting_period = int(indicator_report.total.get(ValueType.CALCULATED, 0)) / 100 + total_cumulative_progress = int(indicator_report.reportable.achieved.get( ValueType.CALCULATED, 0 - ) + )) / 100 else: indicator_report_value_format = None achievement_in_reporting_period = indicator_report.total.get(ValueType.VALUE, 0) diff --git a/django_api/etools_prp/config/settings.py b/django_api/etools_prp/config/settings.py index f01287d7d..62a84185f 100644 --- a/django_api/etools_prp/config/settings.py +++ b/django_api/etools_prp/config/settings.py @@ -495,7 +495,16 @@ # JWT Authentication # production overrides for django-rest-framework-jwt if not DISABLE_JWT_AUTH: - with open(os.path.join(BASE_DIR, 'keys/jwt/certificate.txt'), 'rb') as public_key: + cert_path = "keys/jwt/certificate.txt" + if all([AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY, AZURE_CONTAINER]): + cert_path = "keys/jwt/certificate.pem" + from storages.backends.azure_storage import AzureStorage + storage = AzureStorage() + with storage.open('keys/jwt/certificate.pem') as jwt_cert: + with open(os.path.join(BASE_DIR, 'keys/jwt/certificate.pem'), 'wb+') as new_jwt_cert: + new_jwt_cert.write(jwt_cert.read()) + + with open(os.path.join(BASE_DIR, cert_path), 'rb') as public_key: public_key_text = public_key.read() # noqa: F405 certificate = load_pem_x509_certificate(public_key_text, default_backend()) diff --git a/docker-compose.yml b/docker-compose.yml index 4ea0514cf..33e6bef0f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,8 @@ version: '3.8' +volumes: + pg-data-prp: {} + services: proxy: @@ -88,6 +91,9 @@ services: build: context: ./db dockerfile: ./Dockerfile + volumes: + - pg-data-prp:/var/lib/postgresql/data/ + - ./db/db_dump.bz2:/tmp/psql_data/db_dump.bz2 environment: POSTGRES_HOST_AUTH_METHOD: trust diff --git a/frontend_cluster/Dockerfile-bundle b/frontend_cluster/Dockerfile-bundle index ce1619b10..86656335e 100644 --- a/frontend_cluster/Dockerfile-bundle +++ b/frontend_cluster/Dockerfile-bundle @@ -25,7 +25,7 @@ RUN apk add --update bash WORKDIR /code RUN npm install express@4.17.x -RUN npm install browser-capabilities@1.1.3 +RUN npm install browser-capabilities@1.1.x COPY --from=builder /code/express.js /code/express.js COPY --from=builder /code/build /code/build EXPOSE 8082 diff --git a/frontend_cluster/src_ts/endpoints.ts b/frontend_cluster/src_ts/endpoints.ts index 90ab75ad2..c335aff5f 100644 --- a/frontend_cluster/src_ts/endpoints.ts +++ b/frontend_cluster/src_ts/endpoints.ts @@ -191,7 +191,7 @@ const Endpoints = { }, userSignOut() { - return this._buildUrl('/account/user-logout/'); + return this._buildUrl('/social/unicef-logout/'); }, userLogin() { diff --git a/frontend_ip/Dockerfile-bundle b/frontend_ip/Dockerfile-bundle index 57776636b..cfcf2007e 100644 --- a/frontend_ip/Dockerfile-bundle +++ b/frontend_ip/Dockerfile-bundle @@ -26,7 +26,7 @@ RUN apk add --update bash WORKDIR /code RUN npm install express -RUN npm install browser-capabilities@1.1.3 +RUN npm install browser-capabilities@1.1.x COPY --from=builder /code/express.js /code/express.js COPY --from=builder /code/build /code/build EXPOSE 8082 diff --git a/frontend_ip/express.js b/frontend_ip/express.js index 611da62d3..c7a1058cb 100644 --- a/frontend_ip/express.js +++ b/frontend_ip/express.js @@ -1,17 +1,24 @@ var express = require('express'); // eslint-disable-line var browserCapabilities = require('browser-capabilities'); // eslint-disable-line +const UAParser = require('ua-parser-js').UAParser; // eslint-disable-line const app = express(); const basedir = __dirname + '/build/'; // eslint-disable-line function getSourcesPath(request) { - let clientCapabilities = browserCapabilities.browserCapabilities(request.headers['user-agent']); + const userAgent = request.headers['user-agent']; + let clientCapabilities = browserCapabilities.browserCapabilities(userAgent); + const isEdge = (new UAParser(userAgent).getBrowser().name || '') === 'Edge'; clientCapabilities = new Set(clientCapabilities); // eslint-disable-line if (clientCapabilities.has('modules')) { return basedir + 'esm-bundled/'; } else { - return basedir + 'es6-bundled/'; + if (isEdge) { + return basedir + 'esm-bundled/'; + } else { + return basedir + 'es6-bundled/'; + } } } @@ -19,7 +26,7 @@ app.use('/ip/', (req, res, next) => { express.static(getSourcesPath(req))(req, res, next); }); -app.get(/.*service-worker\.js/, function(req, res) { +app.get(/.*service-worker\.js/, function (req, res) { res.sendFile(getSourcesPath(req) + 'service-worker.js'); }); diff --git a/frontend_ip/package-lock.json b/frontend_ip/package-lock.json index 7617f4d0b..5636387ea 100644 --- a/frontend_ip/package-lock.json +++ b/frontend_ip/package-lock.json @@ -1597,6 +1597,13 @@ "requires": { "@types/ua-parser-js": "^0.7.31", "ua-parser-js": "^0.7.15" + }, + "dependencies": { + "ua-parser-js": { + "version": "0.7.31", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.31.tgz", + "integrity": "sha512-qLK/Xe9E2uzmYI3qLeOmI0tEOt+TBBQyUIAh4aAgU05FVYzeZrKUdkAZfBNVGRaHVgV0TDkdEngJSw/SyQchkQ==" + } } }, "buffer-equal": { @@ -19352,11 +19359,6 @@ "integrity": "sha512-VgYs2A2QIRuGphtzFV7aQJduJ2gyfTljngLzjpfW9FoYZF6xuw1W0vW9ghCKLfcWrCFxK81CSGRAvS1pn4fIUg==", "dev": true }, - "ua-parser-js": { - "version": "0.7.31", - "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.31.tgz", - "integrity": "sha512-qLK/Xe9E2uzmYI3qLeOmI0tEOt+TBBQyUIAh4aAgU05FVYzeZrKUdkAZfBNVGRaHVgV0TDkdEngJSw/SyQchkQ==" - }, "unc-path-regex": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/unc-path-regex/-/unc-path-regex-0.1.2.tgz", diff --git a/frontend_ip/src_ts/elements/filters/dropdown-filter/dropdown-filter-multi.ts b/frontend_ip/src_ts/elements/filters/dropdown-filter/dropdown-filter-multi.ts index 41a7a04f3..07b6f43f6 100644 --- a/frontend_ip/src_ts/elements/filters/dropdown-filter/dropdown-filter-multi.ts +++ b/frontend_ip/src_ts/elements/filters/dropdown-filter/dropdown-filter-multi.ts @@ -26,7 +26,7 @@ class DropdownFilterMulti extends FilterMixin(PolymerElement) { label="[[label]]" options="[[data]]" option-value="id" - option-label="title" + option-label="[[optionLabel]]" selected-values="{{selectedValues}}" trigger-value-change-event on-etools-selected-items-changed="_handleChange" @@ -52,6 +52,9 @@ class DropdownFilterMulti extends FilterMixin(PolymerElement) { @property({type: Array}) selectedValues = []; + @property({type: String}) + optionLabel = 'title'; + public static get observers() { return ['_setSelectedValues(value, data)']; } diff --git a/frontend_ip/src_ts/elements/filters/location-filter-multi/location-filter-multi.ts b/frontend_ip/src_ts/elements/filters/location-filter-multi/location-filter-multi.ts index 3981f24b1..96866c7c7 100644 --- a/frontend_ip/src_ts/elements/filters/location-filter-multi/location-filter-multi.ts +++ b/frontend_ip/src_ts/elements/filters/location-filter-multi/location-filter-multi.ts @@ -26,7 +26,13 @@ class LocationFilterMulti extends LocalizeMixin(FilterDependenciesMixin(ReduxCon - + `; } diff --git a/frontend_ip/src_ts/elements/filters/text-filter/text-filter.ts b/frontend_ip/src_ts/elements/filters/text-filter/text-filter.ts index 93ab2c7de..ae838abb8 100644 --- a/frontend_ip/src_ts/elements/filters/text-filter/text-filter.ts +++ b/frontend_ip/src_ts/elements/filters/text-filter/text-filter.ts @@ -44,15 +44,13 @@ class TextFilter extends FilterMixin(ReduxConnectedElement) { _filterValueChanged() { this._debouncer = Debouncer.debounce(this._debouncer, timeOut.after(250), () => { - if ((this.$.field as PaperInputElement).value) { - const newValue = (this.$.field as PaperInputElement).value!.trim(); + const newValue = (this.$.field as PaperInputElement).value!.trim(); - if (newValue !== this.lastValue) { - fireEvent(this, 'filter-changed', { - name: this.name, - value: newValue - }); - } + if (newValue !== this.lastValue) { + fireEvent(this, 'filter-changed', { + name: this.name, + value: newValue + }); } }); } diff --git a/frontend_ip/src_ts/elements/ip-reporting/js/pd-details-calculation-methods-functions.ts b/frontend_ip/src_ts/elements/ip-reporting/js/pd-details-calculation-methods-functions.ts index afdac5be9..5e77afc7e 100644 --- a/frontend_ip/src_ts/elements/ip-reporting/js/pd-details-calculation-methods-functions.ts +++ b/frontend_ip/src_ts/elements/ip-reporting/js/pd-details-calculation-methods-functions.ts @@ -36,7 +36,7 @@ export function computeFormattedData(data: GenericObject) { } export function computeSelected(data: GenericObject, scope: string) { - return data[scope]; + return (data.display_type === 'ratio' && scope === 'calculation_formula_across_periods') ? 'latest' : data[scope]; } export function computeDisabled(display_type: string) { diff --git a/frontend_ip/src_ts/elements/ip-reporting/pd-details-calculation-methods.ts b/frontend_ip/src_ts/elements/ip-reporting/pd-details-calculation-methods.ts index cd6c19eb6..a8079ca88 100644 --- a/frontend_ip/src_ts/elements/ip-reporting/pd-details-calculation-methods.ts +++ b/frontend_ip/src_ts/elements/ip-reporting/pd-details-calculation-methods.ts @@ -94,6 +94,14 @@ class PdDetailsCalculationMethods extends LocalizeMixin( margin-left: 40px; font-weight: normal; } + + [hidden] { + display: none !important; + } + + paper-radio-button[name='latest'] { + text-transform: uppercase; + } @@ -189,6 +197,13 @@ class PdDetailsCalculationMethods extends LocalizeMixin( [[localize('avg')]] + + [[localize('latest')]] +