diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index 76562c818b..b3041deb89 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -22,7 +22,7 @@ import os import uuid from io import BytesIO, StringIO -from typing import Optional, Union +from typing import Optional from xml.etree.ElementTree import Element, SubElement import geojson @@ -43,7 +43,7 @@ javarosa_to_geojson_geom, parse_and_filter_geojson, ) -from app.models.enums import HTTPStatus, XLSFormType +from app.models.enums import HTTPStatus, TaskStatus, XLSFormType from app.projects import project_schemas @@ -341,7 +341,7 @@ async def update_project_xform( form_file_ext, return_form_data=True, ) - updated_xform_data = await update_survey_xform( + updated_xform_data = await modify_xform_xml( xform_data, category, task_count, @@ -364,7 +364,7 @@ async def read_and_test_xform( input_data: BytesIO, form_file_ext: str, return_form_data: bool = False, -) -> Union[BytesIO, dict]: +) -> BytesIO | dict: """Read and validate an XForm. Args: @@ -441,52 +441,7 @@ async def read_and_test_xform( ) from e -async def update_entity_registration_xform( - form_data: BytesIO, - category: str, -) -> BytesIO: - """Update fields in entity registration to name dataset. - - The CSV media must be named the same as the dataset (entity list). - - Args: - form_data (str): The input registration form data. - category (str): The form category, used to name the dataset (entity list) - and the .csv file containing the geometries. - - Returns: - BytesIO: The XForm data. - """ - log.debug(f"Updating XML keys in Entity Registration XForm: {category}") - - # Parse the XML - root = ElementTree.fromstring(form_data.getvalue()) - - # Define namespaces - namespaces = { - "h": "http://www.w3.org/1999/xhtml", - "xforms": "http://www.w3.org/2002/xforms", - "jr": "http://openrosa.org/javarosa", - "ns3": "http://www.opendatakit.org/xforms/entities", - "odk": "http://www.opendatakit.org/xforms", - } - - # Update the dataset name within the meta section - for meta_elem in root.findall(".//xforms:entity[@dataset]", namespaces): - meta_elem.set("dataset", category) - - # Update the attachment name to {category}.csv, to link to the entity list - for instance_elem in root.findall(".//xforms:instance[@src]", namespaces): - src_value = instance_elem.get("src", "") - if src_value.endswith(".csv"): - # NOTE geojson files require jr://file/{category}.geojson - # NOTE csv files require jr://file-csv/{category}.csv - instance_elem.set("src", f"jr://file-csv/{category}.csv") - - return BytesIO(ElementTree.tostring(root)) - - -async def update_survey_xform( +async def modify_xform_xml( form_data: BytesIO, category: str, task_count: int, @@ -496,7 +451,7 @@ async def update_survey_xform( The 'id' field is set to random UUID (xFormId) unless existing_id is specified The 'name' field is set to the category name. - The upload media must match the (entity) dataset name (with .csv). + The upload media must be equal to 'features.csv'. The task_id options are populated as choices in the form. The form_category value is also injected to display in the instructions. @@ -542,9 +497,9 @@ async def update_survey_xform( for inst in xform_instance_src: src_value = inst.get("src", "") if src_value.endswith(".geojson") or src_value.endswith(".csv"): - # NOTE geojson files require jr://file/{category}.geojson - # NOTE csv files require jr://file-csv/{category}.csv - inst.set("src", f"jr://file-csv/{category}.csv") + # NOTE geojson files require jr://file/features.geojson + # NOTE csv files require jr://file-csv/features.csv + inst.set("src", "jr://file-csv/features.csv") # NOTE add the task ID choices to the XML # must be defined inside root element @@ -713,7 +668,7 @@ async def convert_odk_submission_json_to_geojson( async def get_entities_geojson( odk_creds: project_schemas.ODKCentralDecrypted, odk_id: int, - dataset_name: str, + dataset_name: str = "features", minimal: Optional[bool] = False, ) -> geojson.FeatureCollection: """Get the Entity details for a dataset / Entity list. @@ -808,7 +763,7 @@ async def get_entities_geojson( async def get_entities_data( odk_creds: project_schemas.ODKCentralDecrypted, odk_id: int, - dataset_name: str, + dataset_name: str = "features", fields: str = "__system/updatedAt, osm_id, status, task_id", ) -> list: """Get all the entity mapping statuses. @@ -846,7 +801,10 @@ async def get_entities_data( def entity_to_flat_dict( - entity: Optional[dict], odk_id: int, dataset_name: str, entity_uuid: str + entity: Optional[dict], + odk_id: int, + entity_uuid: str, + dataset_name: str = "features", ) -> dict: """Convert returned Entity from ODK Central to flattened dict.""" if not entity: @@ -872,8 +830,8 @@ def entity_to_flat_dict( async def get_entity_mapping_status( odk_creds: project_schemas.ODKCentralDecrypted, odk_id: int, - dataset_name: str, entity_uuid: str, + dataset_name: str = "features", ) -> dict: """Get an single entity mapping status. @@ -895,16 +853,16 @@ async def get_entity_mapping_status( dataset_name, entity_uuid, ) - return entity_to_flat_dict(entity, odk_id, dataset_name, entity_uuid) + return entity_to_flat_dict(entity, odk_id, entity_uuid, dataset_name) async def update_entity_mapping_status( odk_creds: project_schemas.ODKCentralDecrypted, odk_id: int, - dataset_name: str, entity_uuid: str, label: str, - status: str, + status: TaskStatus, + dataset_name: str = "features", ) -> dict: """Update the Entity mapping status. @@ -913,10 +871,10 @@ async def update_entity_mapping_status( Args: odk_creds (ODKCentralDecrypted): ODK credentials for a project. odk_id (str): The project ID in ODK Central. - dataset_name (str): The dataset / Entity list name in ODK Central. entity_uuid (str): The unique entity UUID for ODK Central. label (str): New label, with emoji prepended for status. - status (str): New TaskStatus to assign, in string form. + status (TaskStatus): New TaskStatus to assign, in string form. + dataset_name (str): Override the default dataset / Entity list name 'features'. Returns: dict: All Entity data in OData JSON format. @@ -931,7 +889,7 @@ async def update_entity_mapping_status( "status": status, }, ) - return entity_to_flat_dict(entity, odk_id, dataset_name, entity_uuid) + return entity_to_flat_dict(entity, odk_id, entity_uuid, dataset_name) def upload_media( diff --git a/src/backend/app/helpers/helper_routes.py b/src/backend/app/helpers/helper_routes.py index c30808ada8..6475391054 100644 --- a/src/backend/app/helpers/helper_routes.py +++ b/src/backend/app/helpers/helper_routes.py @@ -62,7 +62,8 @@ async def download_template( category: XLSFormType, ): """Download an XLSForm template to fill out.""" - xlsform_path = f"{xlsforms_path}/{category}.xls" + filename = XLSFormType(category).name + xlsform_path = f"{xlsforms_path}/{filename}.xls" if Path(xlsform_path).exists(): return FileResponse(xlsform_path, filename="form.xls") else: diff --git a/src/backend/app/models/enums.py b/src/backend/app/models/enums.py index 494b5ec87f..1904a86d54 100644 --- a/src/backend/app/models/enums.py +++ b/src/backend/app/models/enums.py @@ -315,12 +315,19 @@ class GeometryType(str, Enum): class XLSFormType(str, Enum): - """Enum for XLSForm categories.""" - - BUILDING = "buildings" - HIGHWAYS = "highways" - HEALTH = "health" - TOILETS = "toilets" - RELIGIOUS = "religious" - LANDUSAGE = "landusage" - WATERWAYS = "waterways" + """Enum for XLSForm categories. + + The key is the name of the XLSForm file for internal use. + This cannot match an existing OSM tag value, so some words are replaced + (e.g. OSM=healthcare, XLSForm=health). + + The the value is the user facing form name (e.g. healthcare). + """ + + buildings = "buildings" + # highways = "highways" + health = "healthcare" + # toilets = "toilets" + # religious = "religious" + # landusage = "landusage" + # waterways = "waterways" diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 5f988604f0..9cd14ef175 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -36,7 +36,6 @@ from geojson.feature import Feature, FeatureCollection from loguru import logger as log from osm_fieldwork.basemapper import create_basemap_file -from osm_fieldwork.json2osm import json2osm from osm_fieldwork.OdkCentral import OdkAppUser from osm_fieldwork.xlsforms import entities_registration, xlsforms_path from osm_rawdata.postgres import PostgresClient @@ -58,7 +57,7 @@ split_geojson_by_task_areas, task_geojson_dict_to_entity_values, ) -from app.models.enums import HTTPStatus, ProjectRole, ProjectVisibility +from app.models.enums import HTTPStatus, ProjectRole, ProjectVisibility, XLSFormType from app.projects import project_deps, project_schemas from app.s3 import add_obj_to_bucket from app.tasks import tasks_crud @@ -521,20 +520,17 @@ async def split_geojson_into_tasks( # --------------------------- -async def read_and_insert_xlsforms(db, directory): +async def read_and_insert_xlsforms(db, directory) -> None: """Read the list of XLSForms from the disk and insert to DB.""" - existing_titles = set( - title for (title,) in db.query(db_models.DbXLSForm.title).all() - ) - xlsforms_on_disk = [ - file.stem - for file in Path(directory).glob("*.xls") - if not file.stem.startswith("entities") - ] - # Insert new XLSForms to DB and update existing ones - for xlsform_name in xlsforms_on_disk: - file_path = Path(directory) / f"{xlsform_name}.xls" + for xls_type in XLSFormType: + file_name = xls_type.name + category = xls_type.value + file_path = Path(directory) / f"{file_name}.xls" + + if not file_path.exists(): + log.warning(f"{file_path} does not exist!") + continue if file_path.stat().st_size == 0: log.warning(f"{file_path} is empty!") @@ -552,17 +548,21 @@ async def read_and_insert_xlsforms(db, directory): title = EXCLUDED.title, xls = EXCLUDED.xls """ ) - db.execute(insert_query, {"title": xlsform_name, "xls": data}) + db.execute(insert_query, {"title": category, "xls": data}) db.commit() - log.info(f"Inserted or updated {xlsform_name} xlsform to database") + log.info(f"XLSForm for '{category}' present in the database") except Exception as e: log.error( - f"Failed to insert or update {xlsform_name} in the database. Error: {e}" + f"Failed to insert or update {category} in the database. Error: {e}" ) + existing_db_forms = set( + title for (title,) in db.query(db_models.DbXLSForm.title).all() + ) + required_forms = set(xls_type.value for xls_type in XLSFormType) # Delete XLSForms from DB that are not found on disk - for title in existing_titles - set(xlsforms_on_disk): + for title in existing_db_forms - required_forms: delete_query = text( """ DELETE FROM xlsforms WHERE title = :title @@ -570,9 +570,9 @@ async def read_and_insert_xlsforms(db, directory): ) db.execute(delete_query, {"title": title}) db.commit() - log.info(f"Deleted {title} from the database as it was not found on disk.") - - return xlsforms_on_disk + log.info( + f"Deleted {title} from the database as it was not present in XLSFormType." + ) async def get_odk_id_for_project(db: Session, project_id: int): @@ -833,7 +833,7 @@ async def generate_odk_central_project_content( """Populate the project in ODK Central with XForm, Appuser, Permissions.""" project_odk_id = project.odkid # Create an app user (i.e. QR Code) for the project - appuser_name = f"fmtm_user_{form_category}" + appuser_name = "fmtm_user" log.info( f"Creating ODK appuser ({appuser_name}) for ODK project ({project_odk_id})" ) @@ -866,15 +866,11 @@ async def generate_odk_central_project_content( registration_xform = await central_crud.read_and_test_xform( registration_xlsform, "xls", return_form_data=True ) - # Manually modify fields in XML specific to project - updated_reg_xform = await central_crud.update_entity_registration_xform( - registration_xform, form_category - ) # Upload entity registration XForm log.info("Uploading Entity registration XForm to ODK Central") central_crud.create_odk_xform( project_odk_id, - updated_reg_xform, + registration_xform, odk_credentials, ) @@ -883,7 +879,7 @@ async def generate_odk_central_project_content( xlsform, form_file_ext, return_form_data=True ) # Manually modify fields in XML specific to project (id, name, etc) - updated_xform = await central_crud.update_survey_xform( + updated_xform = await central_crud.modify_xform_xml( xform, form_category, task_count, @@ -967,7 +963,8 @@ async def generate_project_files( else: log.debug(f"Using default XLSForm for category: '{form_category}'") - xlsform_path = f"{xlsforms_path}/{form_category}.xls" + form_filename = XLSFormType(form_category).name + xlsform_path = f"{xlsforms_path}/{form_filename}.xls" with open(xlsform_path, "rb") as f: xlsform = BytesIO(f.read()) @@ -1026,7 +1023,7 @@ async def generate_project_files( async with central_deps.get_odk_entity(odk_credentials) as odk_central: entities = await odk_central.createEntities( project_odk_id, - form_category, + "features", entities_data_dict, ) if entities: @@ -1335,13 +1332,13 @@ async def get_background_task_status(task_id: uuid.UUID, db: Session): async def insert_background_task_into_database( - db: Session, name: str = None, project_id: str = None -) -> uuid.uuid4: + db: Session, name: Optional[str] = None, project_id: Optional[int] = None +) -> uuid.UUID: """Inserts a new task into the database. Args: db (Session): database session - name (str): name of the task. + name (int): name of the task. project_id (str): associated project id Returns: @@ -1549,9 +1546,23 @@ async def get_mbtiles_list(db: Session, project_id: int): raise HTTPException(status_code=400, detail=str(e)) from e -async def convert_geojson_to_osm(geojson_file: str): - """Convert a GeoJSON file to OSM format.""" - return json2osm(geojson_file) +# async def convert_geojson_to_osm(geojson_file: str): +# """Convert a GeoJSON file to OSM format.""" +# jsonin = JsonDump() +# geojson_path = Path(geojson_file) +# data = jsonin.parse(geojson_path) + +# osmoutfile = f"{geojson_path.stem}.osm" +# jsonin.createOSM(osmoutfile) + +# for entry in data: +# feature = jsonin.createEntry(entry) + +# # TODO add json2osm back in +# # https://github.com/hotosm/osm-fieldwork/blob/1a94afff65c4653190d735 +# # f104c0644dcfb71e64/osm_fieldwork/json2osm.py#L363 + +# return json2osm(geojson_file) async def get_pagination(page: int, count: int, results_per_page: int, total: int): diff --git a/src/backend/app/projects/project_deps.py b/src/backend/app/projects/project_deps.py index b7c5f384f6..e92bc96eb3 100644 --- a/src/backend/app/projects/project_deps.py +++ b/src/backend/app/projects/project_deps.py @@ -18,6 +18,7 @@ """Project dependencies for use in Depends.""" +from functools import lru_cache from typing import Optional from fastapi import Depends @@ -57,6 +58,7 @@ async def get_project_by_id( return db_project +@lru_cache(maxsize=None) async def get_odk_credentials(db: Session, project_id: int): """Get ODK credentials of a project, or default organization credentials.""" sql = text( diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 522a627ce2..5d9ae27efd 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -61,6 +61,7 @@ TILES_SOURCE, HTTPStatus, ProjectVisibility, + XLSFormType, ) from app.organisations import organisation_deps from app.projects import project_crud, project_deps, project_schemas @@ -208,7 +209,6 @@ async def get_odk_entities_geojson( return await central_crud.get_entities_geojson( odk_credentials, project.odkid, - project.xform_category, minimal=minimal, ) @@ -226,7 +226,6 @@ async def get_odk_entities_mapping_statuses( return await central_crud.get_entities_data( odk_credentials, project.odkid, - project.xform_category, ) @@ -248,7 +247,6 @@ async def get_odk_entities_osm_ids( return await central_crud.get_entities_data( odk_credentials, project.odkid, - project.xform_category, fields="osm_id", ) @@ -266,7 +264,6 @@ async def get_odk_entities_task_ids( return await central_crud.get_entities_data( odk_credentials, project.odkid, - project.xform_category, fields="task_id", ) @@ -285,7 +282,6 @@ async def get_odk_entity_mapping_status( return await central_crud.get_entity_mapping_status( odk_credentials, project.odkid, - project.xform_category, entity_id, ) @@ -312,7 +308,6 @@ async def set_odk_entities_mapping_status( return await central_crud.update_entity_mapping_status( odk_credentials, project.odkid, - project.xform_category, entity_details.entity_id, entity_details.label, entity_details.status, @@ -346,23 +341,37 @@ async def download_tiles( ): """Download the basemap tile archive for a project.""" log.debug("Getting tile archive path from DB") - tiles_path = ( + dbtile_obj = ( db.query(db_models.DbTilesPath) .filter(db_models.DbTilesPath.id == str(tile_id)) .first() ) - log.info(f"User requested download for tiles: {tiles_path.path}") + if not dbtile_obj: + raise HTTPException( + status_code=HTTPStatus.NOT_FOUND, detail="Basemap ID does not exist!" + ) + log.info(f"User requested download for tiles: {dbtile_obj.path}") - project_id = tiles_path.project_id + project_id = dbtile_obj.project_id project = await project_crud.get_project(db, project_id) - filename = Path(tiles_path.path).name.replace( + filename = Path(dbtile_obj.path).name.replace( f"{project_id}_", f"{project.project_name_prefix}_" ) log.debug(f"Sending tile archive to user: {filename}") + if (tiles_path := Path(filename).suffix) == ".mbtiles": + tiles_mime_type = "application/vnd.mapbox-vector-tile" + elif tiles_path == ".pmtiles": + tiles_mime_type = "application/vnd.pmtiles" + else: + tiles_mime_type = "application/vnd.sqlite3" + return FileResponse( - tiles_path.path, - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + dbtile_obj.path, + headers={ + "Content-Disposition": f'attachment; filename="{filename}"', + "Content-Type": tiles_mime_type, + }, ) @@ -818,7 +827,7 @@ async def preview_split_by_square( @router.post("/generate-data-extract/") async def get_data_extract( geojson_file: UploadFile = File(...), - form_category: Optional[str] = Form(None), + form_category: Optional[XLSFormType] = Form(None), # config_file: Optional[str] = Form(None), current_user: AuthUser = Depends(login_required), ): @@ -831,7 +840,8 @@ async def get_data_extract( # Get extract config file from existing data_models if form_category: - data_model = f"{data_models_path}/{form_category}.yaml" + config_filename = XLSFormType(form_category).name + data_model = f"{data_models_path}/{config_filename}.yaml" with open(data_model, "rb") as data_model_yaml: extract_config = BytesIO(data_model_yaml.read()) else: @@ -926,7 +936,8 @@ async def download_form( "Content-Type": "application/media", } if not project.form_xls: - xlsform_path = f"{xlsforms_path}/{project.xform_category}.xls" + form_filename = XLSFormType(project.xform_category).name + xlsform_path = f"{xlsforms_path}/{form_filename}.xls" if os.path.exists(xlsform_path): return FileResponse(xlsform_path, filename="form.xls") else: @@ -937,7 +948,7 @@ async def download_form( @router.post("/update-form") async def update_project_form( xform_id: str = Form(...), - category: str = Form(...), + category: XLSFormType = Form(...), upload: Optional[UploadFile] = File(None), db: Session = Depends(database.get_db), project_user_dict: dict = Depends(project_admin), @@ -967,7 +978,8 @@ async def update_project_form( project.form_xls = new_xform_data new_xform_data = BytesIO(new_xform_data) else: - xlsform_path = Path(f"{xlsforms_path}/{category}.xls") + form_filename = XLSFormType(project.xform_category).name + xlsform_path = Path(f"{xlsforms_path}/{form_filename}.xls") file_ext = xlsform_path.suffix.lower() with open(xlsform_path, "rb") as f: new_xform_data = BytesIO(f.read()) @@ -1120,35 +1132,35 @@ async def convert_fgb_to_geojson( return Response(content=json.dumps(data_extract_geojson), headers=headers) -@router.get("/boundary_in_osm/{project_id}/") -async def download_task_boundary_osm( - project_id: int, - db: Session = Depends(database.get_db), - current_user: AuthUser = Depends(mapper), -): - """Downloads the boundary of a task as a OSM file. +# @router.get("/boundary_in_osm/{project_id}/") +# async def download_task_boundary_osm( +# project_id: int, +# db: Session = Depends(database.get_db), +# current_user: AuthUser = Depends(mapper), +# ): +# """Downloads the boundary of a task as a OSM file. - Args: - project_id (int): The id of the project. - db (Session): The database session, provided automatically. - current_user (AuthUser): Check if user has MAPPER permission. +# Args: +# project_id (int): The id of the project. +# db (Session): The database session, provided automatically. +# current_user (AuthUser): Check if user has MAPPER permission. - Returns: - Response: The HTTP response object containing the downloaded file. - """ - out = await project_crud.get_task_geometry(db, project_id) - file_path = f"/tmp/{project_id}_task_boundary.geojson" +# Returns: +# Response: The HTTP response object containing the downloaded file. +# """ +# out = await project_crud.get_task_geometry(db, project_id) +# file_path = f"/tmp/{project_id}_task_boundary.geojson" - # Write the response content to the file - with open(file_path, "w") as f: - f.write(out) - result = await project_crud.convert_geojson_to_osm(file_path) +# # Write the response content to the file +# with open(file_path, "w") as f: +# f.write(out) +# result = await project_crud.convert_geojson_to_osm(file_path) - with open(result, "r") as f: - content = f.read() +# with open(result, "r") as f: +# content = f.read() - response = Response(content=content, media_type="application/xml") - return response +# response = Response(content=content, media_type="application/xml") +# return response @router.get("/centroid/") diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index b647a3c9b5..1005a9ad16 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -39,7 +39,7 @@ read_wkb, write_wkb, ) -from app.models.enums import ProjectPriority, ProjectStatus, TaskSplitType +from app.models.enums import ProjectPriority, ProjectStatus, TaskSplitType, XLSFormType from app.tasks import tasks_schemas from app.users.user_schemas import User @@ -307,7 +307,7 @@ class ProjectBase(BaseModel): project_info: ProjectInfo status: ProjectStatus # location_str: str - xform_category: Optional[str] = None + xform_category: Optional[XLSFormType] = None hashtags: Optional[List[str]] = None organisation_id: Optional[int] = None diff --git a/src/backend/app/submissions/submission_crud.py b/src/backend/app/submissions/submission_crud.py index 3a5ed2d7c6..19d8b81f0f 100644 --- a/src/backend/app/submissions/submission_crud.py +++ b/src/backend/app/submissions/submission_crud.py @@ -21,7 +21,6 @@ import hashlib import io import json -import os import uuid from collections import Counter from datetime import datetime, timedelta @@ -32,7 +31,8 @@ from asgiref.sync import async_to_sync from fastapi import HTTPException, Response from loguru import logger as log -from osm_fieldwork.json2osm import json2osm + +# from osm_fieldwork.json2osm import json2osm from sqlalchemy.orm import Session from app.central.central_crud import get_odk_form, get_odk_project, list_odk_xforms @@ -43,16 +43,10 @@ from app.s3 import add_obj_to_bucket, get_obj_from_bucket from app.tasks import tasks_crud - -async def convert_json_to_osm(file_path): - """Wrapper for osm-fieldwork json2osm. - - FIXME add json output to osm2json (in addition to default OSM XML output) - """ - # TODO check speed of json2osm - # TODO if slow response, use run_in_threadpool - osm_xml_path = json2osm(file_path) - return osm_xml_path +# async def convert_json_to_osm(file_path): +# """Wrapper for osm-fieldwork json2osm.""" +# osm_xml_path = json2osm(file_path) +# return osm_xml_path # TODO remove this @@ -74,63 +68,64 @@ async def convert_json_to_osm(file_path): # return osmoutfile -def convert_to_osm(db: Session, project_id: int, task_id: Optional[int]): - """Convert submissions to OSM XML format.""" - project_sync = async_to_sync(project_deps.get_project_by_id) - project = project_sync(db, project_id) +# # FIXME 07/06/2024 since osm-fieldwork update +# def convert_to_osm(db: Session, project_id: int, task_id: Optional[int]): +# """Convert submissions to OSM XML format.""" +# project_sync = async_to_sync(project_deps.get_project_by_id) +# project = project_sync(db, project_id) - get_submission_sync = async_to_sync(get_submission_by_project) - data = get_submission_sync(project_id, {}, db) +# get_submission_sync = async_to_sync(get_submission_by_project) +# data = get_submission_sync(project_id, {}, db) - submissions = data.get("value", []) +# submissions = data.get("value", []) - # Create a new ZIP file for the extracted files - final_zip_file_path = f"/tmp/{project.project_name_prefix}_osm.zip" +# # Create a new ZIP file for the extracted files +# final_zip_file_path = f"/tmp/{project.project_name_prefix}_osm.zip" - # Remove the ZIP file if it already exists - if os.path.exists(final_zip_file_path): - os.remove(final_zip_file_path) +# # Remove the ZIP file if it already exists +# if os.path.exists(final_zip_file_path): +# os.remove(final_zip_file_path) - # filter submission by task_id - if task_id: - submissions = [ - sub - for sub in submissions - if sub.get("all", {}).get("task_id") == str(task_id) - ] +# # filter submission by task_id +# if task_id: +# submissions = [ +# sub +# for sub in submissions +# if sub.get("all", {}).get("task_id") == str(task_id) +# ] - if not submissions: - raise HTTPException(status_code=404, detail="Submission not found") +# if not submissions: +# raise HTTPException(status_code=404, detail="Submission not found") - # JSON FILE PATH - jsoninfile = "/tmp/json_infile.json" +# # JSON FILE PATH +# jsoninfile = "/tmp/json_infile.json" - # Write the submission to a file - with open(jsoninfile, "w") as f: - f.write(json.dumps(submissions)) +# # Write the submission to a file +# with open(jsoninfile, "w") as f: +# f.write(json.dumps(submissions)) - # Convert the submission to osm xml format - convert_json_to_osm_sync = async_to_sync(convert_json_to_osm) +# # Convert the submission to osm xml format +# convert_json_to_osm_sync = async_to_sync(convert_json_to_osm) - if osm_file_path := convert_json_to_osm_sync(jsoninfile): - with open(osm_file_path, "r") as osm_file: - osm_data = osm_file.read() - last_osm_index = osm_data.rfind("") - processed_xml_string = ( - osm_data[:last_osm_index] + osm_data[last_osm_index + len("") :] - ) +# if osm_file_path := convert_json_to_osm_sync(jsoninfile): +# with open(osm_file_path, "r") as osm_file: +# osm_data = osm_file.read() +# last_osm_index = osm_data.rfind("") +# processed_xml_string = ( +# osm_data[:last_osm_index] + osm_data[last_osm_index + len("") :] +# ) - with open(osm_file_path, "w") as osm_file: - osm_file.write(processed_xml_string) +# with open(osm_file_path, "w") as osm_file: +# osm_file.write(processed_xml_string) - final_zip_file_path = f"/tmp/{project.project_name_prefix}_osm.zip" - if os.path.exists(final_zip_file_path): - os.remove(final_zip_file_path) +# final_zip_file_path = f"/tmp/{project.project_name_prefix}_osm.zip" +# if os.path.exists(final_zip_file_path): +# os.remove(final_zip_file_path) - with zipfile.ZipFile(final_zip_file_path, mode="a") as final_zip_file: - final_zip_file.write(osm_file_path) +# with zipfile.ZipFile(final_zip_file_path, mode="a") as final_zip_file: +# final_zip_file.write(osm_file_path) - return final_zip_file_path +# return final_zip_file_path async def gather_all_submission_csvs(db, project_id): diff --git a/src/backend/app/submissions/submission_routes.py b/src/backend/app/submissions/submission_routes.py index d11aa634cf..f2848aa35c 100644 --- a/src/backend/app/submissions/submission_routes.py +++ b/src/backend/app/submissions/submission_routes.py @@ -18,7 +18,6 @@ """Routes associated with data submission to and from ODK Central.""" import json -import os from io import BytesIO from typing import Optional @@ -26,8 +25,6 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Response from fastapi.concurrency import run_in_threadpool from fastapi.responses import FileResponse -from osm_fieldwork.odk_merge import OdkMerge -from osm_fieldwork.osmfile import OsmFile from sqlalchemy.orm import Session from app.auth.osm import AuthUser, login_required @@ -153,65 +150,66 @@ async def get_submission_count( return await submission_crud.get_submission_count_of_a_project(db, project_id) -@router.post("/conflate_data") -async def conflate_osm_data( - project_id: int, - db: Session = Depends(database.get_db), - current_user: AuthUser = Depends(login_required), -): - """Conflate submission data against existing OSM data.""" - # All Submissions JSON - # NOTE runs in separate thread using run_in_threadpool - # FIXME we probably need to change this func - submission = await run_in_threadpool( - lambda: submission_crud.get_all_submissions_json(db, project_id) - ) +# FIXME 07/06/2024 since osm-fieldwork update +# @router.post("/conflate_data") +# async def conflate_osm_data( +# project_id: int, +# db: Session = Depends(database.get_db), +# current_user: AuthUser = Depends(login_required), +# ): +# """Conflate submission data against existing OSM data.""" +# # All Submissions JSON +# # NOTE runs in separate thread using run_in_threadpool +# # FIXME we probably need to change this func +# submission = await run_in_threadpool( +# lambda: submission_crud.get_all_submissions_json(db, project_id) +# ) - # Data extracta file - data_extracts_file = "/tmp/data_extracts_file.geojson" - - await project_crud.get_extracted_data_from_db(db, project_id, data_extracts_file) - - # Output file - outfile = "/tmp/output_file.osm" - # JSON FILE PATH - jsoninfile = "/tmp/json_infile.json" - - # # Delete if these files already exist - if os.path.exists(outfile): - os.remove(outfile) - if os.path.exists(jsoninfile): - os.remove(jsoninfile) - - # Write the submission to a file - with open(jsoninfile, "w") as f: - f.write(json.dumps(submission)) - - # Convert the submission to osm xml format - osmoutfile = await submission_crud.convert_json_to_osm(jsoninfile) - - # Remove the extra closing tag from the end of the file - with open(osmoutfile, "r") as f: - osmoutfile_data = f.read() - # Find the last index of the closing tag - last_osm_index = osmoutfile_data.rfind("") - # Remove the extra closing tag from the end - processed_xml_string = ( - osmoutfile_data[:last_osm_index] - + osmoutfile_data[last_osm_index + len("") :] - ) +# # Data extracta file +# data_extracts_file = "/tmp/data_extracts_file.geojson" + +# await project_crud.get_extracted_data_from_db(db, project_id, data_extracts_file) + +# # Output file +# outfile = "/tmp/output_file.osm" +# # JSON FILE PATH +# jsoninfile = "/tmp/json_infile.json" + +# # # Delete if these files already exist +# if os.path.exists(outfile): +# os.remove(outfile) +# if os.path.exists(jsoninfile): +# os.remove(jsoninfile) + +# # Write the submission to a file +# with open(jsoninfile, "w") as f: +# f.write(json.dumps(submission)) + +# # Convert the submission to osm xml format +# osmoutfile = await submission_crud.convert_json_to_osm(jsoninfile) + +# # Remove the extra closing tag from the end of the file +# with open(osmoutfile, "r") as f: +# osmoutfile_data = f.read() +# # Find the last index of the closing tag +# last_osm_index = osmoutfile_data.rfind("") +# # Remove the extra closing tag from the end +# processed_xml_string = ( +# osmoutfile_data[:last_osm_index] +# + osmoutfile_data[last_osm_index + len("") :] +# ) - # Write the modified XML data back to the file - with open(osmoutfile, "w") as f: - f.write(processed_xml_string) +# # Write the modified XML data back to the file +# with open(osmoutfile, "w") as f: +# f.write(processed_xml_string) - odkf = OsmFile(outfile) - osm = odkf.loadFile(osmoutfile) - if osm: - odk_merge = OdkMerge(data_extracts_file, None) - data = odk_merge.conflateData(osm) - return data - return [] +# odkf = OsmFile(outfile) +# osm = odkf.loadFile(osmoutfile) +# if osm: +# odk_merge = OdkMerge(data_extracts_file, None) +# data = odk_merge.conflateData(osm) +# return data +# return [] # TODO remove this redundant endpoint @@ -264,43 +262,44 @@ async def conflate_osm_data( # ) -@router.get("/get_osm_xml/{project_id}") -async def get_osm_xml( - project_id: int, - db: Session = Depends(database.get_db), - current_user: AuthUser = Depends(login_required), -): - """Get the submissions in OSM XML format for a project. - - TODO refactor to put logic in crud for easier testing. - """ - # JSON FILE PATH - jsoninfile = f"/tmp/{project_id}_json_infile.json" - - # # Delete if these files already exist - if os.path.exists(jsoninfile): - os.remove(jsoninfile) +# FIXME 07/06/2024 since osm-fieldwork update +# @router.get("/get_osm_xml/{project_id}") +# async def get_osm_xml( +# project_id: int, +# db: Session = Depends(database.get_db), +# current_user: AuthUser = Depends(login_required), +# ): +# """Get the submissions in OSM XML format for a project. - # All Submissions JSON - # NOTE runs in separate thread using run_in_threadpool - # FIXME we probably need to change this func - submission = await run_in_threadpool( - lambda: submission_crud.get_all_submissions_json(db, project_id) - ) +# TODO refactor to put logic in crud for easier testing. +# """ +# # JSON FILE PATH +# jsoninfile = f"/tmp/{project_id}_json_infile.json" + +# # # Delete if these files already exist +# if os.path.exists(jsoninfile): +# os.remove(jsoninfile) + +# # All Submissions JSON +# # NOTE runs in separate thread using run_in_threadpool +# # FIXME we probably need to change this func +# submission = await run_in_threadpool( +# lambda: submission_crud.get_all_submissions_json(db, project_id) +# ) - # Write the submission to a file - with open(jsoninfile, "w") as f: - f.write(json.dumps(submission)) +# # Write the submission to a file +# with open(jsoninfile, "w") as f: +# f.write(json.dumps(submission)) - # Convert the submission to osm xml format - osmoutfile = await submission_crud.convert_json_to_osm(jsoninfile) +# # Convert the submission to osm xml format +# osmoutfile = await submission_crud.convert_json_to_osm(jsoninfile) - # Remove the extra closing tag from the end of the file - with open(osmoutfile, "r") as f: - osmoutfile_data = f.read() +# # Remove the extra closing tag from the end of the file +# with open(osmoutfile, "r") as f: +# osmoutfile_data = f.read() - # Create a plain XML response - return Response(content=osmoutfile_data, media_type="application/xml") +# # Create a plain XML response +# return Response(content=osmoutfile_data, media_type="application/xml") @router.get("/submission_page/{project_id}") diff --git a/src/backend/pdm.lock b/src/backend/pdm.lock index 136d9a24fa..8b9ccf8c0c 100644 --- a/src/backend/pdm.lock +++ b/src/backend/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "debug", "dev", "docs", "test", "monitoring"] strategy = ["cross_platform"] lock_version = "4.4.1" -content_hash = "sha256:cc2c7bfe8ad025999c448bbef3e0f75dffbcb7eef9fe015e022b9436363f310b" +content_hash = "sha256:3dfb8a9f5d6fb72d99be33d0576d5649f08da14be9108063f0dc1d1352e6bf51" [[package]] name = "aiohttp" @@ -658,7 +658,7 @@ files = [ [[package]] name = "fmtm-splitter" -version = "1.2.1" +version = "1.2.2" requires_python = ">=3.10" summary = "A utility for splitting an AOI into multiple tasks." dependencies = [ @@ -669,8 +669,8 @@ dependencies = [ "shapely>=1.8.1", ] files = [ - {file = "fmtm-splitter-1.2.1.tar.gz", hash = "sha256:51e79cc8f15e4e2ad571d5bff4403dcfff2c0d0a75f4c0a26c4469557708403c"}, - {file = "fmtm_splitter-1.2.1-py3-none-any.whl", hash = "sha256:80d2ae657a2596668a19f193a874d091b153d203415c9e87d76b7f76810b6180"}, + {file = "fmtm-splitter-1.2.2.tar.gz", hash = "sha256:9384dbf00c0e53e24e1f13046ae6693e13567ff3dc0f59f29f4a96ac4a54105e"}, + {file = "fmtm_splitter-1.2.2-py3-none-any.whl", hash = "sha256:bbef78cf0e1f2b67f8c8aeaadb7fd2927bfd333d216927059a12abbbb04a5742"}, ] [[package]] @@ -1662,7 +1662,7 @@ files = [ [[package]] name = "osm-fieldwork" -version = "0.10.1" +version = "0.11.2" requires_python = ">=3.10" summary = "Processing field data from ODK to OpenStreetMap format." dependencies = [ @@ -1687,8 +1687,8 @@ dependencies = [ "xmltodict>=0.13.0", ] files = [ - {file = "osm-fieldwork-0.10.1.tar.gz", hash = "sha256:a437e46e4e8e63f5bd5fedef7830d5648995ee9c896993e70dd63551af30766e"}, - {file = "osm_fieldwork-0.10.1-py3-none-any.whl", hash = "sha256:1f03c63ac482a558bc2a43f6d05ab66c092b2cfee57d7a838e5e744b00042e53"}, + {file = "osm-fieldwork-0.11.2.tar.gz", hash = "sha256:472edd6873a173d526b636a01abb39cb558609d9df14da19cfe82413945954db"}, + {file = "osm_fieldwork-0.11.2-py3-none-any.whl", hash = "sha256:be919fc058811253c7ee9276822d9160e2dff170d777bbf3e95d7fc40ec26def"}, ] [[package]] diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index 64e737d6b8..675141d3c5 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -46,9 +46,9 @@ dependencies = [ "cryptography>=42.0.1", "defusedxml>=0.7.1", "osm-login-python==1.0.3", - "osm-fieldwork==0.10.1", + "osm-fieldwork==0.11.2", "osm-rawdata==0.3.0", - "fmtm-splitter==1.2.1", + "fmtm-splitter==1.2.2", ] requires-python = ">=3.10" readme = "../../README.md" diff --git a/src/frontend/src/api/CreateProjectService.ts b/src/frontend/src/api/CreateProjectService.ts index 1deff22c06..548bcd9606 100755 --- a/src/frontend/src/api/CreateProjectService.ts +++ b/src/frontend/src/api/CreateProjectService.ts @@ -71,13 +71,7 @@ const CreateProjectService: Function = ( formUpload, ), ); - - dispatch(CommonActions.SetLoading(false)); - dispatch(CreateProjectActions.CreateProjectLoading(true)); } catch (error: any) { - dispatch(CommonActions.SetLoading(false)); - dispatch(CreateProjectActions.CreateProjectLoading(true)); - // Added Snackbar toast for error message dispatch( CommonActions.SetSnackBar({ @@ -87,9 +81,9 @@ const CreateProjectService: Function = ( duration: 2000, }), ); - //END - dispatch(CreateProjectActions.CreateProjectLoading(false)); + } finally { + dispatch(CommonActions.SetLoading(false)); dispatch(CreateProjectActions.CreateProjectLoading(false)); } }; @@ -97,6 +91,7 @@ const CreateProjectService: Function = ( await postCreateProjectDetails(url, projectData, taskAreaGeojson, formUpload); }; }; + const FormCategoryService: Function = (url: string) => { return async (dispatch) => { dispatch(CreateProjectActions.GetFormCategoryLoading(true)); diff --git a/src/frontend/src/api/Project.js b/src/frontend/src/api/Project.js index c183ebb534..2bc985e789 100755 --- a/src/frontend/src/api/Project.js +++ b/src/frontend/src/api/Project.js @@ -169,7 +169,7 @@ export const DownloadTile = (url, payload, toOpfs = false) => { responseType: 'arraybuffer', }); - // Get filename from content-disposition header + // Get filename from Content-Disposition header const tileData = response.data; if (toOpfs) { @@ -182,9 +182,9 @@ export const DownloadTile = (url, payload, toOpfs = false) => { return; } - const filename = response.headers['content-disposition'].split('filename=')[1]; + const filename = response.headers['Content-Disposition'].split('filename=')[1]; // Create Blob from ArrayBuffer - const blob = new Blob([tileData], { type: response.headers['content-type'] }); + const blob = new Blob([tileData], { type: response.headers['Content-Type'] }); const downloadUrl = URL.createObjectURL(blob); const a = document.createElement('a'); diff --git a/src/frontend/src/components/createnewproject/SplitTasks.tsx b/src/frontend/src/components/createnewproject/SplitTasks.tsx index 2c2d9c43d4..f1ce77c3e3 100644 --- a/src/frontend/src/components/createnewproject/SplitTasks.tsx +++ b/src/frontend/src/components/createnewproject/SplitTasks.tsx @@ -192,23 +192,34 @@ const SplitTasks = ({ flag, geojsonFile, setGeojsonFile, customDataExtractUpload ); } }; + useEffect(() => { - if (generateQrSuccess) { - const projectId = projectDetailsResponse?.id; - dispatch( - CommonActions.SetSnackBar({ - open: true, - message: 'QR Generation Completed.', - variant: 'success', - duration: 2000, - }), - ); - dispatch(CreateProjectActions.SetGenerateProjectQRSuccess(null)); - navigate(`/project/${projectId}`); - dispatch(CreateProjectActions.ClearCreateProjectFormData()); - dispatch(CreateProjectActions.SetCanSwitchCreateProjectSteps(false)); - } - }, [generateQrSuccess]); + const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms)); + + const handleQRGeneration = async () => { + if (generateQrSuccess) { + const projectId = projectDetailsResponse?.id; + dispatch( + CommonActions.SetSnackBar({ + open: true, + message: 'QR Generation Completed. Redirecting...', + variant: 'success', + duration: 2000, + }), + ); + + // Add 5-second delay to allow backend Entity generation to catch up + await delay(5000); + + dispatch(CreateProjectActions.SetGenerateProjectQRSuccess(null)); + navigate(`/project/${projectId}`); + dispatch(CreateProjectActions.ClearCreateProjectFormData()); + dispatch(CreateProjectActions.SetCanSwitchCreateProjectSteps(false)); + } + }; + + handleQRGeneration(); + }, [generateQrSuccess]); const renderTraceback = (errorText: string) => { if (!errorText) {