Updated generate_project_files to bulk upload entities

src/backend/app/central/central_crud.py

@@ -731,7 +731,7 @@ async def get_entities_geojson(
     Returns:
         dict: Entity data in OData JSON format.
     """
-    async with central_deps.get_odk_entity(odk_creds) as odk_central:
+    async with central_deps.get_odk_dataset(odk_creds) as odk_central:
         entities = await odk_central.getEntityData(
             odk_id,
             dataset_name,
@@ -781,7 +781,7 @@ async def get_entities_data(
         list: JSON list containing Entity info. If updated_at is included,
             the format is string 2022-01-31T23:59:59.999Z.
     """
-    async with central_deps.get_odk_entity(odk_creds) as odk_central:
+    async with central_deps.get_odk_dataset(odk_creds) as odk_central:
         entities = await odk_central.getEntityData(
             odk_id,
             dataset_name,
@@ -847,7 +847,7 @@ async def get_entity_mapping_status(
         dict: JSON containing Entity: id, status, updated_at.
             updated_at is in string format 2022-01-31T23:59:59.999Z.
     """
-    async with central_deps.get_odk_entity(odk_creds) as odk_central:
+    async with central_deps.get_odk_dataset(odk_creds) as odk_central:
         entity = await odk_central.getEntity(
             odk_id,
             dataset_name,
@@ -879,7 +879,7 @@ async def update_entity_mapping_status(
     Returns:
         dict: All Entity data in OData JSON format.
     """
-    async with central_deps.get_odk_entity(odk_creds) as odk_central:
+    async with central_deps.get_odk_dataset(odk_creds) as odk_central:
         entity = await odk_central.updateEntity(
             odk_id,
             dataset_name,

src/backend/app/central/central_deps.py

@@ -21,17 +21,17 @@
 from contextlib import asynccontextmanager
 
 from fastapi.exceptions import HTTPException
-from osm_fieldwork.OdkCentralAsync import OdkEntity
+from osm_fieldwork.OdkCentralAsync import OdkDataset
 
 from app.models.enums import HTTPStatus
 from app.projects.project_schemas import ODKCentralDecrypted
 
 
 @asynccontextmanager
-async def get_odk_entity(odk_creds: ODKCentralDecrypted):
-    """Wrap getting an OdkEntity object with ConnectionError handling."""
+async def get_odk_dataset(odk_creds: ODKCentralDecrypted):
+    """Wrap getting an OdkDataset object with ConnectionError handling."""
     try:
-        async with OdkEntity(
+        async with OdkDataset(
             url=odk_creds.odk_central_url,
             user=odk_creds.odk_central_user,
             passwd=odk_creds.odk_central_password,

src/backend/app/db/postgis_utils.py

@@ -696,6 +696,11 @@ async def feature_geojson_to_entity_dict(
     feature_id = feature.get("id")
 
     geometry = feature.get("geometry", {})
+    if not geometry:
+        msg = "'geometry' data field is mandatory"
+        log.debug(msg)
+        raise ValueError(msg)
+
     javarosa_geom = await geojson_to_javarosa_geom(geometry)
 
     # NOTE all properties MUST be string values for Entities, convert
@@ -708,7 +713,7 @@ async def feature_geojson_to_entity_dict(
     task_id = properties.get("task_id")
     entity_label = f"Task {task_id} Feature {feature_id}"
 
-    return {entity_label: {"geometry": javarosa_geom, **properties}}
+    return {"label": entity_label, "data": {"geometry": javarosa_geom, **properties}}
 
 
 async def task_geojson_dict_to_entity_values(task_geojson_dict):
@@ -720,9 +725,7 @@ async def task_geojson_dict_to_entity_values(task_geojson_dict):
             [feature_geojson_to_entity_dict(feature) for feature in features if feature]
         )
 
-    entity_values = await gather(*asyncio_tasks)
-    # Merge all dicts into a single dict
-    return {k: v for result in entity_values for k, v in result.items()}
+    return await gather(*asyncio_tasks)
 
 
 def multipolygon_to_polygon(

src/backend/app/helpers/helper_routes.py

@@ -192,7 +192,7 @@ def parse_csv(csv_bytes):
     parsed_data = parse_csv(await csv_file.read())
     entities_data_dict = {str(uuid4()): data for data in parsed_data}
 
-    async with central_deps.get_odk_entity(odk_creds) as odk_central:
+    async with central_deps.get_odk_dataset(odk_creds) as odk_central:
         entities = await odk_central.createEntities(
             odk_project_id,
             entity_name,

src/backend/app/projects/project_crud.py

@@ -35,7 +35,7 @@
 from geojson.feature import Feature, FeatureCollection
 from loguru import logger as log
 from osm_fieldwork.basemapper import create_basemap_file
-from osm_fieldwork.xlsforms import entities_registration, xlsforms_path
+from osm_fieldwork.xlsforms import xlsforms_path
 from osm_rawdata.postgres import PostgresClient
 from shapely.geometry import shape
 from sqlalchemy import and_, column, func, select, table, text
@@ -828,36 +828,34 @@ async def generate_odk_central_project_content(
     xlsform: BytesIO,
     form_category: str,
     form_file_ext: str,
-    task_count: int,
+    task_extract_dict: dict,
     db: Session,
 ) -> str:
     """Populate the project in ODK Central with XForm, Appuser, Permissions."""
     project_odk_id = project.odkid
 
-    # NOTE Entity Registration form: this may be removed with future Central
-    # API changes to allow Entity creation
-    with open(entities_registration, "rb") as f:
-        registration_xlsform = BytesIO(f.read())
-    registration_xform = await central_crud.read_and_test_xform(
-        registration_xlsform, "xls", return_form_data=True
-    )
-    # Upload entity registration XForm
-    log.info("Uploading Entity registration XForm to ODK Central")
-    central_crud.create_odk_xform(
-        project_odk_id,
-        registration_xform,
-        odk_credentials,
-    )
+    # The ODK Dataset (Entity List) must exist prior to main XLSForm
+    entities_list = await task_geojson_dict_to_entity_values(task_extract_dict)
+    fields_list = project_schemas.entity_fields_to_list()
+
+    async with central_deps.get_odk_dataset(odk_credentials) as odk_central:
+        await odk_central.createDataset(
+            project_odk_id, datasetName="features", properties=fields_list
+        )
+        await odk_central.createEntities(
+            project_odk_id,
+            "features",
+            entities_list,
+        )
 
-    # NOTE Survey form
     xform = await central_crud.read_and_test_xform(
         xlsform, form_file_ext, return_form_data=True
     )
     # Manually modify fields in XML specific to project (id, name, etc)
     updated_xform = await central_crud.modify_xform_xml(
         xform,
         form_category,
-        task_count,
+        len(task_extract_dict.keys()),
     )
     # Upload survey XForm
     log.info("Uploading survey XForm to ODK Central")
@@ -869,13 +867,13 @@ async def generate_odk_central_project_content(
 
     sql = text(
         """
-                INSERT INTO xforms (
-                        project_id, odk_form_id, category
-                        )
-                    VALUES (
-                        :project_id, :xform_id, :category
-                        )
-                """
+        INSERT INTO xforms (
+            project_id, odk_form_id, category
+        )
+        VALUES (
+            :project_id, :xform_id, :category
+        )
+        """
     )
     db.execute(
         sql,
@@ -906,7 +904,7 @@ async def generate_project_files(
         background_task_id (uuid): the task_id of the background task.
     """
     try:
-        project = await get_project_by_id(db, project_id)
+        project = await project_deps.get_project_by_id(db, project_id)
         form_category = project.xform_category
         log.info(f"Starting generate_project_files for project {project_id}")
         odk_credentials = await project_deps.get_odk_credentials(db, project_id)
@@ -948,7 +946,7 @@ async def generate_project_files(
             xlsform,
             form_category,
             form_file_ext,
-            len(task_extract_dict.keys()),
+            task_extract_dict,
             db,
         )
         log.debug(
@@ -969,22 +967,6 @@ async def generate_project_files(
         # Commit all updated database records
         db.commit()
 
-        # Map geojson to entities dict
-        entities_data_dict = await task_geojson_dict_to_entity_values(task_extract_dict)
-        # Create entities
-        # TODO after Entity creation is a single API call,
-        # TODO move to generate_odk_central_project_content
-        async with central_deps.get_odk_entity(odk_credentials) as odk_central:
-            entities = await odk_central.createEntities(
-                project_odk_id,
-                "features",
-                entities_data_dict,
-            )
-            if entities:
-                log.debug(f"Wrote {len(entities)} entities for project ({project_id})")
-            else:
-                log.debug(f"No entities uploaded for project ({project_id})")
-
         if background_task_id:
             # Update background task status to COMPLETED
             await update_background_task_status_in_database(

src/backend/app/projects/project_schemas.py

@@ -18,6 +18,7 @@
 """Pydantic schemas for Projects."""
 
 import uuid
+from dataclasses import dataclass
 from datetime import datetime
 from typing import Any, List, Optional, Union
 
@@ -436,3 +437,39 @@ def get_last_active(self, value, values):
             return f'{days_difference} day{"s" if days_difference > 1 else ""} ago'
         else:
             return last_active.strftime("%d %b %Y")
+
+
+@dataclass
+class Field:
+    """A data class representing a field with a name and type.
+
+    Args:
+        name (str): The name of the field.
+        type (str): The type of the field.
+
+    Returns:
+        None
+    """
+
+    name: str
+    type: str
+
+
+def entity_fields_to_list() -> List[str]:
+    """Converts a list of Field objects to a list of field names.
+
+    Returns:
+        List[str]: A list of fields.
+    """
+    fields: List[Field] = [
+        Field(name="geometry", type="geopoint"),
+        Field(name="project_id", type="string"),
+        Field(name="task_id", type="string"),
+        Field(name="osm_id", type="string"),
+        Field(name="tags", type="string"),
+        Field(name="version", type="string"),
+        Field(name="changeset", type="string"),
+        Field(name="timestamp", type="datetime"),
+        Field(name="status", type="string"),
+    ]
+    return [field.name for field in fields]

src/backend/pyproject.toml

@@ -47,7 +47,7 @@ dependencies = [
     "pyjwt>=2.8.0",
     "async-lru>=2.0.4",
     "osm-login-python==1.0.3",
-    "osm-fieldwork==0.13.0",
+    "osm-fieldwork==0.14.0",
     "osm-rawdata==0.3.0",
     "fmtm-splitter==1.3.0",
 ]