Refactor submission endpoints backend/frontend (#1507)

* refactor: submission table, submission details, review state and filtering

* frontend_fix: updated end point and task id param

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix: pre-commit fail

* refactor: removed unnecessary brackets

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

src/backend/app/projects/project_deps.py

@@ -34,7 +34,7 @@
 
 async def get_project_by_id(
     db: Session = Depends(get_db), project_id: Optional[int] = None
-) -> Optional[DbProject]:
+) -> DbProject:
     """Get a single project by id."""
     if not project_id:
         # Skip if no project id passed
@@ -96,3 +96,28 @@ async def get_odk_credentials(db: Session, project_id: int):
         odk_central_user=user,
         odk_central_password=password,
     )
+
+
+async def get_project_xform(db, project_id):
+    """Retrieve the transformation associated with a specific project.
+
+    Args:
+        db: Database connection object.
+        project_id: The ID of the project to retrieve the transformation for.
+
+    Returns:
+        The transformation record associated with the specified project.
+
+    Raises:
+        None
+    """
+    sql = text(
+        """
+        SELECT * FROM xforms
+        WHERE project_id = :project_id;
+    """
+    )
+
+    result = db.execute(sql, {"project_id": project_id})
+    db_xform = result.first()
+    return db_xform

src/backend/app/submissions/submission_crud.py

@@ -32,7 +32,6 @@
 
 import sozipfile.sozipfile as zipfile
 from asgiref.sync import async_to_sync
-from dateutil import parser
 from fastapi import HTTPException, Response
 from fastapi.responses import FileResponse
 from loguru import logger as log
@@ -757,25 +756,20 @@ async def get_submissions_by_date(
 
 async def get_submission_by_project(
     project_id: int,
-    skip: 0,
-    limit: 100,
+    filters: dict,
     db: Session,
-    submitted_by: Optional[str] = None,
-    review_state: Optional[str] = None,
-    submitted_date: Optional[str] = None,
+    task_id: Optional[int] = None,
 ):
     """Get submission by project.
 
     Retrieves a paginated list of submissions for a given project.
 
     Args:
         project_id (int): The ID of the project.
-        skip (int): The number of submissions to skip.
-        limit (int): The maximum number of submissions to retrieve.
+        filters (dict): The filters to apply directly to submissions
+            in odk central.
         db (Session): The database session.
-        submitted_by: username of submitter.
-        review_state: reviewState of the submission.
-        submitted_date: date of submissions.
+        task_id (Optional[int]): The index task of the project.
 
     Returns:
         Tuple[int, List]: A tuple containing the total number of submissions and
@@ -786,89 +780,72 @@ async def get_submission_by_project(
 
     """
     project = await project_crud.get_project(db, project_id)
-    s3_project_path = f"/{project.organisation_id}/{project_id}"
-    s3_submission_path = f"/{s3_project_path}/submission.zip"
+    db_xform = await project_deps.get_project_xform(db, project.id)
+    odk_central = await project_deps.get_odk_credentials(db, project_id)
 
-    try:
-        file = get_obj_from_bucket(settings.S3_BUCKET_NAME, s3_submission_path)
-    except ValueError:
-        return 0, []
+    xform = get_odk_form(odk_central)
+    data = xform.listSubmissions(project.odkid, db_xform.odk_form_id, filters)
+    submissions = data.get("value", [])
+    count = data.get("@odata.count", 0)
 
-    with zipfile.ZipFile(file, "r") as zip_ref:
-        with zip_ref.open("submissions.json") as file_in_zip:
-            content = json.loads(file_in_zip.read())
-    if submitted_by:
-        content = [
-            sub for sub in content if submitted_by.lower() in sub["username"].lower()
-        ]
-    if review_state:
-        content = [
-            sub
-            for sub in content
-            if sub.get("__system", {}).get("reviewState") == review_state
-        ]
-    if submitted_date:
-        content = [
+    if task_id:
+        submissions = [
             sub
-            for sub in content
-            if parser.parse(sub.get("end")).date()
-            == parser.parse(submitted_date).date()
+            for sub in submissions
+            if sub.get("all", {}).get("task_id") == str(task_id)
         ]
 
-    start_index = skip
-    end_index = skip + limit
-    paginated_content = content[start_index:end_index]
-    return len(content), paginated_content
+    return count, submissions
 
 
-async def get_submission_by_task(
-    project: db_models.DbProject,
-    task_id: int,
-    filters: dict,
-    db: Session,
-):
-    """Get submissions and count by task.
+# FIXME this is not needed now it can be directly filtered from submission table
+# async def get_submission_by_task(
+#     project: db_models.DbProject,
+#     task_id: int,
+#     filters: dict,
+#     db: Session,
+# ):
+#     """Get submissions and count by task.
 
-    Args:
-        project: The project instance.
-        task_id: The ID of the task.
-        filters: A dictionary of filters.
-        db: The database session.
+#     Args:
+#         project: The project instance.
+#         task_id: The ID of the task.
+#         filters: A dictionary of filters.
+#         db: The database session.
 
-    Returns:
-        Tuple: A tuple containing the list of submissions and the count.
-    """
-    odk_credentials = await project_deps.get_odk_credentials(db, project.id)
+#     Returns:
+#         Tuple: A tuple containing the list of submissions and the count.
+#     """
+#     odk_credentials = await project_deps.get_odk_credentials(db, project.id)
 
-    xform = get_odk_form(odk_credentials)
-    xform_name = f"{project.project_name_prefix}_task_{task_id}"
-    data = xform.listSubmissions(project.odkid, xform_name, filters)
-    submissions = data.get("value", [])
-    count = data.get("@odata.count", 0)
+#     xform = get_odk_form(odk_credentials)
+#     db_xform = await project_deps.get_project_xform(db, project.id)
+#     data = xform.listSubmissions(project.odkid, db_xform.odk_form_id, filters)
+#     submissions = data.get("value", [])
+#     count = data.get("@odata.count", 0)
 
-    return submissions, count
+#     return submissions, count
 
 
 async def get_submission_detail(
-    project: db_models.DbProject,
-    task_id: int,
     submission_id: str,
+    project: db_models.DbProject,
     db: Session,
 ):
     """Get the details of a submission.
 
     Args:
+        submission_id: The intance uuid of the submission.
         project: The project object representing the project.
-        task_id: The ID of the task associated with the submission.
-        submission_id: The ID of the submission.
         db: The database session.
 
     Returns:
         The details of the submission as a JSON object.
     """
     odk_credentials = await project_deps.get_odk_credentials(db, project.id)
     odk_form = get_odk_form(odk_credentials)
-    xform = f"{project.project_name_prefix}_task_{task_id}"
-    submission = odk_form.getSubmissions(project.odkid, xform, submission_id)
-
-    return json.loads(submission)
+    db_xform = await project_deps.get_project_xform(db, project.id)
+    submission = json.loads(
+        odk_form.getSubmissions(project.odkid, db_xform.odk_form_id, submission_id)
+    )
+    return submission.get("value", [])[0]