test(backend): submission routes / different formats (#1917)

* feat(tests): add submission fixture for ODK Central project submissions

* feat(tests): add tests for getting submissions

* fix(tests): improve submission fixture

* fix(tests): improve submission fixture

* feat(tests): add tests for downloading submissions as JSON and ZIP file

* feat(tests): get submission count test

* feat(tests): add test for downloading submissions as GeoJSON

* feat(tests): refactor submission handling to use pyodk client for ODK submissions

* feat(tests): add ODK configuration file and refactor submission handling

src/backend/tests/.pyodk_config.toml

@@ -0,0 +1,4 @@
+[central]
+base_url = "https://proxy"
+username = "[email protected]"
+password = "Password1234"

src/backend/tests/conftest.py

@@ -19,6 +19,7 @@
 
 import json
 import os
+import uuid
 from io import BytesIO
 from pathlib import Path
 from typing import Any, AsyncGenerator
@@ -32,6 +33,7 @@
 from httpx import ASGITransport, AsyncClient
 from loguru import logger as log
 from psycopg import AsyncConnection
+from pyodk.client import Client
 
 from app.auth.auth_routes import get_or_create_user
 from app.auth.auth_schemas import AuthUser, FMTMUser
@@ -52,6 +54,7 @@
 odk_central_url = os.getenv("ODK_CENTRAL_URL")
 odk_central_user = os.getenv("ODK_CENTRAL_USER")
 odk_central_password = encrypt_value(os.getenv("ODK_CENTRAL_PASSWD", ""))
+odk_config_file = str(Path(__file__).parent / ".pyodk_config.toml")
 
 
 def pytest_configure(config):
@@ -275,6 +278,94 @@ async def odk_project(db, client, project, tasks):
     yield project
 
 
+@pytest_asyncio.fixture(scope="function")
+async def submission(client, odk_project):
+    """Set up a submission for a project in ODK Central."""
+    odk_project_id = odk_project.odkid
+    odk_credentials = odk_project.odk_credentials
+    odk_creds = odk_credentials.model_dump()
+    base_url = odk_creds["odk_central_url"]
+    auth = (
+        odk_creds["odk_central_user"],
+        odk_creds["odk_central_password"],
+    )
+
+    def forms(base_url, auth, pid):
+        """Fetch a list of forms in a project."""
+        url = f"{base_url}/v1/projects/{pid}/forms"
+        return requests.get(url, auth=auth)
+
+    forms_response = forms(base_url, auth, odk_project_id)
+    assert forms_response.status_code == 200, "Failed to fetch forms from ODK Central"
+    forms = forms_response.json()
+    assert forms, "No forms found in ODK Central project"
+    odk_form_id = forms[0]["xmlFormId"]
+    odk_form_version = forms[0]["version"]
+
+    submission_id = str(uuid.uuid4())
+
+    submission_xml = f"""
+        <data id="{odk_form_id}" version="{odk_form_version}">
+        <meta>
+            <instanceID>{submission_id}</instanceID>
+        </meta>
+        <start>2024-11-15T12:28:23.641Z</start>
+        <end>2024-11-15T12:29:00.876Z</end>
+        <today>2024-11-15</today>
+        <phonenumber/>
+        <deviceid>collect:OOYOOcNu8uOA2G4b</deviceid>
+        <username>testuser</username>
+        <instructions/>
+        <warmup/>
+        <feature/>
+        <null/>
+        <new_feature>12.750577838121643 -24.776785714285722 0.0 0.0</new_feature>
+        <form_category>building</form_category>
+        <xid/>
+        <xlocation>12.750577838121643 -24.776785714285722 0.0 0.0</xlocation>
+        <task_id/>
+        <status>2</status>
+        <survey_questions>
+            <buildings>
+            <category>housing</category>
+            <name/>
+            <building_material/>
+            <building_levels/>
+            <housing/>
+            <provider/>
+            </buildings>
+            <details>
+            <power/>
+            <water/>
+            <age/>
+            <building_prefab/>
+            <building_floor/>
+            <building_roof/>
+            <condition/>
+            <access_roof/>
+            <levels_underground/>
+            </details>
+            <comment/>
+        </survey_questions>
+        </data>
+    """
+
+    with Client(config_path=odk_config_file) as client:
+        submission_data = client.submissions.create(
+            project_id=odk_project_id,
+            form_id=odk_form_id,
+            xml=submission_xml,
+            device_id=None,
+            encoding="utf-8",
+        )
+
+    yield {
+        "project": odk_project,
+        "odk_form_id": odk_form_id,
+        "submission_data": submission_data,
+    }
+
+
 @pytest_asyncio.fixture(scope="function")
 async def entities(odk_project):
     """Get entities data."""

src/backend/tests/test_submission_routes.py

@@ -0,0 +1,155 @@
+# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+#
+# This file is part of FMTM.
+#
+#     FMTM is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     FMTM is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with FMTM.  If not, see <https:#www.gnu.org/licenses/>.
+#
+"""Tests for submission routes."""
+
+import json
+
+import pytest
+
+
+async def test_read_submissions(client, submission):
+    """Test get submissions with a single submission expected."""
+    odk_project = submission["project"]
+    submission_data = submission["submission_data"]
+
+    response = await client.get(f"/submission?project_id={odk_project.id}")
+    assert response.status_code == 200, f"Failed to fetch submissions: {response.text}"
+
+    submission_list = response.json()
+    assert isinstance(submission_list, list), "Expected a list of submissions"
+
+    first_submission = submission_list[0]
+    test_instance_id = submission_data.instanceId
+    assert first_submission["__id"] == test_instance_id, "Instance ID mismatch"
+    assert (
+        first_submission["meta"]["instanceID"] == test_instance_id
+    ), "Meta instanceID mismatch"
+    assert first_submission["__system"]["submitterId"] == str(
+        submission_data.submitterId
+    ), "Submitter ID mismatch"
+
+
+async def test_download_submission_json(client, submission):
+    """Test downloading submissions as JSON."""
+    odk_project = submission["project"]
+
+    response = await client.get(
+        f"/submission/download?project_id={odk_project.id}&export_json=true"
+    )
+
+    assert response.status_code == 200, (
+        f"Failed to download JSON submissions. " f"Response: {response.text}"
+    )
+    assert (
+        "Content-Disposition" in response.headers
+    ), "Missing Content-Disposition header"
+
+    expected_filename = f"{odk_project.slug}_submissions.json"
+
+    assert response.headers["Content-Disposition"].endswith(
+        expected_filename
+    ), f"Expected file name to end with {expected_filename}"
+
+    submissions = response.json()
+    assert isinstance(submissions, dict), "Expected JSON response to be a dictionary"
+    assert "value" in submissions, "Missing 'value' key in JSON response"
+    assert isinstance(submissions["value"], list), "Expected 'value' to be a list"
+    assert len(submissions["value"]) > 0, "Expected at least one submission in 'value'"
+
+
+async def test_download_submission_file(client, submission):
+    """Test downloading submissions as a ZIP file."""
+    odk_project = submission["project"]
+
+    response = await client.get(
+        f"/submission/download?project_id={odk_project.id}&export_json=false"
+    )
+
+    assert response.status_code == 200, (
+        f"Failed to download submissions as file. " f"Response: {response.text}"
+    )
+    assert (
+        "Content-Disposition" in response.headers
+    ), "Missing Content-Disposition header"
+
+    expected_filename = f"{odk_project.slug}.zip"
+
+    assert response.headers["Content-Disposition"].endswith(
+        expected_filename
+    ), f"Expected file name to end with {expected_filename}"
+    assert len(response.content) > 0, "Expected non-empty ZIP file content"
+
+
+async def test_get_submission_count(client, submission):
+    """Test fetching the submission count for a project."""
+    odk_project = submission["project"]
+
+    response = await client.get(
+        f"/submission/get-submission-count?project_id={odk_project.id}"
+    )
+    assert (
+        response.status_code == 200
+    ), f"Failed to fetch submission count. Response: {response.text}"
+
+    submission_count = response.json()
+    assert isinstance(
+        submission_count, int
+    ), "Expected submission count to be an integer"
+    assert submission_count > 0, "Submission count should be greater than zero"
+
+
+async def test_download_submission_geojson(client, submission):
+    """Test downloading submissions as a GeoJSON file."""
+    odk_project = submission["project"]
+
+    response = await client.get(
+        f"/submission/download-submission-geojson?project_id={odk_project.id}"
+    )
+
+    assert (
+        response.status_code == 200
+    ), f"Failed to download GeoJSON submissions. Response: {response.text}"
+
+    assert (
+        "Content-Disposition" in response.headers
+    ), "Missing Content-Disposition header"
+    expected_filename = f"{odk_project.slug}.geojson"
+    assert response.headers["Content-Disposition"].endswith(
+        expected_filename
+    ), f"Expected file name to end with {expected_filename}"
+
+    submission_geojson = json.loads(response.content)
+    assert isinstance(
+        submission_geojson, dict
+    ), "Expected GeoJSON content to be a dictionary"
+    assert "type" in submission_geojson, "Missing 'type' key in GeoJSON"
+    assert (
+        submission_geojson["type"] == "FeatureCollection"
+    ), "GeoJSON type must be 'FeatureCollection'"
+    assert "features" in submission_geojson, "Missing 'features' key in GeoJSON"
+    assert isinstance(
+        submission_geojson["features"], list
+    ), "Expected 'features' to be a list"
+    assert (
+        len(submission_geojson["features"]) > 0
+    ), "Expected at least one feature in 'features'"
+
+
+if __name__ == "__main__":
+    """Main func if file invoked directly."""
+    pytest.main()