refactor(BA-589): Apply pydantic handling decorator to VFolder APIs in storage-proxy (#3562)

changes/3511.feature.md

@@ -0,0 +1 @@
+Add new Pydantic handling api decorator for Request/Response validation

changes/3565.enhance.md

@@ -0,0 +1 @@
+Apply pydantic handling decorator to VFolder APIs in storage-proxy

docs/manager/rest-reference/openapi.json

@@ -3,7 +3,7 @@
   "info": {
     "title": "Backend.AI Manager API",
     "description": "Backend.AI Manager REST API specification",
-    "version": "24.12.1",
+    "version": "25.1.1",
     "contact": {
       "name": "Lablup Inc.",
       "url": "https://docs.backend.ai",

python.lock

@@ -72,6 +72,7 @@
 //     "pydantic~=2.9.2",
 //     "pyhumps~=3.8.0",
 //     "pyroscope-io~=0.8.8",
+//     "pytest-aiohttp~=1.0.5",
 //     "pytest-dependency>=0.6.0",
 //     "pytest>=8.3.3",
 //     "python-dateutil>=2.9",
@@ -458,13 +459,13 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "6975f31fe5e7f2113a41bd387221f31854f285ecbc05527272cd8ba4c50764a3",
-              "url": "https://files.pythonhosted.org/packages/92/23/04a00b3714803e5a58f893eec230b58956e1e8289d3e223d9e294dac3cda/aioresponses-0.7.7-py2.py3-none-any.whl"
+              "hash": "b73bd4400d978855e55004b23a3a84cb0f018183bcf066a85ad392800b5b9a94",
+              "url": "https://files.pythonhosted.org/packages/12/b7/584157e43c98aa89810bc2f7099e7e01c728ecf905a66cf705106009228f/aioresponses-0.7.8-py2.py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "66292f1d5c94a3cb984f3336d806446042adb17347d3089f2d3962dd6e5ba55a",
-              "url": "https://files.pythonhosted.org/packages/27/eb/a69466280306dc9976687cda06d2c9195ff72533192184627f5e7b1d3f1e/aioresponses-0.7.7.tar.gz"
+              "hash": "b861cdfe5dc58f3b8afac7b0a6973d5d7b2cb608dd0f6253d16b8ee8eaf6df11",
+              "url": "https://files.pythonhosted.org/packages/de/03/532bbc645bdebcf3b6af3b25d46655259d66ce69abba7720b71ebfabbade/aioresponses-0.7.8.tar.gz"
             }
           ],
           "project_name": "aioresponses",
@@ -473,7 +474,7 @@
             "packaging>=22.0"
           ],
           "requires_python": null,
-          "version": "0.7.7"
+          "version": "0.7.8"
         },
         {
           "artifacts": [
@@ -1046,66 +1047,61 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "83e560faaec38a956dfb3d62e05e1703ee50432b45b788c09e25107c5058bd71",
-              "url": "https://files.pythonhosted.org/packages/65/77/8bbca82f70b062181cf0ae53fd43f1ac6556f3078884bfef9da2269c06a3/boto3-1.35.99-py3-none-any.whl"
-            },
-            {
-              "algorithm": "sha256",
-              "hash": "e0abd794a7a591d90558e92e29a9f8837d25ece8e3c120e530526fe27eba5fca",
-              "url": "https://files.pythonhosted.org/packages/f7/99/3e8b48f15580672eda20f33439fc1622bd611f6238b6d05407320e1fb98c/boto3-1.35.99.tar.gz"
+              "hash": "f9843a5d06f501d66ada06f5a5417f671823af2cf319e36ceefa1bafaaaaa953",
+              "url": "https://files.pythonhosted.org/packages/79/97/4697aa8050e306d6139815996adeb263ddc83024399a188e8b42587665db/boto3-1.36.3-py3-none-any.whl"
             }
           ],
           "project_name": "boto3",
           "requires_dists": [
-            "botocore<1.36.0,>=1.35.99",
+            "botocore<1.37.0,>=1.36.3",
             "botocore[crt]<2.0a0,>=1.21.0; extra == \"crt\"",
             "jmespath<2.0.0,>=0.7.1",
-            "s3transfer<0.11.0,>=0.10.0"
+            "s3transfer<0.12.0,>=0.11.0"
           ],
           "requires_python": ">=3.8",
-          "version": "1.35.99"
+          "version": "1.36.3"
         },
         {
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "b22d27b6b617fc2d7342090d6129000af2efd20174215948c0d7ae2da0fab445",
-              "url": "https://files.pythonhosted.org/packages/fc/dd/d87e2a145fad9e08d0ec6edcf9d71f838ccc7acdd919acc4c0d4a93515f8/botocore-1.35.99-py3-none-any.whl"
+              "hash": "536ab828e6f90dbb000e3702ac45fd76642113ae2db1b7b1373ad24104e89255",
+              "url": "https://files.pythonhosted.org/packages/9f/14/f952fed35b9c04aa66453b5fb5d1262a5a9f5dfdcb396d387c1ff0c6da41/botocore-1.36.3-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "1eab44e969c39c5f3d9a3104a0836c24715579a455f12b3979a31d7cde51b3c3",
-              "url": "https://files.pythonhosted.org/packages/7c/9c/1df6deceee17c88f7170bad8325aa91452529d683486273928eecfd946d8/botocore-1.35.99.tar.gz"
+              "hash": "775b835e979da5c96548ed1a0b798101a145aec3cd46541d62e27dda5a94d7f8",
+              "url": "https://files.pythonhosted.org/packages/3a/61/69eb06a803c83e0da733b60b2bc65880c18ef2dee19ee10cf8732794a3c1/botocore-1.36.3.tar.gz"
             }
           ],
           "project_name": "botocore",
           "requires_dists": [
-            "awscrt==0.22.0; extra == \"crt\"",
+            "awscrt==0.23.4; extra == \"crt\"",
             "jmespath<2.0.0,>=0.7.1",
             "python-dateutil<3.0.0,>=2.1",
             "urllib3!=2.2.0,<3,>=1.25.4; python_version >= \"3.10\"",
             "urllib3<1.27,>=1.25.4; python_version < \"3.10\""
           ],
           "requires_python": ">=3.8",
-          "version": "1.35.99"
+          "version": "1.36.3"
         },
         {
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292",
-              "url": "https://files.pythonhosted.org/packages/a4/07/14f8ad37f2d12a5ce41206c21820d8cb6561b728e51fad4530dff0552a67/cachetools-5.5.0-py3-none-any.whl"
+              "hash": "b76651fdc3b24ead3c648bbdeeb940c1b04d365b38b4af66788f9ec4a81d42bb",
+              "url": "https://files.pythonhosted.org/packages/ec/4e/de4ff18bcf55857ba18d3a4bd48c8a9fde6bb0980c9d20b263f05387fd88/cachetools-5.5.1-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a",
-              "url": "https://files.pythonhosted.org/packages/c3/38/a0f315319737ecf45b4319a8cd1f3a908e29d9277b46942263292115eee7/cachetools-5.5.0.tar.gz"
+              "hash": "70f238fbba50383ef62e55c6aff6d9673175fe59f7c6782c7a0b9e38f4a9df95",
+              "url": "https://files.pythonhosted.org/packages/d9/74/57df1ab0ce6bc5f6fa868e08de20df8ac58f9c44330c7671ad922d2bbeae/cachetools-5.5.1.tar.gz"
             }
           ],
           "project_name": "cachetools",
           "requires_dists": [],
           "requires_python": ">=3.7",
-          "version": "5.5.0"
+          "version": "5.5.1"
         },
         {
           "artifacts": [
@@ -3031,21 +3027,21 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "f49a827f90062e411f1ce1f854f2aedb3c23353244f8108b89283587397ac10e",
-              "url": "https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl"
+              "hash": "9b6427eb19e479d98acff65196a307c555eb567989e6d88ebbb1b509d9779198",
+              "url": "https://files.pythonhosted.org/packages/e4/ea/d836f008d33151c7a1f62caf3d8dd782e4d15f6a43897f64480c2b8de2ad/prompt_toolkit-3.0.50-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "d6623ab0477a80df74e646bdbc93621143f5caf104206aa29294d53de1a03d90",
-              "url": "https://files.pythonhosted.org/packages/2d/4f/feb5e137aff82f7c7f3248267b97451da3644f6cdc218edfe549fb354127/prompt_toolkit-3.0.48.tar.gz"
+              "hash": "544748f3860a2623ca5cd6d2795e7a14f3d0e1c3c9728359013f79877fc89bab",
+              "url": "https://files.pythonhosted.org/packages/a1/e1/bd15cb8ffdcfeeb2bdc215de3c3cffca11408d829e4b8416dcfe71ba8854/prompt_toolkit-3.0.50.tar.gz"
             }
           ],
           "project_name": "prompt-toolkit",
           "requires_dists": [
             "wcwidth"
           ],
-          "requires_python": ">=3.7.0",
-          "version": "3.0.48"
+          "requires_python": ">=3.8.0",
+          "version": "3.0.50"
         },
         {
           "artifacts": [
@@ -3567,6 +3563,54 @@
           "requires_python": ">=3.8",
           "version": "8.3.4"
         },
+        {
+          "artifacts": [
+            {
+              "algorithm": "sha256",
+              "hash": "63a5360fd2f34dda4ab8e6baee4c5f5be4cd186a403cabd498fced82ac9c561e",
+              "url": "https://files.pythonhosted.org/packages/9a/a7/6e50ba2c0a27a34859a952162e63362a13142ce3c646e925b76de440e102/pytest_aiohttp-1.0.5-py3-none-any.whl"
+            },
+            {
+              "algorithm": "sha256",
+              "hash": "880262bc5951e934463b15e3af8bb298f11f7d4d3ebac970aab425aff10a780a",
+              "url": "https://files.pythonhosted.org/packages/28/ad/7915ae42ca364a66708755517c5d669a7a4921d70d1070d3b660ea716a3e/pytest-aiohttp-1.0.5.tar.gz"
+            }
+          ],
+          "project_name": "pytest-aiohttp",
+          "requires_dists": [
+            "aiohttp>=3.8.1",
+            "coverage==6.2; extra == \"testing\"",
+            "mypy==0.931; extra == \"testing\"",
+            "pytest-asyncio>=0.17.2",
+            "pytest>=6.1.0"
+          ],
+          "requires_python": ">=3.7",
+          "version": "1.0.5"
+        },
+        {
+          "artifacts": [
+            {
+              "algorithm": "sha256",
+              "hash": "0d0bb693f7b99da304a0634afc0a4b19e49d5e0de2d670f38dc4bfa5727c5075",
+              "url": "https://files.pythonhosted.org/packages/61/d8/defa05ae50dcd6019a95527200d3b3980043df5aa445d40cb0ef9f7f98ab/pytest_asyncio-0.25.2-py3-none-any.whl"
+            },
+            {
+              "algorithm": "sha256",
+              "hash": "3f8ef9a98f45948ea91a0ed3dc4268b5326c0e7bce73892acc654df4262ad45f",
+              "url": "https://files.pythonhosted.org/packages/72/df/adcc0d60f1053d74717d21d58c0048479e9cab51464ce0d2965b086bd0e2/pytest_asyncio-0.25.2.tar.gz"
+            }
+          ],
+          "project_name": "pytest-asyncio",
+          "requires_dists": [
+            "coverage>=6.2; extra == \"testing\"",
+            "hypothesis>=5.7.1; extra == \"testing\"",
+            "pytest<9,>=8.2",
+            "sphinx-rtd-theme>=1; extra == \"docs\"",
+            "sphinx>=5.3; extra == \"docs\""
+          ],
+          "requires_python": ">=3.9",
+          "version": "0.25.2"
+        },
         {
           "artifacts": [
             {
@@ -3929,22 +3973,22 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e",
-              "url": "https://files.pythonhosted.org/packages/66/05/7957af15543b8c9799209506df4660cba7afc4cf94bfb60513827e96bed6/s3transfer-0.10.4-py3-none-any.whl"
+              "hash": "8fa0aa48177be1f3425176dfe1ab85dcd3d962df603c3dbfc585e6bf857ef0ff",
+              "url": "https://files.pythonhosted.org/packages/5f/ce/22673f4a85ccc640735b4f8d12178a0f41b5d3c6eda7f33756d10ce56901/s3transfer-0.11.1-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "29edc09801743c21eb5ecbc617a152df41d3c287f67b615f73e5f750583666a7",
-              "url": "https://files.pythonhosted.org/packages/c0/0a/1cdbabf9edd0ea7747efdf6c9ab4e7061b085aa7f9bfc36bb1601563b069/s3transfer-0.10.4.tar.gz"
+              "hash": "3f25c900a367c8b7f7d8f9c34edc87e300bde424f779dc9f0a8ae4f9df9264f6",
+              "url": "https://files.pythonhosted.org/packages/1a/aa/fdd958c626b00e3f046d4004363e7f1a2aba4354f78d65ceb3b217fa5eb8/s3transfer-0.11.1.tar.gz"
             }
           ],
           "project_name": "s3transfer",
           "requires_dists": [
-            "botocore<2.0a.0,>=1.33.2",
-            "botocore[crt]<2.0a.0,>=1.33.2; extra == \"crt\""
+            "botocore<2.0a.0,>=1.36.0",
+            "botocore[crt]<2.0a.0,>=1.36.0; extra == \"crt\""
           ],
           "requires_python": ">=3.8",
-          "version": "0.10.4"
+          "version": "0.11.1"
         },
         {
           "artifacts": [
@@ -5110,6 +5154,7 @@
     "pydantic~=2.9.2",
     "pyhumps~=3.8.0",
     "pyroscope-io~=0.8.8",
+    "pytest-aiohttp~=1.0.5",
     "pytest-dependency>=0.6.0",
     "pytest>=8.3.3",
     "python-dateutil>=2.9",

requirements.txt

@@ -83,6 +83,7 @@ zipstream-new~=1.1.8
 
 # required by ai.backend.test (integration test suite)
 pytest>=8.3.3
+pytest-aiohttp~=1.0.5
 pytest-dependency>=0.6.0
 
 # type stubs

src/ai/backend/common/exception.py

@@ -1,4 +1,7 @@
-from typing import Any, Mapping
+import json
+from typing import Any, Mapping, Optional
+
+from aiohttp import web
 
 
 class ConfigurationError(Exception):
@@ -110,3 +113,86 @@ class VolumeUnmountFailed(RuntimeError):
     """
     Represents a umount process failure.
     """
+
+
+class BackendError(web.HTTPError):
+    """
+    An RFC-7807 error class as a drop-in replacement of the original
+    aiohttp.web.HTTPError subclasses.
+    """
+
+    error_type: str = "https://api.backend.ai/probs/general-error"
+    error_title: str = "General Backend API Error."
+
+    content_type: str
+    extra_msg: Optional[str]
+
+    body_dict: dict[str, Any]
+
+    def __init__(self, extra_msg: str | None = None, extra_data: Optional[Any] = None, **kwargs):
+        super().__init__(**kwargs)
+        self.args = (self.status_code, self.reason, self.error_type)
+        self.empty_body = False
+        self.content_type = "application/problem+json"
+        self.extra_msg = extra_msg
+        self.extra_data = extra_data
+        body = {
+            "type": self.error_type,
+            "title": self.error_title,
+        }
+        if extra_msg is not None:
+            body["msg"] = extra_msg
+        if extra_data is not None:
+            body["data"] = extra_data
+        self.body_dict = body
+        self.body = json.dumps(body).encode()
+
+    def __str__(self):
+        lines = []
+        if self.extra_msg:
+            lines.append(f"{self.error_title} ({self.extra_msg})")
+        else:
+            lines.append(self.error_title)
+        if self.extra_data:
+            lines.append(" -> extra_data: " + repr(self.extra_data))
+        return "\n".join(lines)
+
+    def __repr__(self):
+        lines = []
+        if self.extra_msg:
+            lines.append(
+                f"<{type(self).__name__}({self.status}): {self.error_title} ({self.extra_msg})>"
+            )
+        else:
+            lines.append(f"<{type(self).__name__}({self.status}): {self.error_title}>")
+        if self.extra_data:
+            lines.append(" -> extra_data: " + repr(self.extra_data))
+        return "\n".join(lines)
+
+    def __reduce__(self):
+        return (
+            type(self),
+            (),  # empty the constructor args to make unpickler to use
+            # only the exact current state in __dict__
+            self.__dict__,
+        )
+
+
+class MalformedRequestBody(BackendError, web.HTTPBadRequest):
+    error_type = "https://api.backend.ai/probs/generic-bad-request"
+    error_title = "Malformed request body."
+
+
+class InvalidAPIParameters(BackendError, web.HTTPBadRequest):
+    error_type = "https://api.backend.ai/probs/generic-bad-request"
+    error_title = "Invalid or Missing API Parameters."
+
+
+class MiddlewareParamParsingFailed(BackendError, web.HTTPInternalServerError):
+    error_type = "https://api.backend.ai/probs/internal-server-error"
+    error_title = "Middleware parameter parsing failed."
+
+
+class ParameterNotParsedError(BackendError, web.HTTPInternalServerError):
+    error_type = "https://api.backend.ai/probs/internal-server-error"
+    error_title = "Parameter Not Parsed Error"