From 1741cb5262181182129fd3c9b27cdace7d34dd31 Mon Sep 17 00:00:00 2001 From: tangy5 <58751975+tangy5@users.noreply.github.com> Date: Thu, 16 May 2024 23:52:11 -0700 Subject: [PATCH 1/6] Jwt package for 0.8.2 (#1690) * change to pyjwt replace josejwt Signed-off-by: tangy5 * change to pyjwt replace josejwt Signed-off-by: tangy5 * change to pyjwt replace josejwt Signed-off-by: tangy5 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: tangy5 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- monailabel/endpoints/user/auth.py | 5 +++-- requirements.txt | 2 +- setup.cfg | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/monailabel/endpoints/user/auth.py b/monailabel/endpoints/user/auth.py index 030c6bc7d..2abc0150f 100644 --- a/monailabel/endpoints/user/auth.py +++ b/monailabel/endpoints/user/auth.py @@ -11,11 +11,12 @@ import logging from typing import List, Sequence, Union +import jwt import requests from cachetools import cached from fastapi import Depends, HTTPException, Security, status from fastapi.security import OAuth2PasswordBearer -from jose import JWTError, jwt +from jwt import InvalidTokenError from passlib.context import CryptContext from pydantic import BaseModel @@ -114,7 +115,7 @@ async def get_current_user(token: str = Depends(oauth2_scheme) if settings.MONAI ) try: return from_token(token) - except JWTError as e: + except InvalidTokenError as e: logger.error(e) raise credentials_exception diff --git a/requirements.txt b/requirements.txt index 4887249dc..40c6d9ee0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,7 +35,7 @@ einops==0.7.0 pyyaml==6.0.1 filelock==3.11.0 passlib==1.7.4 -python-jose[cryptography]==3.3.0 +pyjwt==2.8.0 bcrypt==4.1.2 shapely==2.0.4 requests==2.31.0 diff --git a/setup.cfg b/setup.cfg index f360a6fce..01d3413f8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -61,7 +61,7 @@ install_requires = pyyaml==6.0.1 filelock==3.11.0 passlib==1.7.4 - python-jose[cryptography]==3.3.0 + pyjwt==2.8.0 bcrypt==4.1.2 shapely==2.0.4 requests==2.31.0 From bedb27eac988b842647d1c6a631f1e71c0074e6a Mon Sep 17 00:00:00 2001 From: tangy5 <58751975+tangy5@users.noreply.github.com> Date: Tue, 21 May 2024 21:25:13 -0700 Subject: [PATCH 2/6] update bundle versions to avoid conflicts (#1682) * update bundle versions to avoid conflicts Signed-off-by: tangy5 * change monai dependencies to 131 Signed-off-by: tangy5 --------- Signed-off-by: tangy5 Signed-off-by: tangy5 Co-authored-by: tangy5 --- Dockerfile | 2 +- requirements.txt | 2 +- sample-apps/endoscopy/lib/configs/inbody.py | 2 +- sample-apps/pathology/lib/configs/classification_nuclei.py | 2 +- setup.cfg | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8dee1be25..3a5d20268 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ # please run `./runtests.sh --clean && DOCKER_BUILDKIT=1 docker build -t projectmonai/monailabel:latest .` # to use different version of MONAI pass `--build-arg MONAI_IMAGE=...` -ARG MONAI_IMAGE=projectmonai/monai:1.3.0 +ARG MONAI_IMAGE=projectmonai/monai:1.3.1 ARG NODE_IMAGE=node:slim FROM ${NODE_IMAGE} as ohifbuild diff --git a/requirements.txt b/requirements.txt index 40c6d9ee0..d8a8b35c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -monai[nibabel, skimage, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, openslide, fire, mlflow]>=1.3.0 +monai[nibabel, skimage, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, openslide, fire, mlflow]>=1.3.1 uvicorn==0.29.0 pydantic==2.7.0 pydantic-settings==2.2.1 diff --git a/sample-apps/endoscopy/lib/configs/inbody.py b/sample-apps/endoscopy/lib/configs/inbody.py index 7997ab288..12bd5a32e 100644 --- a/sample-apps/endoscopy/lib/configs/inbody.py +++ b/sample-apps/endoscopy/lib/configs/inbody.py @@ -35,7 +35,7 @@ def init(self, name: str, model_dir: str, conf: Dict[str, str], planner: Any, ** super().init(name, model_dir, conf, planner, **kwargs) bundle_name = "endoscopic_inbody_classification" - version = conf.get("inbody", "0.4.4") + version = conf.get("inbody", "0.4.8") zoo_source = conf.get("zoo_source", settings.MONAI_ZOO_SOURCE) self.bundle_path = os.path.join(self.model_dir, bundle_name) diff --git a/sample-apps/pathology/lib/configs/classification_nuclei.py b/sample-apps/pathology/lib/configs/classification_nuclei.py index 3edc0f50b..da04b1cb3 100644 --- a/sample-apps/pathology/lib/configs/classification_nuclei.py +++ b/sample-apps/pathology/lib/configs/classification_nuclei.py @@ -31,7 +31,7 @@ def init(self, name: str, model_dir: str, conf: Dict[str, str], planner: Any, ** bundle_name = "pathology_nuclei_classification" zoo_source = conf.get("zoo_source", settings.MONAI_ZOO_SOURCE) - version = conf.get("classification_nuclei", "0.1.4") + version = conf.get("classification_nuclei", "0.1.7") self.bundle_path = os.path.join(self.model_dir, bundle_name) if not os.path.exists(self.bundle_path): diff --git a/setup.cfg b/setup.cfg index 01d3413f8..419fbd595 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ setup_requires = torch ninja install_requires = - monai[nibabel, skimage, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, openslide, fire, mlflow]>=1.3.0 + monai[nibabel, skimage, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, openslide, fire, mlflow]>=1.3.1 uvicorn==0.29.0 pydantic==2.7.0 pydantic-settings==2.2.1 From ac65ad3e1077d7157c35bce88783268cda405383 Mon Sep 17 00:00:00 2001 From: tangy5 <58751975+tangy5@users.noreply.github.com> Date: Fri, 24 May 2024 13:49:00 -0700 Subject: [PATCH 3/6] update more bundle versions (#1693) Signed-off-by: tangy5 --- sample-apps/endoscopy/lib/configs/tooltracking.py | 2 +- sample-apps/pathology/lib/configs/hovernet_nuclei.py | 2 +- sample-apps/pathology/lib/configs/nuclick.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sample-apps/endoscopy/lib/configs/tooltracking.py b/sample-apps/endoscopy/lib/configs/tooltracking.py index dc84e0910..4c3a87771 100644 --- a/sample-apps/endoscopy/lib/configs/tooltracking.py +++ b/sample-apps/endoscopy/lib/configs/tooltracking.py @@ -35,7 +35,7 @@ def init(self, name: str, model_dir: str, conf: Dict[str, str], planner: Any, ** super().init(name, model_dir, conf, planner, **kwargs) bundle_name = "endoscopic_tool_segmentation" - version = conf.get("tooltracking", "0.5.5") + version = conf.get("tooltracking", "0.5.9") zoo_source = conf.get("zoo_source", settings.MONAI_ZOO_SOURCE) self.bundle_path = os.path.join(self.model_dir, bundle_name) diff --git a/sample-apps/pathology/lib/configs/hovernet_nuclei.py b/sample-apps/pathology/lib/configs/hovernet_nuclei.py index b34c8f105..a5843e2c1 100644 --- a/sample-apps/pathology/lib/configs/hovernet_nuclei.py +++ b/sample-apps/pathology/lib/configs/hovernet_nuclei.py @@ -31,7 +31,7 @@ def init(self, name: str, model_dir: str, conf: Dict[str, str], planner: Any, ** bundle_name = "pathology_nuclei_segmentation_classification" zoo_source = conf.get("zoo_source", settings.MONAI_ZOO_SOURCE) - version = conf.get("hovernet_nuclei", "0.2.1") + version = conf.get("hovernet_nuclei", "0.2.4") self.bundle_path = os.path.join(self.model_dir, bundle_name) if not os.path.exists(self.bundle_path): diff --git a/sample-apps/pathology/lib/configs/nuclick.py b/sample-apps/pathology/lib/configs/nuclick.py index eca4fffd6..2165a4602 100644 --- a/sample-apps/pathology/lib/configs/nuclick.py +++ b/sample-apps/pathology/lib/configs/nuclick.py @@ -31,7 +31,7 @@ def init(self, name: str, model_dir: str, conf: Dict[str, str], planner: Any, ** bundle_name = "pathology_nuclick_annotation" zoo_source = conf.get("zoo_source", settings.MONAI_ZOO_SOURCE) - version = conf.get("nuclick", "0.1.4") + version = conf.get("nuclick", "0.1.9") self.bundle_path = os.path.join(self.model_dir, bundle_name) if not os.path.exists(self.bundle_path): From 8318962e3784377c7c525632457aa672204d5fd8 Mon Sep 17 00:00:00 2001 From: AdrianaNam <94240054+AdrianaNam@users.noreply.github.com> Date: Mon, 17 Jun 2024 16:23:38 +0100 Subject: [PATCH 4/6] fix bug with docker container directory (#1709) --- sample-apps/endoscopy/update_cvat_model.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sample-apps/endoscopy/update_cvat_model.sh b/sample-apps/endoscopy/update_cvat_model.sh index 073866237..d7a08b994 100755 --- a/sample-apps/endoscopy/update_cvat_model.sh +++ b/sample-apps/endoscopy/update_cvat_model.sh @@ -38,13 +38,13 @@ fi if [ $FUNC_NAME == "deepedit" ];then MODEL_PATH="$APP_ROOT/model/$FUNC_NAME.pt" # Replace prior pretrained model with lastest model as current pre-trained model - MODEL_CONTAINER="/opt/conda/monailabel/sample-apps/endoscopy/model/pretrained_$FUNC_NAME.pt" # default model path at function container + MODEL_CONTAINER="/usr/local/monailabel/sample-apps/endoscopy/model/pretrained_$FUNC_NAME.pt" # default model path at function container else # if bundle is used, get bundle name and fetch the model BUNDLE_NAME=${BUNDLENAMES[$FUNC_NAME]} MODEL_PATH="$APP_ROOT/model/$BUNDLE_NAME/models/model.pt" # Update to bundle nuclio container - MODEL_CONTAINER="/opt/conda/monailabel/sample-apps/endoscopy/model/$BUNDLE_NAME/models/model.pt" # default model path at function container + MODEL_CONTAINER="/usr/local/monailabel/sample-apps/endoscopy/model/$BUNDLE_NAME/models/model.pt" # default model path at function container fi # Check if latest model checkpoint is done and saved. From 702379d580b85b585f751cc14741fdee9334a67a Mon Sep 17 00:00:00 2001 From: David Manthey Date: Mon, 17 Jun 2024 11:45:24 -0400 Subject: [PATCH 5/6] Improve the efficiency of working with the DSA. (#1691) * Improve the efficiency of working with the DSA. Before, this often queried all of the items, files, or annotations in the system. Now it only queries what is required to perform the desired task. I've disabled a status check (by having it always return 0 values). If this is actually needed, we should add an endpoint to the DSA to do this efficiently. The listing endpoints do return counts in their headers. The slow query is getting the count of images with annotations. Signed-off-by: David Manthey * Find items regardless of folder; be faster for known folders --------- Signed-off-by: David Manthey --- monailabel/datastore/dsa.py | 43 ++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/monailabel/datastore/dsa.py b/monailabel/datastore/dsa.py index 0b87ff915..365cef24e 100644 --- a/monailabel/datastore/dsa.py +++ b/monailabel/datastore/dsa.py @@ -84,7 +84,7 @@ def get_label_by_image_id(self, image_id: str, tag: str) -> str: def get_annotations_by_image_id(self, image_id: str) -> Dict[str, Dict[str, List]]: image_id, name = self._name_to_id(image_id) - data = self.gc.get("annotation", parameters={"limit": 0}) + data = self.gc.get(f"annotation/item/{image_id}", parameters={"limit": 0}) result: Dict[str, Dict[str, List]] = {} # TODO(avirodov): probably can request only annotation for a given image_id, need to check how. @@ -136,26 +136,40 @@ def get_image(self, image_id: str, params=None) -> Any: def _name_to_id(self, name): folders = self.folders if self.folders else self._get_all_folders() for folder in folders: + # First check if the name is directly present + data = self.gc.get("item", parameters={"folderId": folder, "name": name, "limit": 0}) + for d in data: + if d.get("largeImage"): + return d["_id"], d["name"] + # next check if the name is present in a stem form data = self.gc.get("item", parameters={"folderId": folder, "limit": 0}) for d in data: if d.get("largeImage") and d["name"] == name or Path(d["name"]).stem == name: return d["_id"], d["name"] - return name + # Next check if the name is anywhere in the system + data = self.gc.get("item", parameters={"text": f'"{name}"' if '"' not in name else name, "limit": 0}) + for d in data: + if d.get("largeImage") and d["name"] == name or Path(d["name"]).stem == name: + return d["_id"], d["name"] + # If we fail to find the item, the best we can do is return the name + return name, name def get_image_uri(self, image_id: str) -> str: try: - name = self.get_image_info(image_id)["name"] + info = self.get_image_info(image_id) + name = info["name"] + file_id = info.get("largeImage", {}).get("fileId") except girder_client.HttpError: image_id, name = self._name_to_id(image_id) + file_id = None if self.asset_store_path: - data = self.gc.get(f"item/{image_id}/files", parameters={"limit": 0}) - assets = [d["assetstoreId"] for d in data] - for asset in assets: - files = self.gc.get(f"assetstore/{asset}/files", parameters={"limit": 0}) - for f in files: - if f["itemId"] == image_id: - return str(os.path.join(self.asset_store_path, f["path"])) + if file_id is None: + data = self.gc.get(f"item/{image_id}/files", parameters={"limit": 0}) + file_id = data[0]["_id"] + f = self.gc.get(f"resource/{file_id}?type=file") + if "path" in f and os.path.exists(os.path.join(self.asset_store_path, f["path"])): + return str(os.path.join(self.asset_store_path, f["path"])) else: cached = os.path.join(self.cache_path, name) if os.path.exists(cached): @@ -243,9 +257,14 @@ def get_dataset_archive(self, limit_cases: Optional[int]) -> str: raise NotImplementedError def status(self) -> Dict[str, Any]: + # This is a very costly query, disable it for now + # return { + # "total": len(self.list_images()), + # "completed": len(self.get_labeled_images()), + # } return { - "total": len(self.list_images()), - "completed": len(self.get_labeled_images()), + "total": 0, + "completed": 0, } def json(self): From fc9b5676edbce419fe963f5a4a5361ce5bc2fb82 Mon Sep 17 00:00:00 2001 From: YanxuanLiu <104543031+YanxuanLiu@users.noreply.github.com> Date: Mon, 24 Jun 2024 21:33:44 +0800 Subject: [PATCH 6/6] change to acl format (#1706) Signed-off-by: YanxuanLiu --- .github/workflows/blossom-ci.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/blossom-ci.yml b/.github/workflows/blossom-ci.yml index 737f90233..fd8d7947b 100644 --- a/.github/workflows/blossom-ci.yml +++ b/.github/workflows/blossom-ci.yml @@ -39,14 +39,16 @@ jobs: args: ${{ env.args }} # This job only runs for pull request comments - if: contains('\ - Nic-Ma,\ - SachidanandAlle,\ - diazandr3s,\ - tangy5,\ - wyli,\ - YanxuanLiu,\ - ', format('{0},', github.actor)) && github.event.comment.body == '/build' + if: | + github.event.comment.body == '/build' && + ( + github.actor == 'Nic-Ma' || + github.actor == 'SachidanandAlle' || + github.actor == 'diazandr3s' || + github.actor == 'tangy5' || + github.actor == 'wyli' || + github.actor == 'YanxuanLiu' + ) steps: - name: Check if comment is issued by authorized person run: blossom-ci