From 908f6d9e8a7e00f4ffbf520c9a92eab26b86d162 Mon Sep 17 00:00:00 2001 From: star <15031259256@163.com> Date: Mon, 27 Nov 2023 18:04:53 +0800 Subject: [PATCH] feat(controller): support online eval in fine-tune space (#3031) --- client/starwhale/base/client/models/models.py | 8 ++++++++ .../ai/starwhale/mlops/api/FineTuneController.java | 14 ++++++++++++++ .../java/ai/starwhale/mlops/api/JobController.java | 2 ++ .../mlops/domain/ft/FineTuneAppService.java | 13 +++++++++++++ .../ai/starwhale/mlops/domain/job/JobType.java | 2 +- .../mlops/domain/job/converter/JobConverter.java | 8 ++++---- .../mlops/domain/ft/FineTuneAppServiceTest.java | 10 ++++++++++ .../mlops/domain/job/mapper/JobMapperTest.java | 3 +++ 8 files changed, 55 insertions(+), 5 deletions(-) diff --git a/client/starwhale/base/client/models/models.py b/client/starwhale/base/client/models/models.py index 108085fa4d..fc5346b93b 100644 --- a/client/starwhale/base/client/models/models.py +++ b/client/starwhale/base/client/models/models.py @@ -208,6 +208,7 @@ class BizType(Enum): class Type1(Enum): evaluation = 'EVALUATION' + online_eval = 'ONLINE_EVAL' train = 'TRAIN' fine_tune = 'FINE_TUNE' serving = 'SERVING' @@ -834,6 +835,7 @@ class ExposedLinkVo(SwBaseModel): class JobType(Enum): evaluation = 'EVALUATION' + online_eval = 'ONLINE_EVAL' train = 'TRAIN' fine_tune = 'FINE_TUNE' serving = 'SERVING' @@ -1614,6 +1616,12 @@ class ResponseMessageGraph(SwBaseModel): data: Graph +class ResponseMessageListJobVo(SwBaseModel): + code: str + message: str + data: List[JobVo] + + class FineTuneVo(SwBaseModel): id: int job: JobVo diff --git a/server/controller/src/main/java/ai/starwhale/mlops/api/FineTuneController.java b/server/controller/src/main/java/ai/starwhale/mlops/api/FineTuneController.java index c3f895024f..c5f53d8e42 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/api/FineTuneController.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/api/FineTuneController.java @@ -21,6 +21,7 @@ import ai.starwhale.mlops.api.protocol.ft.FineTuneMigrationRequest; import ai.starwhale.mlops.api.protocol.ft.FineTuneSpaceCreateRequest; import ai.starwhale.mlops.api.protocol.ft.FineTuneSpaceVo; +import ai.starwhale.mlops.api.protocol.job.JobVo; import ai.starwhale.mlops.api.protocol.model.ModelViewVo; import ai.starwhale.mlops.common.IdConverter; import ai.starwhale.mlops.configuration.FeaturesProperties; @@ -156,6 +157,19 @@ public ResponseEntity>> listFineTune( return ResponseEntity.ok(Code.success.asResponse(pageInfo)); } + @Operation(summary = "List online eval") + @GetMapping( + value = "/project/{projectId}/ftspace/{spaceId}/online-eval", + produces = MediaType.APPLICATION_JSON_VALUE + ) + @PreAuthorize("hasAnyRole('OWNER', 'MAINTAINER', 'GUEST')") + public ResponseEntity>> listOnlineEval( + @PathVariable("projectId") Long projectId, + @PathVariable("spaceId") Long spaceId + ) { + return ResponseEntity.ok(Code.success.asResponse(fineTuneAppService.listOnlineEval(projectId, spaceId))); + } + @Operation(summary = "Get fine-tune info") @GetMapping(value = "/project/{projectId}/ftspace/{spaceId}/ft/{ftId}", produces = MediaType.APPLICATION_JSON_VALUE) @PreAuthorize("hasAnyRole('OWNER', 'MAINTAINER', 'GUEST')") diff --git a/server/controller/src/main/java/ai/starwhale/mlops/api/JobController.java b/server/controller/src/main/java/ai/starwhale/mlops/api/JobController.java index 9d9b006517..54e4cf83e0 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/api/JobController.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/api/JobController.java @@ -221,6 +221,8 @@ public ResponseEntity> createJob( jobId = fineTuneAppService.createFineTune(projectUrl, spaceId, jobRequest); } else if (jobRequest.getType() == JobType.EVALUATION) { jobId = fineTuneAppService.createEvaluationJob(projectUrl, spaceId, jobRequest); + } else if (jobRequest.getType() == JobType.ONLINE_EVAL) { + jobId = jobServiceForWeb.createJob(userJobConverter.convert(projectUrl, jobRequest)); } } else { jobId = jobServiceForWeb.createJob(userJobConverter.convert(projectUrl, jobRequest)); diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/FineTuneAppService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/FineTuneAppService.java index 17dd6ebe0d..cd01d1febb 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/FineTuneAppService.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/FineTuneAppService.java @@ -19,6 +19,7 @@ import static ai.starwhale.mlops.domain.evaluation.EvaluationService.TABLE_NAME_FORMAT; import ai.starwhale.mlops.api.protocol.job.JobRequest; +import ai.starwhale.mlops.api.protocol.job.JobVo; import ai.starwhale.mlops.api.protocol.model.ModelViewVo; import ai.starwhale.mlops.api.protocol.model.ModelVo; import ai.starwhale.mlops.common.Constants; @@ -431,6 +432,18 @@ public void releaseFt( } } + public List listOnlineEval(Long projectId, Long spaceId) { + var onlineEvaluations = jobMapper.listBizJobs( + projectId, + BizType.FINE_TUNE.name(), + String.valueOf(spaceId), + JobType.ONLINE_EVAL.name(), + null + ); + return onlineEvaluations.stream() + .map(jobConverter::convert).collect(Collectors.toList()); + } + public List listModelVersionView(Long projectId, Long spaceId) { return modelService.listFtSpaceModelVersionView(String.valueOf(projectId), spaceId); } diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobType.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobType.java index 505a47a33e..be24c06524 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobType.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobType.java @@ -17,5 +17,5 @@ package ai.starwhale.mlops.domain.job; public enum JobType { - EVALUATION, TRAIN, FINE_TUNE, SERVING, BUILT_IN + EVALUATION, ONLINE_EVAL, TRAIN, FINE_TUNE, SERVING, BUILT_IN } diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/converter/JobConverter.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/converter/JobConverter.java index 4632f96c29..ae7b05c710 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/converter/JobConverter.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/converter/JobConverter.java @@ -88,7 +88,7 @@ public JobConverter( this.webServerInTask = webServerInTask; } - private ModelVo findModelByVersionIds(Long versionId) { + private ModelVo findModelByVersionId(Long versionId) { if (null == versionId) { return null; } @@ -99,7 +99,7 @@ private ModelVo findModelByVersionIds(Long versionId) { return modelVos.get(0); } - private RuntimeVo findRuntimeByVersionIds(Long versionId) { + private RuntimeVo findRuntimeByVersionId(Long versionId) { if (null == versionId) { return null; } @@ -186,7 +186,7 @@ private List generateJobExposedLinks(Long jobId) { } public JobVo convert(JobEntity jobEntity) throws ConvertException { - var runtime = findRuntimeByVersionIds(jobEntity.getRuntimeVersionId()); + var runtime = findRuntimeByVersionId(jobEntity.getRuntimeVersionId()); var datasetList = findDatasetVersionsByJobId(jobEntity.getId()); Long pinnedTime = jobEntity.getPinnedTime() != null ? jobEntity.getPinnedTime().getTime() : null; @@ -197,7 +197,7 @@ public JobVo convert(JobEntity jobEntity) throws ConvertException { .owner(UserVo.fromEntity(jobEntity.getOwner(), idConvertor)) .modelName(jobEntity.getModelName()) .modelVersion(jobEntity.getModelVersion().getVersionName()) - .model(findModelByVersionIds(jobEntity.getModelVersionId())) + .model(findModelByVersionId(jobEntity.getModelVersionId())) .jobName(extractJobName(jobEntity.getStepSpec())) .createdTime(jobEntity.getCreatedTime().getTime()) .runtime(runtime) diff --git a/server/controller/src/test/java/ai/starwhale/mlops/domain/ft/FineTuneAppServiceTest.java b/server/controller/src/test/java/ai/starwhale/mlops/domain/ft/FineTuneAppServiceTest.java index b286be7303..eac1ac8ec8 100644 --- a/server/controller/src/test/java/ai/starwhale/mlops/domain/ft/FineTuneAppServiceTest.java +++ b/server/controller/src/test/java/ai/starwhale/mlops/domain/ft/FineTuneAppServiceTest.java @@ -42,7 +42,9 @@ import ai.starwhale.mlops.domain.ft.po.FineTuneEntity; import ai.starwhale.mlops.domain.ft.po.FineTuneSpaceEntity; import ai.starwhale.mlops.domain.ft.vo.FineTuneVo; +import ai.starwhale.mlops.domain.job.BizType; import ai.starwhale.mlops.domain.job.JobCreator; +import ai.starwhale.mlops.domain.job.JobType; import ai.starwhale.mlops.domain.job.bo.Job; import ai.starwhale.mlops.domain.job.bo.UserJobCreateRequest; import ai.starwhale.mlops.domain.job.converter.JobConverter; @@ -174,6 +176,14 @@ void listFt() { assertEquals(1, fineTuneAppService.list(1L, 1, 1).getSize()); } + @Test + void listFtOnlineEval() { + when(jobMapper.listBizJobs(1L, BizType.FINE_TUNE.name(), "1", JobType.ONLINE_EVAL.name(), null)) + .thenReturn(List.of(JobEntity.builder().id(1L).build(), JobEntity.builder().id(2L).build())); + when(jobConverter.convert(any())).thenReturn(JobVo.builder().build()); + assertEquals(2, fineTuneAppService.listOnlineEval(1L, 1L).size()); + } + @Test void ftInfo() { when(fineTuneMapper.findById(anyLong(), anyLong())).thenReturn(FineTuneEntity.builder().jobId(1L).build()); diff --git a/server/controller/src/test/java/ai/starwhale/mlops/domain/job/mapper/JobMapperTest.java b/server/controller/src/test/java/ai/starwhale/mlops/domain/job/mapper/JobMapperTest.java index f0d9198f16..0f47ce6562 100644 --- a/server/controller/src/test/java/ai/starwhale/mlops/domain/job/mapper/JobMapperTest.java +++ b/server/controller/src/test/java/ai/starwhale/mlops/domain/job/mapper/JobMapperTest.java @@ -177,6 +177,9 @@ public void testListBizJobs() { jobs = jobMapper.listBizJobs(project.getId(), BizType.FINE_TUNE.name(), null, JobType.EVALUATION.name(), null); Assertions.assertEquals(1, jobs.size()); + jobs = jobMapper.listBizJobs(project.getId(), BizType.FINE_TUNE.name(), null, JobType.ONLINE_EVAL.name(), null); + Assertions.assertEquals(0, jobs.size()); + jobs = jobMapper.listBizJobs( project.getId(), BizType.FINE_TUNE.name(),