diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/job/JobExportResponse.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/job/JobExportResponse.java new file mode 100644 index 0000000000..05b3e2f417 --- /dev/null +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/job/JobExportResponse.java @@ -0,0 +1,7 @@ +package com.marklogic.hub.job; + +public class JobExportResponse { + public long totalJobs = 0; + public long totalTraces = 0; + public String fullPath = ""; +} diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/job/JobManager.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/job/JobManager.java index 2ec7eff1d1..7a007d41b3 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/job/JobManager.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/job/JobManager.java @@ -107,12 +107,13 @@ public JobDeleteResponse deleteJobs(String jobIds) { * @param exportFilePath specifies where the zip file will be written * @param jobIds a comma-separated list of jobIds; if null, all will be exported */ - public void exportJobs(Path exportFilePath, String jobIds) { + public JobExportResponse exportJobs(Path exportFilePath, String[] jobIds) { + JobExportResponse response = new JobExportResponse(); + response.fullPath = exportFilePath.toAbsolutePath().toString(); + File zipFile = exportFilePath.toFile(); WriteToZipConsumer zipConsumer = new WriteToZipConsumer(zipFile); - String[] jobsArray = jobIds != null ? jobIds.split(",") : null; - QueryManager qm = jobClient.newQueryManager(); // Build a query that will match everything @@ -124,11 +125,11 @@ public void exportJobs(Path exportFilePath, String jobIds) { DataMovementManager dmm = jobClient.newDataMovementManager(); QueryBatcher batcher = null; StructuredQueryDefinition query = null; - if (jobsArray == null) { + if (jobIds == null) { batcher = dmm.newQueryBatcher(emptyQuery); } else { - batcher = dmm.newQueryBatcher(sqb.value(sqb.jsonProperty("jobId"), jobsArray)); + batcher = dmm.newQueryBatcher(sqb.value(sqb.jsonProperty("jobId"), jobIds)); } batcher.onUrisReady(new ExportListener().onDocumentReady(zipConsumer)); JobTicket jobTicket = dmm.startJob(batcher); @@ -139,24 +140,29 @@ public void exportJobs(Path exportFilePath, String jobIds) { JobReport report = dmm.getJobReport(jobTicket); long jobCount = report.getSuccessEventsCount(); + response.totalJobs = jobCount; if (jobCount > 0) { // Get the traces that go with the job(s) dmm = this.traceClient.newDataMovementManager(); - if (jobsArray == null) { + if (jobIds == null) { batcher = dmm.newQueryBatcher(emptyQuery); } else { - batcher = dmm.newQueryBatcher(sqb.value(sqb.element(new QName("jobId")), jobsArray)); + batcher = dmm.newQueryBatcher(sqb.value(sqb.element(new QName("jobId")), jobIds)); } batcher.onUrisReady(new ExportListener().onDocumentReady(zipConsumer)); - dmm.startJob(batcher); + jobTicket = dmm.startJob(batcher); batcher.awaitCompletion(); dmm.stopJob(batcher); dmm.release(); + report = dmm.getJobReport(jobTicket); + long traceCount = report.getSuccessEventsCount(); + response.totalTraces = traceCount; + zipConsumer.close(); } else { @@ -165,6 +171,7 @@ public void exportJobs(Path exportFilePath, String jobIds) { zipFile.delete(); } + return response; } /** diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/job/JobManagerTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/job/JobManagerTest.java index 5f61050a81..324de8b857 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/job/JobManagerTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/job/JobManagerTest.java @@ -205,7 +205,8 @@ public void exportOneJob() throws IOException { File zipFile = exportPath.toFile(); assertFalse(zipFile.exists()); - manager.exportJobs(exportPath, jobIds.get(0)); + String[] jobs = { jobIds.get(0) }; + manager.exportJobs(exportPath, jobs); assertTrue(zipFile.exists()); @@ -227,7 +228,8 @@ public void exportMultipleJobs() throws IOException { File zipFile = exportPath.toFile(); assertFalse(zipFile.exists()); - manager.exportJobs(exportPath, new StringJoiner(",").add(jobIds.get(0)).add(jobIds.get(1)).toString()); + String[] jobs = { jobIds.get(0), jobIds.get(1) }; + manager.exportJobs(exportPath, jobs); assertTrue(zipFile.exists()); diff --git a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/ExportJobsTask.groovy b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/ExportJobsTask.groovy index d689377c5f..99b63891a4 100644 --- a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/ExportJobsTask.groovy +++ b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/ExportJobsTask.groovy @@ -7,7 +7,7 @@ import java.nio.file.Path class ExportJobsTask extends HubTask { @Input - public String jobIds + public String[] jobIds public String filename @TaskAction @@ -16,7 +16,9 @@ class ExportJobsTask extends HubTask { filename = project.hasProperty("filename") ? project.property("filename") : "jobexport.zip" } if (jobIds == null) { - jobIds = project.hasProperty("jobIds") ? project.property("jobIds") : null + if (project.hasProperty("jobIds")) { + jobIds = project.property("jobIds").split(",") + } } if (jobIds == null) { println("Exporting all jobs to " + filename) diff --git a/quick-start/src/main/java/com/marklogic/quickstart/model/JobExport.java b/quick-start/src/main/java/com/marklogic/quickstart/model/JobExport.java new file mode 100644 index 0000000000..11f12f5154 --- /dev/null +++ b/quick-start/src/main/java/com/marklogic/quickstart/model/JobExport.java @@ -0,0 +1,5 @@ +package com.marklogic.quickstart.model; + +public class JobExport { + public String[] jobIds; +} diff --git a/quick-start/src/main/java/com/marklogic/quickstart/service/JobService.java b/quick-start/src/main/java/com/marklogic/quickstart/service/JobService.java index 787abdc72e..f6022a817f 100644 --- a/quick-start/src/main/java/com/marklogic/quickstart/service/JobService.java +++ b/quick-start/src/main/java/com/marklogic/quickstart/service/JobService.java @@ -24,9 +24,14 @@ import com.marklogic.client.query.StructuredQueryBuilder; import com.marklogic.client.query.StructuredQueryDefinition; import com.marklogic.hub.job.JobDeleteResponse; +import com.marklogic.hub.job.JobExportResponse; import com.marklogic.hub.job.JobManager; import com.marklogic.quickstart.model.JobQuery; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; public class JobService extends SearchableService { @@ -89,6 +94,13 @@ public JobDeleteResponse deleteJobs(String jobIds) { return this.jobMgr.deleteJobs(jobIds); } + public File exportJobs(String[] jobIds) throws IOException { + Path exportPath = Files.createTempFile("jobexport", ".zip"); + JobExportResponse jobExportResponse = this.jobMgr.exportJobs(exportPath, jobIds); + File zipfile = new File(jobExportResponse.fullPath); + return zipfile; + } + public void cancelJob(long jobId) { } diff --git a/quick-start/src/main/java/com/marklogic/quickstart/web/JobsController.java b/quick-start/src/main/java/com/marklogic/quickstart/web/JobsController.java index 817147d159..7840e67df3 100644 --- a/quick-start/src/main/java/com/marklogic/quickstart/web/JobsController.java +++ b/quick-start/src/main/java/com/marklogic/quickstart/web/JobsController.java @@ -17,8 +17,11 @@ import com.marklogic.hub.job.JobDeleteResponse; import com.marklogic.quickstart.EnvironmentAware; +import com.marklogic.quickstart.exception.DataHubException; +import com.marklogic.quickstart.model.JobExport; import com.marklogic.quickstart.model.JobQuery; import com.marklogic.quickstart.service.JobService; +import org.apache.commons.io.IOUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Scope; @@ -30,6 +33,8 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; +import java.io.*; + @Controller @RequestMapping(value="/api/jobs") public class JobsController extends EnvironmentAware { @@ -55,4 +60,21 @@ public JobDeleteResponse deleteJobs(@RequestBody String jobIds) { return jobService.deleteJobs(jobIds); } + @RequestMapping(value = "/export", method = RequestMethod.POST) + @ResponseBody + public byte[] exportJobs(@RequestBody JobExport jobExport) { + byte[] response = null; + try { + File zipFile = jobService.exportJobs(jobExport.jobIds); + InputStream is = new FileInputStream(zipFile); + response = IOUtils.toByteArray(is); + } catch (FileNotFoundException e) { + throw new DataHubException(e.getMessage(), e); + } catch (IOException e) { + throw new DataHubException(e.getMessage(), e); + } + + return response; + } + } diff --git a/quick-start/src/main/ui/app/app.module.ts b/quick-start/src/main/ui/app/app.module.ts index b20aa83ba9..0a6f2a50bf 100644 --- a/quick-start/src/main/ui/app/app.module.ts +++ b/quick-start/src/main/ui/app/app.module.ts @@ -61,6 +61,7 @@ import { ObjectToArrayPipe } from './object-to-array.pipe'; import { DatePipeModule } from './date-pipe/date-pipe.module'; import { SelectKeyValuesComponent } from './select-key-values/select-key-values.component'; +import {JobExportDialogComponent} from "./jobs/job-export.component"; @NgModule({ @@ -77,6 +78,7 @@ import { SelectKeyValuesComponent } from './select-key-values/select-key-values. EntityModelerComponent, ExternalDefDialogComponent, JobsComponent, + JobExportDialogComponent, JobOutputComponent, LoginComponent, MlcpUiComponent, @@ -110,7 +112,8 @@ import { SelectKeyValuesComponent } from './select-key-values/select-key-values. EntityEditorComponent, NewEntityComponent, NewFlowComponent, - JobOutputComponent + JobOutputComponent, + JobExportDialogComponent ], imports: [ BrowserModule, diff --git a/quick-start/src/main/ui/app/jobs/index.ts b/quick-start/src/main/ui/app/jobs/index.ts index 77242a224a..94d475083f 100644 --- a/quick-start/src/main/ui/app/jobs/index.ts +++ b/quick-start/src/main/ui/app/jobs/index.ts @@ -1,2 +1,3 @@ export * from './jobs.component'; export * from './job-output.component'; +export * from './job-export.component'; diff --git a/quick-start/src/main/ui/app/jobs/job-export.component.html b/quick-start/src/main/ui/app/jobs/job-export.component.html new file mode 100644 index 0000000000..cc47f83498 --- /dev/null +++ b/quick-start/src/main/ui/app/jobs/job-export.component.html @@ -0,0 +1,10 @@ +
+

Export Jobs and Traces

+
+ +
+
+ + +
+
diff --git a/quick-start/src/main/ui/app/jobs/job-export.component.ts b/quick-start/src/main/ui/app/jobs/job-export.component.ts new file mode 100644 index 0000000000..0099e19e4f --- /dev/null +++ b/quick-start/src/main/ui/app/jobs/job-export.component.ts @@ -0,0 +1,58 @@ +import {Component, HostListener, Inject, Input} from '@angular/core'; +import {MdlDialogReference, MdlDialogService} from '@angular-mdl/core'; +import {JobService} from "./jobs.service"; + +@Component({ + selector: 'job-export-dialog', + templateUrl: 'job-export.component.html' +}) +export class JobExportDialogComponent { + + jobIds: string[]; + question: string; + + constructor( + public dialog: MdlDialogReference, + private dialogService: MdlDialogService, + private jobService: JobService, + @Inject('jobIds') jobIds: string[] + ) { + + this.jobIds = jobIds; + this.question = "Export and download "; + if (jobIds.length === 0) { + this.question += "all jobs and their traces?"; + } else if (this.jobIds.length === 1) { + this.question += "1 job and its traces?"; + } else { + this.question += this.jobIds.length + " jobs and their traces?"; + } + } + + public export() { + this.dialog.hide(); + this.jobService.exportJobs(this.jobIds) + .subscribe(response => { + let body = response['_body']; + + // Create a download anchor tag and click it. + var blob = new Blob([body], {type: 'application/zip'}); + let a = document.createElement("a"); + a.style.display = "none"; + document.body.appendChild(a); + let url = window.URL.createObjectURL(blob); + a.href = url; + a.download = 'jobexport.zip'; + a.click(); + window.URL.revokeObjectURL(url); + }, + () => { + this.dialogService.alert("Unable to export jobs"); + }); + } + + @HostListener('keydown.esc') + public onEsc(): void { + this.dialog.hide(); + } +} diff --git a/quick-start/src/main/ui/app/jobs/jobs.component.html b/quick-start/src/main/ui/app/jobs/jobs.component.html index c219a93636..c52f9e0502 100644 --- a/quick-start/src/main/ui/app/jobs/jobs.component.html +++ b/quick-start/src/main/ui/app/jobs/jobs.component.html @@ -35,10 +35,13 @@

Jobs

Duration Output - - + + + + Export Jobs and Traces + Delete Jobs and Traces + diff --git a/quick-start/src/main/ui/app/jobs/jobs.component.scss b/quick-start/src/main/ui/app/jobs/jobs.component.scss index 19ffbae026..6bbf535926 100644 --- a/quick-start/src/main/ui/app/jobs/jobs.component.scss +++ b/quick-start/src/main/ui/app/jobs/jobs.component.scss @@ -67,14 +67,13 @@ input[type="search"] { th { padding: 12px 8px 12px 8px; color: white; + } - &:last-of-type { - padding-right: 0px; + th.job-actions { + button { + color: white; } } - th .delete-jobs { - color: white; - } } } diff --git a/quick-start/src/main/ui/app/jobs/jobs.component.ts b/quick-start/src/main/ui/app/jobs/jobs.component.ts index 977217d6a5..dca9e18a31 100644 --- a/quick-start/src/main/ui/app/jobs/jobs.component.ts +++ b/quick-start/src/main/ui/app/jobs/jobs.component.ts @@ -9,6 +9,7 @@ import { MdlDialogService, MdlDialogReference } from '@angular-mdl/core'; import { differenceInSeconds } from 'date-fns'; import * as _ from 'lodash'; +import {JobExportDialogComponent} from "./job-export.component"; @Component({ selector: 'app-jobs', @@ -25,7 +26,7 @@ export class JobsComponent implements OnChanges, OnDestroy, OnInit { loadingJobs: boolean = false; searchResponse: SearchResponse; jobs: Array; - jobsToDelete: string[] = []; + selectedJobs: string[] = []; runningFlows: Map = new Map(); facetNames: Array = ['entityName', 'status', 'flowName', 'flowType']; @@ -120,7 +121,7 @@ export class JobsComponent implements OnChanges, OnDestroy, OnInit { this.currentPage, this.pageLength ).subscribe(response => { - this.jobsToDelete.length = 0; + this.selectedJobs.length = 0; this.searchResponse = response; this.jobs = _.map(response.results, (result: any) => { return result.content; @@ -183,19 +184,21 @@ export class JobsComponent implements OnChanges, OnDestroy, OnInit { } toggleDeleteJob(jobId) { - let index = this.jobsToDelete.indexOf(jobId); + let index = this.selectedJobs.indexOf(jobId); if (index > -1) { - this.jobsToDelete.splice(index, 1); + this.selectedJobs.splice(index, 1); } else { - this.jobsToDelete.push(jobId); + this.selectedJobs.push(jobId); } } deleteJobs() { - if (this.jobsToDelete.length > 0) { - const message = 'Delete ' + this.jobsToDelete.length + ' jobs and their traces?'; + if (this.selectedJobs.length > 0) { + let message = 'Delete ' + this.selectedJobs.length + + (this.selectedJobs.length === 1 ? ' job and its traces?' : ' jobs and their traces?'); + this.dialogService.confirm(message, 'Cancel', 'Delete').subscribe(() => { - this.jobService.deleteJobs(this.jobsToDelete) + this.jobService.deleteJobs(this.selectedJobs) .subscribe(response => { this.getJobs(); }, @@ -207,6 +210,18 @@ export class JobsComponent implements OnChanges, OnDestroy, OnInit { } } + exportJobs() { + let pDialog = this.dialogService.showCustomDialog({ + component: JobExportDialogComponent, + providers: [{provide: 'jobIds', useValue: this.selectedJobs}], + isModal: true, + styles: {'width': '350px'}, + clickOutsideToClose: true, + enterTransitionDuration: 400, + leaveTransitionDuration: 400 + }); + } + render(o) { return JSON.stringify(o); } diff --git a/quick-start/src/main/ui/app/jobs/jobs.service.ts b/quick-start/src/main/ui/app/jobs/jobs.service.ts index d687d2589b..0ad591c59b 100644 --- a/quick-start/src/main/ui/app/jobs/jobs.service.ts +++ b/quick-start/src/main/ui/app/jobs/jobs.service.ts @@ -1,5 +1,5 @@ import { Injectable } from '@angular/core'; -import { Http, Response } from '@angular/http'; +import { Headers, Http, RequestOptions, Response, ResponseContentType} from '@angular/http'; @Injectable() export class JobService { @@ -27,6 +27,21 @@ export class JobService { return this.http.post('/api/jobs/delete', jobIds.join(",")); } + exportJobs(jobIds: string[]) { + let options: RequestOptions = new RequestOptions(); + options.responseType = ResponseContentType.Blob; + + return this.http.post( + '/api/jobs/export', + JSON.stringify( + { + "jobIds": jobIds.length ===0 ? null : jobIds + } + ), + options + ); + } + private extractData = (res: Response) => { return res.json(); } diff --git a/quick-start/src/test/java/com/marklogic/quickstart/web/HubConfigJsonTest.java b/quick-start/src/test/java/com/marklogic/quickstart/web/HubConfigJsonTest.java index 0651f668c5..35afe7ab2c 100644 --- a/quick-start/src/test/java/com/marklogic/quickstart/web/HubConfigJsonTest.java +++ b/quick-start/src/test/java/com/marklogic/quickstart/web/HubConfigJsonTest.java @@ -30,7 +30,8 @@ public class HubConfigJsonTest { @Test public void testDeserialize() throws IOException { - String projectPath = new File(PROJECT_PATH).getAbsolutePath(); + // need to escape the backslashes on Windows. On Linux, there won't be any, so the replace will have no effect + String projectPath = new File(PROJECT_PATH).getAbsolutePath().replace("\\", "\\\\"); String content = "{\n" + " \"host\": null,\n" + " \"stagingDbName\": \"data-hub-STAGING\",\n" + @@ -72,7 +73,8 @@ public void testDeserialize() throws IOException { @Test public void testSerialize() throws IOException { - String projectPath = new File(PROJECT_PATH).getAbsolutePath(); + // need to escape the backslashes on Windows. On Linux, there won't be any, so the replace will have no effect + String projectPath = new File(PROJECT_PATH).getAbsolutePath().replace("\\", "\\\\"); HubConfig hubConfig = HubConfigBuilder.newHubConfigBuilder(PROJECT_PATH).withPropertiesFromEnvironment().build(); String expected = "{\n" + " \"stagingDbName\": \"data-hub-STAGING\",\n" +