From 4824b8e1ea6100d3d5c83c55310637bdcb2fcf72 Mon Sep 17 00:00:00 2001 From: Srinath S Date: Wed, 13 Feb 2019 17:20:20 -0800 Subject: [PATCH] Moving uuid() to hub-utils.sjs, refactoring create and delete document methods (#1873) --- .../ml-modules/root/data-hub/5/impl/flow.sjs | 2 +- .../root/data-hub/5/impl/hub-utils.sjs | 57 +++++++++---------- .../ml-modules/root/data-hub/5/impl/jobs.sjs | 19 +++---- .../root/data-hub/5/impl/process.sjs | 2 +- .../ml-modules/root/data-hub/5/impl/prov.sjs | 28 +++------ 5 files changed, 44 insertions(+), 64 deletions(-) diff --git a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/flow.sjs b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/flow.sjs index 0e4ca76079..9178375b43 100644 --- a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/flow.sjs +++ b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/flow.sjs @@ -75,7 +75,7 @@ class Flow { cts.jsonPropertyValueQuery("name", flowName)])); for (let doc of uris) { if (fn.docAvailable(doc)){ - this.hubUtils.deleteStagingDocument(doc); + this.hubUtils.deleteDocument(doc, this.config.STAGINGDATABASE); } } } diff --git a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/hub-utils.sjs b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/hub-utils.sjs index 10e0c25a1a..e95c4672fd 100644 --- a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/hub-utils.sjs +++ b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/hub-utils.sjs @@ -23,61 +23,56 @@ class HubUtils { } this.config = config; } + getConfig() { return this.config; } - writeJobDocument(docUri, job, collections){ - xdmp.eval('xdmp.documentInsert("' + docUri + '",' + 'job,' + '{permissions:xdmp.defaultPermissions(),collections:[' + collections +']})', + + writeDocument(docUri, content, permissions, collections, database){ + xdmp.eval('xdmp.documentInsert("' + docUri + '",' + 'content,' + '{permissions:' + permissions + ',collections:[' + collections +']})', { - job: job, + content: content, docUri:docUri, + permissions:permissions, collections:collections }, { - database: xdmp.database(this.config.JOBDATABASE), + database: xdmp.database(database), commit: 'auto', update: 'true', ignoreAmps: true }) } - deleteJobDocument(docUri){ + + deleteDocument(docUri, database){ xdmp.eval('xdmp.documentDelete("' + docUri + '")', { docUri:docUri }, { - database: xdmp.database(this.config.JOBDATABASE), + database: xdmp.database(database), commit: 'auto', update: 'true', ignoreAmps: true }) } - writeStagingDocument(docUri, collections){ - xdmp.eval('xdmp.documentInsert("' + docUri + '",' + '{permissions:xdmp.defaultPermissions(),collections:[' + collections +']})', - { - job: job, - docUri:docUri, - collections:collections - }, - { - database: xdmp.database(this.config.STAGINGDATABASE), - commit: 'auto', - update: 'true', - ignoreAmps: true - }) - } - deleteStagingDocument(docUri){ - xdmp.eval('xdmp.documentDelete("' + docUri + '")', - { - docUri:docUri - }, - { - database: xdmp.database(this.config.STAGINGDATABASE), - commit: 'auto', - update: 'true', - ignoreAmps: true - }) + + /** + * Generate and return a UUID + */ + uuid() { + var uuid = "", i, random; + for (i = 0; i < 32; i++) { + random = Math.random() * 16 | 0; + + if (i == 8 || i == 12 || i == 16 || i == 20) { + uuid += "-" + } + uuid += (i == 12 ? 4 : (i == 16 ? (random & 3 | 8) : random)).toString(16); + } + return uuid; } + } module.exports = HubUtils; diff --git a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/jobs.sjs b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/jobs.sjs index 960f8eb36d..7e92ac2c60 100644 --- a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/jobs.sjs +++ b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/jobs.sjs @@ -29,9 +29,8 @@ class Jobs { createJob(flowName, id = null ) { let job = null; - if(id == null){ - //todo find a better way to create a UUID - id = xdmp.random(); + if(!id) { + id = hubutils.uuid(); } job = { job: { @@ -46,7 +45,7 @@ class Jobs { } }; - hubutils.writeJobDocument("/jobs/"+job.job.jobId+".json", job, "'Jobs','Job'"); + hubutils.writeDocument("/jobs/"+job.job.jobId+".json", job, "xdmp.defaultPermissions()", "'Jobs','Job'", this.config.JOBDATABASE); return job; } @@ -68,7 +67,7 @@ class Jobs { cts.jsonPropertyValueQuery("jobId", jobId)])); for (let doc of uris) { if (fn.docAvailable(doc)){ - hubutils.deleteJobDocument(doc); + hubutils.deleteDocument(doc, this.config.JOBDATABASE); } } } @@ -84,7 +83,7 @@ class Jobs { if (jobStatus === "finished" || jobStatus === "finished_with_errors" || jobStatus === "failed"){ docObj.job.timeEnded = fn.currentDateTime(); } - hubutils.writeJobDocument("/jobs/"+ jobId +".json", docObj, "'Jobs','Job'"); + hubutils.writeDocument("/jobs/"+ jobId +".json", docObj, "xdmp.defaultPermissions()", "'Jobs','Job'", this.config.JOBDATABASE); } getLastStepAttempted(jobId) { @@ -123,7 +122,7 @@ class Jobs { batch = { batch: { jobId: jobId, - batchId: xdmp.random(), + batchId: hubutils.uuid(), processor: processor, step: step, batchStatus: "started", @@ -133,14 +132,14 @@ class Jobs { } }; - hubutils.writeJobDocument("/jobs/batches/" + batch.batch.batchId + ".json", batch , "'Jobs','Batch'"); + hubutils.writeDocument("/jobs/batches/" + batch.batch.batchId + ".json", batch , "xdmp.defaultPermissions()", "'Jobs','Batch'", this.config.JOBDATABASE); return jobId; } getBatchDocs(jobId, step=null) { let docs = []; let query = [cts.directoryQuery("/jobs/batches/"),cts.jsonPropertyValueQuery("jobId", jobId)]; - if(step != null) { + if(step) { query.push(cts.jsonPropertyValueQuery("step", step)); } let uris = cts.uris("", null ,cts.andQuery(query)); @@ -167,7 +166,7 @@ class Jobs { if (batchStatus === "finished" || batchStatus === "finished_with_errors" || batchStatus === "failed") { docObj.batch.timeEnded = fn.currentDateTime(); } - hubutils.JobDocument("/jobs/batches/"+ batchId +".json", docObj, "'Jobs','Batch'"); + hubutils.writeDocument("/jobs/batches/"+ batchId +".json", docObj, "xdmp.defaultPermissions()", "'Jobs','Batch'", this.config.JOBDATABASE); } getBatchDoc(jobId, batchId) { diff --git a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/process.sjs b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/process.sjs index b5296f16ee..51b6fecfd9 100644 --- a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/process.sjs +++ b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/process.sjs @@ -90,7 +90,7 @@ class Process { cts.jsonPropertyValueQuery("name", name), cts.jsonPropertyValueQuery("type", type)])); for (let doc of uris) { if (fn.docAvailable(doc)){ - this.hubUtils.deleteStagingDocument(doc); + this.hubUtils.deleteDocument(doc, this.config.STAGINGDATABASE); } } } diff --git a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/prov.sjs b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/prov.sjs index 8a767c7a35..e4cea147f4 100644 --- a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/prov.sjs +++ b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/prov.sjs @@ -17,6 +17,8 @@ const defaultConfig = require("/com.marklogic.hub/config.sjs"); const ps = require('/MarkLogic/provenance'); const op = require('/MarkLogic/optic'); +const HubUtils = require("/data-hub/5/impl/hub-utils.sjs"); +const hubutils = new HubUtils(); class Prov { /** @@ -29,23 +31,7 @@ class Prov { this.config = {}; this.config.granularityLevel = config && config.granularityLevel || 'coarse'; this.config.JOBDATABASE = defaultConfig.JOBDATABASE || 'data-hub-JOBS'; - } - - /** - * Generate and return a UUID for provenance document creation - */ - _uuid() { - var uuid = "", i, random; - for (i = 0; i < 32; i++) { - random = Math.random() * 16 | 0; - - if (i == 8 || i == 12 || i == 16 || i == 20) { - uuid += "-" - } - uuid += (i == 12 ? 4 : (i == 16 ? (random & 3 | 8) : random)).toString(16); - } - return uuid; - } + } /** * Get array of provTypes for a given step type for provenance record creation @@ -164,7 +150,7 @@ class Prov { * - location (doc URI) */ _createIngestStepRecord(jobId, flowId, stepType, docURI, info) { - let provId = this._uuid(); + let provId = hubutils.uuid(); let recordOpts = { provTypes : this._getProvTypesByStep(stepType), relations: { @@ -202,7 +188,7 @@ class Prov { * - location (doc URI) */ _createMappingStepRecord(jobId, flowId, stepType, docURI, info) { - let provId = this._uuid(); + let provId = hubutils.uuid(); let recordOpts = { provTypes : this._getProvTypesByStep(stepType), relations: { @@ -241,7 +227,7 @@ class Prov { * - location (doc URI) */ _createMasteringStepRecord(jobId, flowId, stepType, docURI, info) { - let provId = this._uuid(); + let provId = hubutils.uuid(); let recordOpts = { provTypes : this._getProvTypesByStep(stepType), relations: { @@ -277,7 +263,7 @@ class Prov { * - location (doc URI) */ _createCustomStepRecord(jobId, flowId, stepType, docURI, info) { - let provId = this._uuid(); + let provId = hubutils.uuid(); let recordOpts = { provTypes : this._getProvTypesByStep(stepType), relations: {