Skip to content

Commit

Permalink
Moving uuid() to hub-utils.sjs, refactoring create and delete documen…
Browse files Browse the repository at this point in the history 
…t methods (#1873)
  • Loading branch information
@srinathgit @aebadirad
srinathgit authored and aebadirad committed Feb 14, 2019
1 parent a1728a8 commit 4824b8e
Show file tree
Hide file tree
Showing 5 changed files with 44 additions and 64 deletions.
2 changes: 1 addition & 1 deletion marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/flow.sjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ class Flow {
cts.jsonPropertyValueQuery("name", flowName)]));
for (let doc of uris) {
if (fn.docAvailable(doc)){
this.hubUtils.deleteStagingDocument(doc);
this.hubUtils.deleteDocument(doc, this.config.STAGINGDATABASE);
}
}
}
Expand Down
57 changes: 26 additions & 31 deletions marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/hub-utils.sjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,61 +23,56 @@ class HubUtils {
}
this.config = config;
}

getConfig() {
return this.config;
}
writeJobDocument(docUri, job, collections){
xdmp.eval('xdmp.documentInsert("' + docUri + '",' + 'job,' + '{permissions:xdmp.defaultPermissions(),collections:[' + collections +']})',

writeDocument(docUri, content, permissions, collections, database){
xdmp.eval('xdmp.documentInsert("' + docUri + '",' + 'content,' + '{permissions:' + permissions + ',collections:[' + collections +']})',
{
job: job,
content: content,
docUri:docUri,
permissions:permissions,
collections:collections
},
{
database: xdmp.database(this.config.JOBDATABASE),
database: xdmp.database(database),
commit: 'auto',
update: 'true',
ignoreAmps: true
})
}
deleteJobDocument(docUri){

deleteDocument(docUri, database){
xdmp.eval('xdmp.documentDelete("' + docUri + '")',
{
docUri:docUri
},
{
database: xdmp.database(this.config.JOBDATABASE),
database: xdmp.database(database),
commit: 'auto',
update: 'true',
ignoreAmps: true
})
}
writeStagingDocument(docUri, collections){
xdmp.eval('xdmp.documentInsert("' + docUri + '",' + '{permissions:xdmp.defaultPermissions(),collections:[' + collections +']})',
{
job: job,
docUri:docUri,
collections:collections
},
{
database: xdmp.database(this.config.STAGINGDATABASE),
commit: 'auto',
update: 'true',
ignoreAmps: true
})
}
deleteStagingDocument(docUri){
xdmp.eval('xdmp.documentDelete("' + docUri + '")',
{
docUri:docUri
},
{
database: xdmp.database(this.config.STAGINGDATABASE),
commit: 'auto',
update: 'true',
ignoreAmps: true
})

/**
* Generate and return a UUID
*/
uuid() {
var uuid = "", i, random;
for (i = 0; i < 32; i++) {
random = Math.random() * 16 | 0;

if (i == 8 || i == 12 || i == 16 || i == 20) {
uuid += "-"
}
uuid += (i == 12 ? 4 : (i == 16 ? (random & 3 | 8) : random)).toString(16);
}
return uuid;
}

}

module.exports = HubUtils;
19 changes: 9 additions & 10 deletions marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/jobs.sjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,8 @@ class Jobs {

createJob(flowName, id = null ) {
let job = null;
if(id == null){
//todo find a better way to create a UUID
id = xdmp.random();
if(!id) {
id = hubutils.uuid();
}
job = {
job: {
Expand All @@ -46,7 +45,7 @@ class Jobs {
}
};

hubutils.writeJobDocument("/jobs/"+job.job.jobId+".json", job, "'Jobs','Job'");
hubutils.writeDocument("/jobs/"+job.job.jobId+".json", job, "xdmp.defaultPermissions()", "'Jobs','Job'", this.config.JOBDATABASE);
return job;
}

Expand All @@ -68,7 +67,7 @@ class Jobs {
cts.jsonPropertyValueQuery("jobId", jobId)]));
for (let doc of uris) {
if (fn.docAvailable(doc)){
hubutils.deleteJobDocument(doc);
hubutils.deleteDocument(doc, this.config.JOBDATABASE);
}
}
}
Expand All @@ -84,7 +83,7 @@ class Jobs {
if (jobStatus === "finished" || jobStatus === "finished_with_errors" || jobStatus === "failed"){
docObj.job.timeEnded = fn.currentDateTime();
}
hubutils.writeJobDocument("/jobs/"+ jobId +".json", docObj, "'Jobs','Job'");
hubutils.writeDocument("/jobs/"+ jobId +".json", docObj, "xdmp.defaultPermissions()", "'Jobs','Job'", this.config.JOBDATABASE);
}

getLastStepAttempted(jobId) {
Expand Down Expand Up @@ -123,7 +122,7 @@ class Jobs {
batch = {
batch: {
jobId: jobId,
batchId: xdmp.random(),
batchId: hubutils.uuid(),
processor: processor,
step: step,
batchStatus: "started",
Expand All @@ -133,14 +132,14 @@ class Jobs {
}
};

hubutils.writeJobDocument("/jobs/batches/" + batch.batch.batchId + ".json", batch , "'Jobs','Batch'");
hubutils.writeDocument("/jobs/batches/" + batch.batch.batchId + ".json", batch , "xdmp.defaultPermissions()", "'Jobs','Batch'", this.config.JOBDATABASE);
return jobId;
}

getBatchDocs(jobId, step=null) {
let docs = [];
let query = [cts.directoryQuery("/jobs/batches/"),cts.jsonPropertyValueQuery("jobId", jobId)];
if(step != null) {
if(step) {
query.push(cts.jsonPropertyValueQuery("step", step));
}
let uris = cts.uris("", null ,cts.andQuery(query));
Expand All @@ -167,7 +166,7 @@ class Jobs {
if (batchStatus === "finished" || batchStatus === "finished_with_errors" || batchStatus === "failed") {
docObj.batch.timeEnded = fn.currentDateTime();
}
hubutils.JobDocument("/jobs/batches/"+ batchId +".json", docObj, "'Jobs','Batch'");
hubutils.writeDocument("/jobs/batches/"+ batchId +".json", docObj, "xdmp.defaultPermissions()", "'Jobs','Batch'", this.config.JOBDATABASE);
}

getBatchDoc(jobId, batchId) {
Expand Down
2 changes: 1 addition & 1 deletion marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/process.sjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ class Process {
cts.jsonPropertyValueQuery("name", name), cts.jsonPropertyValueQuery("type", type)]));
for (let doc of uris) {
if (fn.docAvailable(doc)){
this.hubUtils.deleteStagingDocument(doc);
this.hubUtils.deleteDocument(doc, this.config.STAGINGDATABASE);
}
}
}
Expand Down
28 changes: 7 additions & 21 deletions marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/prov.sjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
const defaultConfig = require("/com.marklogic.hub/config.sjs");
const ps = require('/MarkLogic/provenance');
const op = require('/MarkLogic/optic');
const HubUtils = require("/data-hub/5/impl/hub-utils.sjs");
const hubutils = new HubUtils();

class Prov {
/**
Expand All @@ -29,23 +31,7 @@ class Prov {
this.config = {};
this.config.granularityLevel = config && config.granularityLevel || 'coarse';
this.config.JOBDATABASE = defaultConfig.JOBDATABASE || 'data-hub-JOBS';
}

/**
* Generate and return a UUID for provenance document creation
*/
_uuid() {
var uuid = "", i, random;
for (i = 0; i < 32; i++) {
random = Math.random() * 16 | 0;

if (i == 8 || i == 12 || i == 16 || i == 20) {
uuid += "-"
}
uuid += (i == 12 ? 4 : (i == 16 ? (random & 3 | 8) : random)).toString(16);
}
return uuid;
}
}

/**
* Get array of provTypes for a given step type for provenance record creation
Expand Down Expand Up @@ -164,7 +150,7 @@ class Prov {
* - location (doc URI)
*/
_createIngestStepRecord(jobId, flowId, stepType, docURI, info) {
let provId = this._uuid();
let provId = hubutils.uuid();
let recordOpts = {
provTypes : this._getProvTypesByStep(stepType),
relations: {
Expand Down Expand Up @@ -202,7 +188,7 @@ class Prov {
* - location (doc URI)
*/
_createMappingStepRecord(jobId, flowId, stepType, docURI, info) {
let provId = this._uuid();
let provId = hubutils.uuid();
let recordOpts = {
provTypes : this._getProvTypesByStep(stepType),
relations: {
Expand Down Expand Up @@ -241,7 +227,7 @@ class Prov {
* - location (doc URI)
*/
_createMasteringStepRecord(jobId, flowId, stepType, docURI, info) {
let provId = this._uuid();
let provId = hubutils.uuid();
let recordOpts = {
provTypes : this._getProvTypesByStep(stepType),
relations: {
Expand Down Expand Up @@ -277,7 +263,7 @@ class Prov {
* - location (doc URI)
*/
_createCustomStepRecord(jobId, flowId, stepType, docURI, info) {
let provId = this._uuid();
let provId = hubutils.uuid();
let recordOpts = {
provTypes : this._getProvTypesByStep(stepType),
relations: {
Expand Down

0 comments on commit 4824b8e

Please sign in to comment.