From 0bcf923745f6787d1e4f6ac75f12e9c8e527ff25 Mon Sep 17 00:00:00 2001 From: Martin Trajanovski Date: Fri, 3 Mar 2023 15:50:30 +0100 Subject: [PATCH 1/2] test: Improve tests with big randomized amount of data --- test/RandomizedDatasetPermissions.js | 408 +++++++++++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100644 test/RandomizedDatasetPermissions.js diff --git a/test/RandomizedDatasetPermissions.js b/test/RandomizedDatasetPermissions.js new file mode 100644 index 000000000..00e424fcb --- /dev/null +++ b/test/RandomizedDatasetPermissions.js @@ -0,0 +1,408 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +"use strict"; + +const { faker } = require("@faker-js/faker"); +var utils = require("./LoginUtils"); + +let accessTokenIngestor = null, + accessTokenUser1 = null, + accessTokenUser2 = null, + accessTokenUser3 = null; +const NUMBER_OF_DATASETS_TO_CREATE = 100; + +let groupedDatasets = { + 1: [], + 2: [], + 3: [], + 4: [], +}; + +function generateRandomDataset() { + return { + principalInvestigator: faker.internet.email(), + endTime: faker.date.past().toISOString(), + creationLocation: faker.system.directoryPath(), + dataFormat: faker.random.words(3), + scientificMetadata: { + approx_file_size_mb: { + value: faker.random.numeric(5), + unit: faker.random.words(2), + }, + beamlineParameters: { + Monostripe: "Ru/C", + "Ring current": { + v: 0.402246, + u: "A", + }, + "Beam energy": { + v: 22595, + u: "eV", + }, + }, + detectorParameters: { + Objective: 20, + Scintillator: "LAG 20um", + "Exposure time": { + v: 0.4, + u: "s", + }, + }, + scanParameters: { + "Number of projections": 1801, + "Rot Y min position": { + v: 0, + u: "deg", + }, + "Inner scan flag": 0, + "File Prefix": "817b_B2_", + "Sample In": { + v: 0, + u: "m", + }, + "Sample folder": "/ramjet/817b_B2_", + "Number of darks": 10, + "Rot Y max position": { + v: 180, + u: "deg", + }, + "Angular step": { + v: 0.1, + u: "deg", + }, + "Number of flats": 120, + "Sample Out": { + v: -0.005, + u: "m", + }, + "Flat frequency": 0, + "Number of inter-flats": 0, + }, + }, + owner: faker.name.fullName(), + ownerEmail: faker.internet.email(), + orcidOfOwner: faker.database.mongodbObjectId(), + contactEmail: faker.internet.email(), + sourceFolder: faker.system.directoryPath(), + size: 0, + packedSize: 0, + numberOfFiles: 0, + numberOfFilesArchived: 0, + creationTime: faker.date.past().toISOString(), + description: faker.random.words(10), + datasetName: faker.random.words(2), + classification: "AV=medium,CO=low", + license: "CC BY-SA 4.0", + isPublished: false, + ownerGroup: faker.helpers.arrayElement([ + "group1", + "group2", + "group3", + "group4", + ]), + accessGroups: [], + proposalId: process.env.PID_PREFIX + ? process.env.PID_PREFIX + : "" + faker.random.numeric(6), + type: "raw", + keywords: ["sls", "protein"], + }; +} + +function randomIntFromInterval(min, max) { + return Math.floor(Math.random() * (max - min + 1) + min); +} + +function addDataset() { + const dataset = generateRandomDataset(); + + return request(appUrl) + .post("/api/v3/Datasets") + .send(dataset) + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenIngestor}` }) + .expect(200) + .expect("Content-Type", /json/) + .then((result) => { + if (result.body) { + return result.body; + } else { + throw new Error( + "Something went wrong while creating dataset:" + + JSON.stringify(dataset), + ); + } + }) + .catch((error) => { + throw new Error( + "Something went wrong while creating dataset:" + + JSON.stringify(dataset) + + error, + ); + }); +} + +async function addAllDatasets() { + const allPromises = []; + + for (let index = 0; index < NUMBER_OF_DATASETS_TO_CREATE; index++) { + allPromises.push(addDataset()); + } + + await Promise.all(allPromises).then(function (values) { + groupedDatasets[1] = values.filter( + (value) => value.ownerGroup === "group1", + ); + groupedDatasets[2] = values.filter( + (value) => value.ownerGroup === "group2", + ); + groupedDatasets[3] = values.filter( + (value) => value.ownerGroup === "group3", + ); + groupedDatasets[4] = values.filter( + (value) => value.ownerGroup === "group4", + ); + }); +} + +describe("Randomized Datasets: permission test with bigger amount of data", async () => { + beforeEach((done) => { + utils.getToken( + appUrl, + { + username: "ingestor", + password: "aman", + }, + (tokenVal) => { + accessTokenIngestor = tokenVal; + utils.getToken( + appUrl, + { + username: "user1", + password: "a609316768619f154ef58db4d847b75e", + }, + (tokenVal) => { + accessTokenUser1 = tokenVal; + utils.getToken( + appUrl, + { + username: "user2", + password: "f522d1d715970073a6413474ca0e0f63", + }, + (tokenVal) => { + accessTokenUser2 = tokenVal; + utils.getToken( + appUrl, + { + username: "user3", + password: "70dc489e8ee823ae815e18d664424df2", + }, + (tokenVal) => { + accessTokenUser3 = tokenVal; + done(); + }, + ); + }, + ); + }, + ); + }, + ); + }); + + it("access private dataset as unauthenticated user", async () => { + await addAllDatasets(); + + return request(appUrl) + .get( + "/api/v3/Datasets/" + + encodeURIComponent( + groupedDatasets[randomIntFromInterval(0, 3)][0].pid, + ), + ) + .set("Accept", "application/json") + .expect("Content-Type", /json/) + .expect(403); + }); + + it("list of datasets for ingestor", async () => { + return request(appUrl) + .get("/api/v3/Datasets") + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenIngestor}` }) + .expect(200) + .expect("Content-Type", /json/) + .then((res) => { + res.body.should.be + .an("array") + .to.have.lengthOf(NUMBER_OF_DATASETS_TO_CREATE); + }); + }); + + it("access any dataset as ingestor", async () => { + const randomGroupIndex = randomIntFromInterval(1, 4); + const randomIndex = randomIntFromInterval( + 0, + groupedDatasets[randomGroupIndex].length - 1, + ); + const randomDatasetPid = groupedDatasets[randomGroupIndex][randomIndex].pid; + + return request(appUrl) + .get("/api/v3/Datasets/" + encodeURIComponent(randomDatasetPid)) + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenIngestor}` }) + .expect("Content-Type", /json/) + .expect(200) + .then((res) => { + res.body["pid"].should.be.equal(randomDatasetPid); + }); + }); + + it("full query for datasets for ingestor", async () => { + return request(appUrl) + .get("/api/v3/Datasets/fullquery") + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenIngestor}` }) + .expect(200) + .expect("Content-Type", /json/) + .then((res) => { + res.body.should.be + .an("array") + .to.have.lengthOf(NUMBER_OF_DATASETS_TO_CREATE); + }); + }); + + it("list of datasets for user 1", async () => { + return request(appUrl) + .get("/api/v3/Datasets") + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenUser1}` }) + .expect(200) + .expect("Content-Type", /json/) + .then((res) => { + res.body.should.be + .an("array") + .to.have.lengthOf(groupedDatasets[1].length); + }); + }); + + it("access any dataset from group1 ownerGroup as user 1", async () => { + const randomIndex = randomIntFromInterval(0, groupedDatasets[1].length - 1); + const randomDatasetPid = groupedDatasets[1][randomIndex].pid; + + return request(appUrl) + .get("/api/v3/Datasets/" + encodeURIComponent(randomDatasetPid)) + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenUser1}` }) + .expect("Content-Type", /json/) + .expect(200) + .then((res) => { + res.body["pid"].should.be.equal(randomDatasetPid); + }); + }); + + it("access dataset from another ownerGroup as user 1", async () => { + const randomGroupIndex = randomIntFromInterval(1, 3); + const randomIndex = randomIntFromInterval( + 0, + groupedDatasets[randomGroupIndex].length - 1, + ); + const randomDatasetPid = groupedDatasets[randomGroupIndex][randomIndex].pid; + + return request(appUrl) + .get("/api/v3/Datasets/" + encodeURIComponent(randomDatasetPid)) + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenUser1}` }) + .expect("Content-Type", /json/) + .expect(403); + }); + + it("list of datasets for user 2", async () => { + return request(appUrl) + .get("/api/v3/Datasets") + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenUser2}` }) + .expect(200) + .expect("Content-Type", /json/) + .then((res) => { + res.body.should.be + .an("array") + .to.have.lengthOf(groupedDatasets[2].length); + }); + }); + + it("access any dataset from group2 ownerGroup as user 2", async () => { + const randomIndex = randomIntFromInterval(0, groupedDatasets[2].length - 1); + const randomDatasetPid = groupedDatasets[2][randomIndex].pid; + + return request(appUrl) + .get("/api/v3/Datasets/" + encodeURIComponent(randomDatasetPid)) + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenUser2}` }) + .expect("Content-Type", /json/) + .expect(200) + .then((res) => { + res.body["pid"].should.be.equal(randomDatasetPid); + }); + }); + + it("access dataset from another ownerGroup as user 2", async () => { + const randomGroupIndex = faker.helpers.arrayElement(["1", "3", "4"]); + const randomIndex = randomIntFromInterval( + 0, + groupedDatasets[randomGroupIndex].length - 1, + ); + const randomDatasetPid = groupedDatasets[randomGroupIndex][randomIndex].pid; + + return request(appUrl) + .get("/api/v3/Datasets/" + encodeURIComponent(randomDatasetPid)) + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenUser2}` }) + .expect("Content-Type", /json/) + .expect(403); + }); + + it("list of datasets for user 3", async () => { + return request(appUrl) + .get("/api/v3/Datasets") + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenUser3}` }) + .expect(200) + .expect("Content-Type", /json/) + .then((res) => { + res.body.should.be + .an("array") + .to.have.lengthOf(groupedDatasets[3].length); + }); + }); + + it("access any dataset from group3 ownerGroup as user 3", async () => { + const randomIndex = randomIntFromInterval(0, groupedDatasets[3].length - 1); + const randomDatasetPid = groupedDatasets[3][randomIndex].pid; + + return request(appUrl) + .get("/api/v3/Datasets/" + encodeURIComponent(randomDatasetPid)) + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenUser3}` }) + .expect("Content-Type", /json/) + .expect(200) + .then((res) => { + res.body["pid"].should.be.equal(randomDatasetPid); + }); + }); + + it("access dataset from another ownerGroup as user 3", async () => { + const randomGroupIndex = faker.helpers.arrayElement(["1", "2", "4"]); + const randomIndex = randomIntFromInterval( + 0, + groupedDatasets[randomGroupIndex].length - 1, + ); + const randomDatasetPid = groupedDatasets[randomGroupIndex][randomIndex].pid; + + return request(appUrl) + .get("/api/v3/Datasets/" + encodeURIComponent(randomDatasetPid)) + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenUser3}` }) + .expect("Content-Type", /json/) + .expect(403); + }); +}); From c8fff1b2f9479833304273549c5eee26fbbc5da9 Mon Sep 17 00:00:00 2001 From: Martin Trajanovski Date: Mon, 6 Mar 2023 15:14:37 +0100 Subject: [PATCH 2/2] fix tests and remove data after finished --- test/DatasetAuthorization.js | 42 +++++++++++++++++++- test/RandomizedDatasetPermissions.js | 58 ++++++++++++++++++++++++---- 2 files changed, 91 insertions(+), 9 deletions(-) diff --git a/test/DatasetAuthorization.js b/test/DatasetAuthorization.js index f5da90969..edbfce086 100644 --- a/test/DatasetAuthorization.js +++ b/test/DatasetAuthorization.js @@ -8,7 +8,8 @@ const sandbox = require("sinon").createSandbox(); let accessTokenIngestor = null, accessTokenUser1 = null, accessTokenUser2 = null, - accessTokenUser3 = null; + accessTokenUser3 = null, + accessTokenArchiveManager = null; let datasetPid1 = null, encodedDatasetPid1 = null, @@ -72,7 +73,17 @@ describe("DatasetAuthorization: Test access to dataset", () => { }, (tokenVal) => { accessTokenUser3 = tokenVal; - done(); + utils.getToken( + appUrl, + { + username: "archiveManager", + password: "aman", + }, + (tokenVal) => { + accessTokenArchiveManager = tokenVal; + done(); + }, + ); }, ); }, @@ -595,4 +606,31 @@ describe("DatasetAuthorization: Test access to dataset", () => { .expect("Content-Type", /json/) .expect(200); }); + + it("should delete dataset 1", async () => { + return request(appUrl) + .delete("/api/v3/datasets/" + encodedDatasetPid1) + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenArchiveManager}` }) + .expect(200) + .expect("Content-Type", /json/); + }); + + it("should delete dataset 2", async () => { + return request(appUrl) + .delete("/api/v3/datasets/" + encodedDatasetPid2) + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenArchiveManager}` }) + .expect(200) + .expect("Content-Type", /json/); + }); + + it("should delete dataset 3", async () => { + return request(appUrl) + .delete("/api/v3/datasets/" + encodedDatasetPid3) + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenArchiveManager}` }) + .expect(200) + .expect("Content-Type", /json/); + }); }); diff --git a/test/RandomizedDatasetPermissions.js b/test/RandomizedDatasetPermissions.js index 00e424fcb..cfbdd4d28 100644 --- a/test/RandomizedDatasetPermissions.js +++ b/test/RandomizedDatasetPermissions.js @@ -4,11 +4,13 @@ const { faker } = require("@faker-js/faker"); var utils = require("./LoginUtils"); +const NUMBER_OF_DATASETS_TO_CREATE = 100; + let accessTokenIngestor = null, accessTokenUser1 = null, accessTokenUser2 = null, - accessTokenUser3 = null; -const NUMBER_OF_DATASETS_TO_CREATE = 100; + accessTokenUser3 = null, + accessTokenArchiveManager = null; let groupedDatasets = { 1: [], @@ -26,7 +28,7 @@ function generateRandomDataset() { scientificMetadata: { approx_file_size_mb: { value: faker.random.numeric(5), - unit: faker.random.words(2), + unit: "bytes", }, beamlineParameters: { Monostripe: "Ru/C", @@ -141,6 +143,18 @@ function addDataset() { }); } +function removeDataset(datasetPid) { + return request(appUrl) + .delete("/api/v3/datasets/" + encodeURIComponent(datasetPid)) + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenArchiveManager}` }) + .expect(200) + .expect("Content-Type", /json/) + .then((result) => { + return result.body; + }); +} + async function addAllDatasets() { const allPromises = []; @@ -164,6 +178,21 @@ async function addAllDatasets() { }); } +async function removeAllDatasets() { + const allPromises = []; + const allDatasets = groupedDatasets[1] + .concat(groupedDatasets[2]) + .concat(groupedDatasets[3]) + .concat(groupedDatasets[4]); + + for (let index = 0; index < allDatasets.length; index++) { + const dataset = allDatasets[index]; + allPromises.push(removeDataset(dataset.pid)); + } + + return Promise.all(allPromises); +} + describe("Randomized Datasets: permission test with bigger amount of data", async () => { beforeEach((done) => { utils.getToken( @@ -198,7 +227,17 @@ describe("Randomized Datasets: permission test with bigger amount of data", asyn }, (tokenVal) => { accessTokenUser3 = tokenVal; - done(); + utils.getToken( + appUrl, + { + username: "archiveManager", + password: "aman", + }, + (tokenVal) => { + accessTokenArchiveManager = tokenVal; + done(); + }, + ); }, ); }, @@ -211,13 +250,12 @@ describe("Randomized Datasets: permission test with bigger amount of data", asyn it("access private dataset as unauthenticated user", async () => { await addAllDatasets(); + const randomGroup = randomIntFromInterval(1, 4); return request(appUrl) .get( "/api/v3/Datasets/" + - encodeURIComponent( - groupedDatasets[randomIntFromInterval(0, 3)][0].pid, - ), + encodeURIComponent(groupedDatasets[randomGroup][0].pid), ) .set("Accept", "application/json") .expect("Content-Type", /json/) @@ -405,4 +443,10 @@ describe("Randomized Datasets: permission test with bigger amount of data", asyn .expect("Content-Type", /json/) .expect(403); }); + + it("should remove all created random datasets", async () => { + return await removeAllDatasets().then((values) => { + values.length.should.be.equal(NUMBER_OF_DATASETS_TO_CREATE); + }); + }); });