From 86f50a9565f06eec91efb2c915171fdebf7451c6 Mon Sep 17 00:00:00 2001 From: "Joseph P. White" Date: Mon, 4 Feb 2019 15:40:07 -0500 Subject: [PATCH 001/217] Fix reload button --- html/gui/js/modules/Usage.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/html/gui/js/modules/Usage.js b/html/gui/js/modules/Usage.js index 60533211b9..e97e4133a7 100644 --- a/html/gui/js/modules/Usage.js +++ b/html/gui/js/modules/Usage.js @@ -557,6 +557,8 @@ Ext.extend(XDMoD.Module.Usage, XDMoD.PortalModule, { }; //updateDisabledMenus + var tree; + // --------------------------------------------------------- function reloadTree() { @@ -567,6 +569,8 @@ Ext.extend(XDMoD.Module.Usage, XDMoD.PortalModule, { if (selNode) selModel.unselect(selNode, true); tree.root.removeAll(true); + delete tree.loader.baseParams.category; + delete tree.loader.baseParams.group_by; tree.loader.on('load', selectFirstNode, this, { single: true }); @@ -668,7 +672,7 @@ Ext.extend(XDMoD.Module.Usage, XDMoD.PortalModule, { // --------------------------------------------------------- - var tree = new Ext.tree.TreePanel({ + tree = new Ext.tree.TreePanel({ id: 'tree_' + this.id, useArrows: true, From 12e7b5480ea68d6dc6c873aa38ae8f7318012eb3 Mon Sep 17 00:00:00 2001 From: Ben Plessinger Date: Wed, 6 Feb 2019 09:28:29 -0500 Subject: [PATCH 002/217] Remove definer as it isnt needed, and causes issues for people with multi server setup (#791) --- configuration/etl/etl_tables.d/xdb/users.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/configuration/etl/etl_tables.d/xdb/users.json b/configuration/etl/etl_tables.d/xdb/users.json index 7ff9716e8b..11ada42fb7 100644 --- a/configuration/etl/etl_tables.d/xdb/users.json +++ b/configuration/etl/etl_tables.d/xdb/users.json @@ -129,7 +129,6 @@ "event": "INSERT", "table": "Users", "body": "\n\n\tSET NEW.time_created = NOW();\n\n\tSET NEW.time_last_updated = NOW();\n\n", - "definer": "xdmod@localhost", "schema": "moddb", "name": "users_insert_timestamp" }, @@ -138,7 +137,6 @@ "event": "UPDATE", "table": "Users", "body": "\n\n\tIF NEW.password <> OLD.password THEN\n\n\t\tSET NEW.password_last_updated = NOW(); \n\n\tEND IF; \n\n\tSET NEW.time_last_updated = NOW();\n\n", - "definer": "xdmod@localhost", "schema": "moddb", "name": "users_update_timestamp" } From 1af7c6ed39381ea65356e41020e152186ac1bd70 Mon Sep 17 00:00:00 2001 From: "Joseph P. White" Date: Wed, 6 Feb 2019 13:43:04 -0500 Subject: [PATCH 003/217] Update cloud tests with correct values --- .../regression_tests/lib/Controllers/UsageExplorerTest.php | 2 +- .../configuration/cloud_core_time/aggregate-Day-reference.csv | 4 ++-- .../cloud_core_time/aggregate-Month-reference.csv | 4 ++-- .../cloud_core_time/aggregate-Quarter-reference.csv | 4 ++-- .../cloud_core_time/aggregate-Year-reference.csv | 4 ++-- .../cloud_core_time/timeseries-Month-reference.csv | 2 +- .../cloud_core_time/timeseries-Quarter-reference.csv | 2 +- .../cloud_core_time/timeseries-Year-reference.csv | 2 +- .../configuration/cloud_wall_time/aggregate-Day-reference.csv | 4 ++-- .../cloud_wall_time/aggregate-Month-reference.csv | 4 ++-- .../cloud_wall_time/aggregate-Quarter-reference.csv | 4 ++-- .../cloud_wall_time/aggregate-Year-reference.csv | 4 ++-- .../cloud_wall_time/timeseries-Month-reference.csv | 2 +- .../cloud_wall_time/timeseries-Quarter-reference.csv | 2 +- .../cloud_wall_time/timeseries-Year-reference.csv | 2 +- .../none/cloud_avg_cores_reserved/aggregate-Day-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Month-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Quarter-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Year-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Day-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Month-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Quarter-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Year-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Day-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Month-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Quarter-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Year-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Day-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Month-reference.csv | 2 +- .../timeseries-Quarter-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Year-reference.csv | 2 +- .../Cloud/none/cloud_core_time/aggregate-Day-reference.csv | 2 +- .../Cloud/none/cloud_core_time/aggregate-Month-reference.csv | 2 +- .../none/cloud_core_time/aggregate-Quarter-reference.csv | 2 +- .../Cloud/none/cloud_core_time/aggregate-Year-reference.csv | 2 +- .../Cloud/none/cloud_core_time/timeseries-Month-reference.csv | 2 +- .../none/cloud_core_time/timeseries-Quarter-reference.csv | 2 +- .../Cloud/none/cloud_core_time/timeseries-Year-reference.csv | 2 +- .../Cloud/none/cloud_wall_time/aggregate-Day-reference.csv | 2 +- .../Cloud/none/cloud_wall_time/aggregate-Month-reference.csv | 2 +- .../none/cloud_wall_time/aggregate-Quarter-reference.csv | 2 +- .../Cloud/none/cloud_wall_time/aggregate-Year-reference.csv | 2 +- .../Cloud/none/cloud_wall_time/timeseries-Month-reference.csv | 2 +- .../none/cloud_wall_time/timeseries-Quarter-reference.csv | 2 +- .../Cloud/none/cloud_wall_time/timeseries-Year-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Day-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Month-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Quarter-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Year-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Day-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Month-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Quarter-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Year-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Day-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Month-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Quarter-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Year-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Day-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Month-reference.csv | 2 +- .../timeseries-Quarter-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Year-reference.csv | 2 +- .../Cloud/project/cloud_core_time/aggregate-Day-reference.csv | 2 +- .../project/cloud_core_time/aggregate-Month-reference.csv | 2 +- .../project/cloud_core_time/aggregate-Quarter-reference.csv | 2 +- .../project/cloud_core_time/aggregate-Year-reference.csv | 2 +- .../project/cloud_core_time/timeseries-Month-reference.csv | 2 +- .../project/cloud_core_time/timeseries-Quarter-reference.csv | 2 +- .../project/cloud_core_time/timeseries-Year-reference.csv | 2 +- .../Cloud/project/cloud_wall_time/aggregate-Day-reference.csv | 2 +- .../project/cloud_wall_time/aggregate-Month-reference.csv | 2 +- .../project/cloud_wall_time/aggregate-Quarter-reference.csv | 2 +- .../project/cloud_wall_time/aggregate-Year-reference.csv | 2 +- .../project/cloud_wall_time/timeseries-Month-reference.csv | 2 +- .../project/cloud_wall_time/timeseries-Quarter-reference.csv | 2 +- .../project/cloud_wall_time/timeseries-Year-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Day-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Month-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Quarter-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Year-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Day-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Month-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Quarter-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Year-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Day-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Month-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Quarter-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Year-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Day-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Month-reference.csv | 2 +- .../timeseries-Quarter-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Year-reference.csv | 2 +- .../resource/cloud_core_time/aggregate-Day-reference.csv | 2 +- .../resource/cloud_core_time/aggregate-Month-reference.csv | 2 +- .../resource/cloud_core_time/aggregate-Quarter-reference.csv | 2 +- .../resource/cloud_core_time/aggregate-Year-reference.csv | 2 +- .../resource/cloud_core_time/timeseries-Month-reference.csv | 2 +- .../resource/cloud_core_time/timeseries-Quarter-reference.csv | 2 +- .../resource/cloud_core_time/timeseries-Year-reference.csv | 2 +- .../resource/cloud_wall_time/aggregate-Day-reference.csv | 2 +- .../resource/cloud_wall_time/aggregate-Month-reference.csv | 2 +- .../resource/cloud_wall_time/aggregate-Quarter-reference.csv | 2 +- .../resource/cloud_wall_time/aggregate-Year-reference.csv | 2 +- .../resource/cloud_wall_time/timeseries-Month-reference.csv | 2 +- .../resource/cloud_wall_time/timeseries-Quarter-reference.csv | 2 +- .../resource/cloud_wall_time/timeseries-Year-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Day-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Month-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Quarter-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Year-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Day-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Month-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Quarter-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Year-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Day-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Month-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Quarter-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Year-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Day-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Month-reference.csv | 2 +- .../timeseries-Quarter-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Year-reference.csv | 2 +- .../cloud_core_time/aggregate-Day-reference.csv | 2 +- .../cloud_core_time/aggregate-Month-reference.csv | 2 +- .../cloud_core_time/aggregate-Quarter-reference.csv | 2 +- .../cloud_core_time/aggregate-Year-reference.csv | 2 +- .../cloud_core_time/timeseries-Month-reference.csv | 2 +- .../cloud_core_time/timeseries-Quarter-reference.csv | 2 +- .../cloud_core_time/timeseries-Year-reference.csv | 2 +- .../cloud_wall_time/aggregate-Day-reference.csv | 2 +- .../cloud_wall_time/aggregate-Month-reference.csv | 2 +- .../cloud_wall_time/aggregate-Quarter-reference.csv | 2 +- .../cloud_wall_time/aggregate-Year-reference.csv | 2 +- .../cloud_wall_time/timeseries-Month-reference.csv | 2 +- .../cloud_wall_time/timeseries-Quarter-reference.csv | 2 +- .../cloud_wall_time/timeseries-Year-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Day-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Month-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Quarter-reference.csv | 2 +- .../cloud_avg_memory_reserved/aggregate-Year-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Day-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Month-reference.csv | 2 +- .../timeseries-Quarter-reference.csv | 2 +- .../cloud_avg_memory_reserved/timeseries-Year-reference.csv | 2 +- .../Cloud/vm_size/cloud_core_time/aggregate-Day-reference.csv | 2 +- .../vm_size/cloud_core_time/aggregate-Month-reference.csv | 2 +- .../vm_size/cloud_core_time/aggregate-Quarter-reference.csv | 2 +- .../vm_size/cloud_core_time/aggregate-Year-reference.csv | 2 +- .../vm_size/cloud_core_time/timeseries-Month-reference.csv | 2 +- .../vm_size/cloud_core_time/timeseries-Quarter-reference.csv | 2 +- .../vm_size/cloud_core_time/timeseries-Year-reference.csv | 2 +- .../Cloud/vm_size/cloud_wall_time/aggregate-Day-reference.csv | 2 +- .../vm_size/cloud_wall_time/aggregate-Month-reference.csv | 2 +- .../vm_size/cloud_wall_time/aggregate-Quarter-reference.csv | 2 +- .../vm_size/cloud_wall_time/aggregate-Year-reference.csv | 2 +- .../vm_size/cloud_wall_time/timeseries-Month-reference.csv | 2 +- .../vm_size/cloud_wall_time/timeseries-Quarter-reference.csv | 2 +- .../vm_size/cloud_wall_time/timeseries-Year-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Day-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Month-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Quarter-reference.csv | 2 +- .../cloud_avg_cores_reserved/aggregate-Year-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Day-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Month-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Quarter-reference.csv | 2 +- .../cloud_avg_cores_reserved/timeseries-Year-reference.csv | 2 +- .../cloud_core_time/aggregate-Day-reference.csv | 4 ++-- .../cloud_core_time/aggregate-Month-reference.csv | 4 ++-- .../cloud_core_time/aggregate-Quarter-reference.csv | 4 ++-- .../cloud_core_time/aggregate-Year-reference.csv | 4 ++-- .../cloud_core_time/timeseries-Month-reference.csv | 2 +- .../cloud_core_time/timeseries-Quarter-reference.csv | 2 +- .../cloud_core_time/timeseries-Year-reference.csv | 2 +- .../cloud_wall_time/aggregate-Day-reference.csv | 4 ++-- .../cloud_wall_time/aggregate-Month-reference.csv | 4 ++-- .../cloud_wall_time/aggregate-Quarter-reference.csv | 4 ++-- .../cloud_wall_time/aggregate-Year-reference.csv | 4 ++-- .../cloud_wall_time/timeseries-Month-reference.csv | 2 +- .../cloud_wall_time/timeseries-Quarter-reference.csv | 2 +- .../cloud_wall_time/timeseries-Year-reference.csv | 2 +- 179 files changed, 195 insertions(+), 195 deletions(-) diff --git a/open_xdmod/modules/xdmod/regression_tests/lib/Controllers/UsageExplorerTest.php b/open_xdmod/modules/xdmod/regression_tests/lib/Controllers/UsageExplorerTest.php index d530920de4..079feaa8ae 100644 --- a/open_xdmod/modules/xdmod/regression_tests/lib/Controllers/UsageExplorerTest.php +++ b/open_xdmod/modules/xdmod/regression_tests/lib/Controllers/UsageExplorerTest.php @@ -18,7 +18,7 @@ class UsageExplorerTest extends \PHPUnit_Framework_TestCase protected static $replacements = array( 'REPLACED' ); - protected $delta = 1.0e-2; + protected $delta = 1.0e-8; /* * Allow for skipping of certain tests that have known issues */ diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Day-reference.csv index 4959314478..1b070456c3 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Day-reference.csv @@ -6,9 +6,9 @@ start,end 2018-04-18,2018-04-30 --------- "Instance Type","Core Hours: Total" -c1.m4,906.3831 +c1.m4,904.6564 c4.m16,423.1789 c2.m4,419.2372 -c1.m1,6.6369 +c1.m1,6.6203 c2.m8,0.4717 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Month-reference.csv index 4959314478..1b070456c3 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Month-reference.csv @@ -6,9 +6,9 @@ start,end 2018-04-18,2018-04-30 --------- "Instance Type","Core Hours: Total" -c1.m4,906.3831 +c1.m4,904.6564 c4.m16,423.1789 c2.m4,419.2372 -c1.m1,6.6369 +c1.m1,6.6203 c2.m8,0.4717 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Quarter-reference.csv index 4959314478..1b070456c3 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Quarter-reference.csv @@ -6,9 +6,9 @@ start,end 2018-04-18,2018-04-30 --------- "Instance Type","Core Hours: Total" -c1.m4,906.3831 +c1.m4,904.6564 c4.m16,423.1789 c2.m4,419.2372 -c1.m1,6.6369 +c1.m1,6.6203 c2.m8,0.4717 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Year-reference.csv index 4959314478..1b070456c3 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/aggregate-Year-reference.csv @@ -6,9 +6,9 @@ start,end 2018-04-18,2018-04-30 --------- "Instance Type","Core Hours: Total" -c1.m4,906.3831 +c1.m4,904.6564 c4.m16,423.1789 c2.m4,419.2372 -c1.m1,6.6369 +c1.m1,6.6203 c2.m8,0.4717 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/timeseries-Month-reference.csv index 72d5a7eb74..0746769345 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[c1.m4] Core Hours: Total","[c4.m16] Core Hours: Total","[c2.m4] Core Hours: Total","[c1.m1] Core Hours: Total","[c2.m8] Core Hours: Total" -2018-04,906.3831,423.1789,419.2372,6.6369,0.4717 +2018-04,904.6564,423.1789,419.2372,6.6203,0.4717 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/timeseries-Quarter-reference.csv index 90c77cabfe..9eb8811ef4 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[c1.m4] Core Hours: Total","[c4.m16] Core Hours: Total","[c2.m4] Core Hours: Total","[c1.m1] Core Hours: Total","[c2.m8] Core Hours: Total" -"2018 Q2",906.3831,423.1789,419.2372,6.6369,0.4717 +"2018 Q2",904.6564,423.1789,419.2372,6.6203,0.4717 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/timeseries-Year-reference.csv index 7db12857b3..5ddfcdfbe8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_core_time/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[c1.m4] Core Hours: Total","[c4.m16] Core Hours: Total","[c2.m4] Core Hours: Total","[c1.m1] Core Hours: Total","[c2.m8] Core Hours: Total" -2018,906.3831,423.1789,419.2372,6.6369,0.4717 +2018,904.6564,423.1789,419.2372,6.6203,0.4717 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Day-reference.csv index 061a22c916..5c3bbacbfa 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Day-reference.csv @@ -6,9 +6,9 @@ start,end 2018-04-18,2018-04-30 --------- "Instance Type","Wall Hours: Total" -c1.m4,906.3831 +c1.m4,904.6564 c2.m4,209.6186 c4.m16,105.7947 -c1.m1,6.6369 +c1.m1,6.6203 c2.m8,0.2358 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Month-reference.csv index 061a22c916..5c3bbacbfa 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Month-reference.csv @@ -6,9 +6,9 @@ start,end 2018-04-18,2018-04-30 --------- "Instance Type","Wall Hours: Total" -c1.m4,906.3831 +c1.m4,904.6564 c2.m4,209.6186 c4.m16,105.7947 -c1.m1,6.6369 +c1.m1,6.6203 c2.m8,0.2358 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Quarter-reference.csv index 061a22c916..5c3bbacbfa 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Quarter-reference.csv @@ -6,9 +6,9 @@ start,end 2018-04-18,2018-04-30 --------- "Instance Type","Wall Hours: Total" -c1.m4,906.3831 +c1.m4,904.6564 c2.m4,209.6186 c4.m16,105.7947 -c1.m1,6.6369 +c1.m1,6.6203 c2.m8,0.2358 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Year-reference.csv index 061a22c916..5c3bbacbfa 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/aggregate-Year-reference.csv @@ -6,9 +6,9 @@ start,end 2018-04-18,2018-04-30 --------- "Instance Type","Wall Hours: Total" -c1.m4,906.3831 +c1.m4,904.6564 c2.m4,209.6186 c4.m16,105.7947 -c1.m1,6.6369 +c1.m1,6.6203 c2.m8,0.2358 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/timeseries-Month-reference.csv index 369e787218..3cadb3af4e 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[c1.m4] Wall Hours: Total","[c2.m4] Wall Hours: Total","[c4.m16] Wall Hours: Total","[c1.m1] Wall Hours: Total","[c2.m8] Wall Hours: Total" -2018-04,906.3831,209.6186,105.7947,6.6369,0.2358 +2018-04,904.6564,209.6186,105.7947,6.6203,0.2358 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/timeseries-Quarter-reference.csv index 92360b4a1a..b4a485e47b 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[c1.m4] Wall Hours: Total","[c2.m4] Wall Hours: Total","[c4.m16] Wall Hours: Total","[c1.m1] Wall Hours: Total","[c2.m8] Wall Hours: Total" -"2018 Q2",906.3831,209.6186,105.7947,6.6369,0.2358 +"2018 Q2",904.6564,209.6186,105.7947,6.6203,0.2358 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/timeseries-Year-reference.csv index b414c1b3f1..5d592929a1 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/configuration/cloud_wall_time/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[c1.m4] Wall Hours: Total","[c2.m4] Wall Hours: Total","[c4.m16] Wall Hours: Total","[c1.m1] Wall Hours: Total","[c2.m8] Wall Hours: Total" -2018,906.3831,209.6186,105.7947,6.6369,0.2358 +2018,904.6564,209.6186,105.7947,6.6203,0.2358 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Day-reference.csv index 8d91b13ca6..fa520cffe7 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Average Cores Reserved Weighted By Wall Hours" -Screwdriver,1.4291 +Screwdriver,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Month-reference.csv index 8d91b13ca6..fa520cffe7 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Average Cores Reserved Weighted By Wall Hours" -Screwdriver,1.4291 +Screwdriver,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv index 8d91b13ca6..fa520cffe7 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Average Cores Reserved Weighted By Wall Hours" -Screwdriver,1.4291 +Screwdriver,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Year-reference.csv index 8d91b13ca6..fa520cffe7 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Average Cores Reserved Weighted By Wall Hours" -Screwdriver,1.4291 +Screwdriver,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Day-reference.csv index ad58e909e9..20ddfdc626 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Day-reference.csv @@ -7,7 +7,7 @@ start,end --------- Day,"[Screwdriver] Average Cores Reserved Weighted By Wall Hours" 2018-04-18,1.3753 -2018-04-19,1.1273 +2018-04-19,1.1291 2018-04-20,1.3285 2018-04-21,1.2500 2018-04-22,1.2500 diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Month-reference.csv index 6816a3999a..306bc4acaf 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[Screwdriver] Average Cores Reserved Weighted By Wall Hours" -2018-04,1.4291 +2018-04,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv index 0255312e24..ffd1bd8e31 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[Screwdriver] Average Cores Reserved Weighted By Wall Hours" -"2018 Q2",1.4291 +"2018 Q2",1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Year-reference.csv index afdee35951..29cd926ec8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_cores_reserved/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[Screwdriver] Average Cores Reserved Weighted By Wall Hours" -2018,1.4291 +2018,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Day-reference.csv index 5ce44ca7b6..fe712c28a3 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Average Memory Reserved Weighted By Wall Hours (Bytes)" -Screwdriver,5387847669.7281 +Screwdriver,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Month-reference.csv index 5ce44ca7b6..fe712c28a3 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Average Memory Reserved Weighted By Wall Hours (Bytes)" -Screwdriver,5387847669.7281 +Screwdriver,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv index 5ce44ca7b6..fe712c28a3 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Average Memory Reserved Weighted By Wall Hours (Bytes)" -Screwdriver,5387847669.7281 +Screwdriver,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Year-reference.csv index 5ce44ca7b6..fe712c28a3 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Average Memory Reserved Weighted By Wall Hours (Bytes)" -Screwdriver,5387847669.7281 +Screwdriver,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Day-reference.csv index f80aa4697a..7e91d991f7 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Day-reference.csv @@ -7,7 +7,7 @@ start,end --------- Day,"[Screwdriver] Average Memory Reserved Weighted By Wall Hours (Bytes)" 2018-04-18,5825397016.2531 -2018-04-19,4270295264.2175 +2018-04-19,4270382534.7701 2018-04-20,4294967296.0000 2018-04-21,4294967296.0000 2018-04-22,4294967296.0000 diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Month-reference.csv index e32c2bf914..c6f088cb4f 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[Screwdriver] Average Memory Reserved Weighted By Wall Hours (Bytes)" -2018-04,5387847669.7281 +2018-04,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv index 6123fb4df1..e8221ae844 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[Screwdriver] Average Memory Reserved Weighted By Wall Hours (Bytes)" -"2018 Q2",5387847669.7281 +"2018 Q2",5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Year-reference.csv index ff38b00aa0..ab173c513f 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_avg_memory_reserved/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[Screwdriver] Average Memory Reserved Weighted By Wall Hours (Bytes)" -2018,5387847669.7281 +2018,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Day-reference.csv index ce160ff4ea..af99fe216f 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Core Hours: Total" -Screwdriver,1755.9078 +Screwdriver,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Month-reference.csv index ce160ff4ea..af99fe216f 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Core Hours: Total" -Screwdriver,1755.9078 +Screwdriver,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Quarter-reference.csv index ce160ff4ea..af99fe216f 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Core Hours: Total" -Screwdriver,1755.9078 +Screwdriver,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Year-reference.csv index ce160ff4ea..af99fe216f 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Core Hours: Total" -Screwdriver,1755.9078 +Screwdriver,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/timeseries-Month-reference.csv index 42d604c893..c6f5513b59 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[Screwdriver] Core Hours: Total" -2018-04,1755.9078 +2018-04,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/timeseries-Quarter-reference.csv index 71edf21d7c..ccff2fb556 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[Screwdriver] Core Hours: Total" -"2018 Q2",1755.9078 +"2018 Q2",1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/timeseries-Year-reference.csv index 26edb4ab5c..44d5f90fbe 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_core_time/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[Screwdriver] Core Hours: Total" -2018,1755.9078 +2018,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Day-reference.csv index 6b8c822b8e..345a0bbc52 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Wall Hours: Total" -Screwdriver,1228.6692 +Screwdriver,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Month-reference.csv index 6b8c822b8e..345a0bbc52 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Wall Hours: Total" -Screwdriver,1228.6692 +Screwdriver,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Quarter-reference.csv index 6b8c822b8e..345a0bbc52 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Wall Hours: Total" -Screwdriver,1228.6692 +Screwdriver,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Year-reference.csv index 6b8c822b8e..345a0bbc52 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Summary,"Wall Hours: Total" -Screwdriver,1228.6692 +Screwdriver,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/timeseries-Month-reference.csv index c1d186f514..069878259c 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[Screwdriver] Wall Hours: Total" -2018-04,1228.6692 +2018-04,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/timeseries-Quarter-reference.csv index 71eb8b35dd..56ae189ab7 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[Screwdriver] Wall Hours: Total" -"2018 Q2",1228.6692 +"2018 Q2",1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/timeseries-Year-reference.csv index 8a059c710d..53fec9cdd6 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/none/cloud_wall_time/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[Screwdriver] Wall Hours: Total" -2018,1228.6692 +2018,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Day-reference.csv index df69cb2eed..e7248c9ab8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Average Cores Reserved Weighted By Wall Hours" -zealous,1.4291 +zealous,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Month-reference.csv index df69cb2eed..e7248c9ab8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Average Cores Reserved Weighted By Wall Hours" -zealous,1.4291 +zealous,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv index df69cb2eed..e7248c9ab8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Average Cores Reserved Weighted By Wall Hours" -zealous,1.4291 +zealous,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Year-reference.csv index df69cb2eed..e7248c9ab8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Average Cores Reserved Weighted By Wall Hours" -zealous,1.4291 +zealous,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Day-reference.csv index 5d23617c0e..c7e417b028 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Day-reference.csv @@ -7,7 +7,7 @@ start,end --------- Day,"[zealous] Average Cores Reserved Weighted By Wall Hours" 2018-04-18,1.3753 -2018-04-19,1.1273 +2018-04-19,1.1291 2018-04-20,1.3285 2018-04-21,1.2500 2018-04-22,1.2500 diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Month-reference.csv index 9300508965..2fe8635c86 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[zealous] Average Cores Reserved Weighted By Wall Hours" -2018-04,1.4291 +2018-04,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv index 9664336342..9b021cc8ad 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[zealous] Average Cores Reserved Weighted By Wall Hours" -"2018 Q2",1.4291 +"2018 Q2",1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Year-reference.csv index 63014232bf..456e11463c 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_cores_reserved/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[zealous] Average Cores Reserved Weighted By Wall Hours" -2018,1.4291 +2018,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Day-reference.csv index 1a803657d5..5f89d26d21 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Average Memory Reserved Weighted By Wall Hours (Bytes)" -zealous,5387847669.7281 +zealous,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Month-reference.csv index 1a803657d5..5f89d26d21 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Average Memory Reserved Weighted By Wall Hours (Bytes)" -zealous,5387847669.7281 +zealous,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv index 1a803657d5..5f89d26d21 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Average Memory Reserved Weighted By Wall Hours (Bytes)" -zealous,5387847669.7281 +zealous,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Year-reference.csv index 1a803657d5..5f89d26d21 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Average Memory Reserved Weighted By Wall Hours (Bytes)" -zealous,5387847669.7281 +zealous,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Day-reference.csv index b938fe8752..070afb8f21 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Day-reference.csv @@ -7,7 +7,7 @@ start,end --------- Day,"[zealous] Average Memory Reserved Weighted By Wall Hours (Bytes)" 2018-04-18,5825397016.2531 -2018-04-19,4270295264.2175 +2018-04-19,4270382534.7701 2018-04-20,4294967296.0000 2018-04-21,4294967296.0000 2018-04-22,4294967296.0000 diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Month-reference.csv index 00510552cb..d3acabfb1e 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[zealous] Average Memory Reserved Weighted By Wall Hours (Bytes)" -2018-04,5387847669.7281 +2018-04,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv index 3d91448c89..da9ca91fda 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[zealous] Average Memory Reserved Weighted By Wall Hours (Bytes)" -"2018 Q2",5387847669.7281 +"2018 Q2",5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Year-reference.csv index 342925084c..baf866052e 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_avg_memory_reserved/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[zealous] Average Memory Reserved Weighted By Wall Hours (Bytes)" -2018,5387847669.7281 +2018,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Day-reference.csv index e602941397..1d8cf668e0 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Core Hours: Total" -zealous,1755.9078 +zealous,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Month-reference.csv index e602941397..1d8cf668e0 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Core Hours: Total" -zealous,1755.9078 +zealous,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Quarter-reference.csv index e602941397..1d8cf668e0 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Core Hours: Total" -zealous,1755.9078 +zealous,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Year-reference.csv index e602941397..1d8cf668e0 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Core Hours: Total" -zealous,1755.9078 +zealous,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/timeseries-Month-reference.csv index c41ca8d0c0..b7f1a4a072 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[zealous] Core Hours: Total" -2018-04,1755.9078 +2018-04,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/timeseries-Quarter-reference.csv index 4cdaf39573..01de1fa80a 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[zealous] Core Hours: Total" -"2018 Q2",1755.9078 +"2018 Q2",1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/timeseries-Year-reference.csv index af089d344d..ebcb6fa078 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_core_time/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[zealous] Core Hours: Total" -2018,1755.9078 +2018,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Day-reference.csv index 3277cede32..de37bd2597 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Wall Hours: Total" -zealous,1228.6692 +zealous,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Month-reference.csv index 3277cede32..de37bd2597 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Wall Hours: Total" -zealous,1228.6692 +zealous,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Quarter-reference.csv index 3277cede32..de37bd2597 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Wall Hours: Total" -zealous,1228.6692 +zealous,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Year-reference.csv index 3277cede32..de37bd2597 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Project,"Wall Hours: Total" -zealous,1228.6692 +zealous,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/timeseries-Month-reference.csv index 55f974152b..6627d62242 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[zealous] Wall Hours: Total" -2018-04,1228.6692 +2018-04,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/timeseries-Quarter-reference.csv index d96fb0972a..59713c6d76 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[zealous] Wall Hours: Total" -"2018 Q2",1228.6692 +"2018 Q2",1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/timeseries-Year-reference.csv index 4a7109ba16..b4180d9f31 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/project/cloud_wall_time/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[zealous] Wall Hours: Total" -2018,1228.6692 +2018,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Day-reference.csv index db4d9db2e1..9cb15e18b8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Average Cores Reserved Weighted By Wall Hours" -openstack,1.4291 +openstack,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Month-reference.csv index db4d9db2e1..9cb15e18b8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Average Cores Reserved Weighted By Wall Hours" -openstack,1.4291 +openstack,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv index db4d9db2e1..9cb15e18b8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Average Cores Reserved Weighted By Wall Hours" -openstack,1.4291 +openstack,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Year-reference.csv index db4d9db2e1..9cb15e18b8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Average Cores Reserved Weighted By Wall Hours" -openstack,1.4291 +openstack,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Day-reference.csv index 5087ad9a70..8692ba65ef 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Day-reference.csv @@ -7,7 +7,7 @@ start,end --------- Day,"[openstack] Average Cores Reserved Weighted By Wall Hours" 2018-04-18,1.3753 -2018-04-19,1.1273 +2018-04-19,1.1291 2018-04-20,1.3285 2018-04-21,1.2500 2018-04-22,1.2500 diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Month-reference.csv index 9291a07418..bdd47c403c 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[openstack] Average Cores Reserved Weighted By Wall Hours" -2018-04,1.4291 +2018-04,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv index 0f4b094718..70202fae90 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[openstack] Average Cores Reserved Weighted By Wall Hours" -"2018 Q2",1.4291 +"2018 Q2",1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Year-reference.csv index dc3ca0cb91..ef08410b02 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_cores_reserved/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[openstack] Average Cores Reserved Weighted By Wall Hours" -2018,1.4291 +2018,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Day-reference.csv index 8d49919884..30a0443a98 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Average Memory Reserved Weighted By Wall Hours (Bytes)" -openstack,5387847669.7281 +openstack,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Month-reference.csv index 8d49919884..30a0443a98 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Average Memory Reserved Weighted By Wall Hours (Bytes)" -openstack,5387847669.7281 +openstack,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv index 8d49919884..30a0443a98 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Average Memory Reserved Weighted By Wall Hours (Bytes)" -openstack,5387847669.7281 +openstack,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Year-reference.csv index 8d49919884..30a0443a98 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Average Memory Reserved Weighted By Wall Hours (Bytes)" -openstack,5387847669.7281 +openstack,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Day-reference.csv index 9c2f2fb963..62cdbed463 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Day-reference.csv @@ -7,7 +7,7 @@ start,end --------- Day,"[openstack] Average Memory Reserved Weighted By Wall Hours (Bytes)" 2018-04-18,5825397016.2531 -2018-04-19,4270295264.2175 +2018-04-19,4270382534.7701 2018-04-20,4294967296.0000 2018-04-21,4294967296.0000 2018-04-22,4294967296.0000 diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Month-reference.csv index 702e53a2c4..2e81f35e0c 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[openstack] Average Memory Reserved Weighted By Wall Hours (Bytes)" -2018-04,5387847669.7281 +2018-04,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv index 1b00c29cf1..ad40c093fe 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[openstack] Average Memory Reserved Weighted By Wall Hours (Bytes)" -"2018 Q2",5387847669.7281 +"2018 Q2",5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Year-reference.csv index 35214f165a..71fc774ef1 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_avg_memory_reserved/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[openstack] Average Memory Reserved Weighted By Wall Hours (Bytes)" -2018,5387847669.7281 +2018,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Day-reference.csv index 49de1460a9..dd580505bd 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Core Hours: Total" -openstack,1755.9078 +openstack,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Month-reference.csv index 49de1460a9..dd580505bd 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Core Hours: Total" -openstack,1755.9078 +openstack,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Quarter-reference.csv index 49de1460a9..dd580505bd 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Core Hours: Total" -openstack,1755.9078 +openstack,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Year-reference.csv index 49de1460a9..dd580505bd 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Core Hours: Total" -openstack,1755.9078 +openstack,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/timeseries-Month-reference.csv index 5ced406816..f2a7d4fc5e 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[openstack] Core Hours: Total" -2018-04,1755.9078 +2018-04,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/timeseries-Quarter-reference.csv index 5f36012680..fe45343a33 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[openstack] Core Hours: Total" -"2018 Q2",1755.9078 +"2018 Q2",1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/timeseries-Year-reference.csv index d8dc1cd0df..a55a6ad448 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_core_time/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[openstack] Core Hours: Total" -2018,1755.9078 +2018,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Day-reference.csv index 893d47353e..5051bde60b 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Wall Hours: Total" -openstack,1228.6692 +openstack,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Month-reference.csv index 893d47353e..5051bde60b 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Wall Hours: Total" -openstack,1228.6692 +openstack,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Quarter-reference.csv index 893d47353e..5051bde60b 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Wall Hours: Total" -openstack,1228.6692 +openstack,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Year-reference.csv index 893d47353e..5051bde60b 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Resource,"Wall Hours: Total" -openstack,1228.6692 +openstack,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/timeseries-Month-reference.csv index 2613e9a7c4..235b622cf8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[openstack] Wall Hours: Total" -2018-04,1228.6692 +2018-04,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/timeseries-Quarter-reference.csv index 088e0e6af2..ca1fb35ee6 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[openstack] Wall Hours: Total" -"2018 Q2",1228.6692 +"2018 Q2",1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/timeseries-Year-reference.csv index a58b170e0c..ab94f65806 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/resource/cloud_wall_time/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[openstack] Wall Hours: Total" -2018,1228.6692 +2018,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Day-reference.csv index fa35e68910..bd165bfa4c 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Average Cores Reserved Weighted By Wall Hours" -"OpenStack API",1.4291 +"OpenStack API",1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Month-reference.csv index fa35e68910..bd165bfa4c 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Average Cores Reserved Weighted By Wall Hours" -"OpenStack API",1.4291 +"OpenStack API",1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv index fa35e68910..bd165bfa4c 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Average Cores Reserved Weighted By Wall Hours" -"OpenStack API",1.4291 +"OpenStack API",1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Year-reference.csv index fa35e68910..bd165bfa4c 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Average Cores Reserved Weighted By Wall Hours" -"OpenStack API",1.4291 +"OpenStack API",1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Day-reference.csv index a0ee5cb230..f67cf49ffc 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Day-reference.csv @@ -7,7 +7,7 @@ start,end --------- Day,"[OpenStack API] Average Cores Reserved Weighted By Wall Hours" 2018-04-18,1.3753 -2018-04-19,1.1273 +2018-04-19,1.1291 2018-04-20,1.3285 2018-04-21,1.2500 2018-04-22,1.2500 diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Month-reference.csv index 5a36ec9abb..d762c0b2ee 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[OpenStack API] Average Cores Reserved Weighted By Wall Hours" -2018-04,1.4291 +2018-04,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv index 6cb99e5551..f984c8a152 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[OpenStack API] Average Cores Reserved Weighted By Wall Hours" -"2018 Q2",1.4291 +"2018 Q2",1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Year-reference.csv index 45388dd3c2..82330b0324 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_cores_reserved/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[OpenStack API] Average Cores Reserved Weighted By Wall Hours" -2018,1.4291 +2018,1.4297 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Day-reference.csv index fe9f60c2c6..d4cf8b31d4 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Average Memory Reserved Weighted By Wall Hours (Bytes)" -"OpenStack API",5387847669.7281 +"OpenStack API",5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Month-reference.csv index fe9f60c2c6..d4cf8b31d4 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Average Memory Reserved Weighted By Wall Hours (Bytes)" -"OpenStack API",5387847669.7281 +"OpenStack API",5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv index fe9f60c2c6..d4cf8b31d4 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Average Memory Reserved Weighted By Wall Hours (Bytes)" -"OpenStack API",5387847669.7281 +"OpenStack API",5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Year-reference.csv index fe9f60c2c6..d4cf8b31d4 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Average Memory Reserved Weighted By Wall Hours (Bytes)" -"OpenStack API",5387847669.7281 +"OpenStack API",5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Day-reference.csv index 9ffba3d3f0..88aa69a56c 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Day-reference.csv @@ -7,7 +7,7 @@ start,end --------- Day,"[OpenStack API] Average Memory Reserved Weighted By Wall Hours (Bytes)" 2018-04-18,5825397016.2531 -2018-04-19,4270295264.2175 +2018-04-19,4270382534.7701 2018-04-20,4294967296.0000 2018-04-21,4294967296.0000 2018-04-22,4294967296.0000 diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Month-reference.csv index c4b9331c39..efc1400c33 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[OpenStack API] Average Memory Reserved Weighted By Wall Hours (Bytes)" -2018-04,5387847669.7281 +2018-04,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv index 8df5df7f83..5a44430fc9 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[OpenStack API] Average Memory Reserved Weighted By Wall Hours (Bytes)" -"2018 Q2",5387847669.7281 +"2018 Q2",5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Year-reference.csv index b56090664b..1aec155550 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_avg_memory_reserved/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[OpenStack API] Average Memory Reserved Weighted By Wall Hours (Bytes)" -2018,5387847669.7281 +2018,5389444295.8833 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Day-reference.csv index 98e61e099c..6fec54ac58 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Core Hours: Total" -"OpenStack API",1755.9078 +"OpenStack API",1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Month-reference.csv index 98e61e099c..6fec54ac58 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Core Hours: Total" -"OpenStack API",1755.9078 +"OpenStack API",1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Quarter-reference.csv index 98e61e099c..6fec54ac58 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Core Hours: Total" -"OpenStack API",1755.9078 +"OpenStack API",1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Year-reference.csv index 98e61e099c..6fec54ac58 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Core Hours: Total" -"OpenStack API",1755.9078 +"OpenStack API",1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/timeseries-Month-reference.csv index 831d620b1c..e25c2f76b4 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[OpenStack API] Core Hours: Total" -2018-04,1755.9078 +2018-04,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/timeseries-Quarter-reference.csv index f6f4b7cbde..71f6e306b6 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[OpenStack API] Core Hours: Total" -"2018 Q2",1755.9078 +"2018 Q2",1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/timeseries-Year-reference.csv index c4bca92eb8..e0c3a2db4b 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_core_time/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[OpenStack API] Core Hours: Total" -2018,1755.9078 +2018,1754.1644 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Day-reference.csv index 352e976d04..9fbf58676f 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Day-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Wall Hours: Total" -"OpenStack API",1228.6692 +"OpenStack API",1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Month-reference.csv index 352e976d04..9fbf58676f 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Wall Hours: Total" -"OpenStack API",1228.6692 +"OpenStack API",1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Quarter-reference.csv index 352e976d04..9fbf58676f 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Wall Hours: Total" -"OpenStack API",1228.6692 +"OpenStack API",1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Year-reference.csv index 352e976d04..9fbf58676f 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/aggregate-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- "Submission Venue","Wall Hours: Total" -"OpenStack API",1228.6692 +"OpenStack API",1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/timeseries-Month-reference.csv index db037f7b2d..9c2a8b0dfa 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[OpenStack API] Wall Hours: Total" -2018-04,1228.6692 +2018-04,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/timeseries-Quarter-reference.csv index a9c8b6ab13..4635af6fef 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[OpenStack API] Wall Hours: Total" -"2018 Q2",1228.6692 +"2018 Q2",1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/timeseries-Year-reference.csv index c1cc0bd974..2d0affe326 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/submission_venue/cloud_wall_time/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[OpenStack API] Wall Hours: Total" -2018,1228.6692 +2018,1226.9258 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Day-reference.csv index 1f23036387..f21563ddc8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Day-reference.csv @@ -6,7 +6,7 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Cores","Average Memory Reserved Weighted By Wall Hours (Bytes)" -1,4271551495.1406 +1,4271565613.2953 "2 - 3",4299793957.9116 "4 - 7",17179869184.0000 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Month-reference.csv index 1f23036387..f21563ddc8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Month-reference.csv @@ -6,7 +6,7 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Cores","Average Memory Reserved Weighted By Wall Hours (Bytes)" -1,4271551495.1406 +1,4271565613.2953 "2 - 3",4299793957.9116 "4 - 7",17179869184.0000 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv index 1f23036387..f21563ddc8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Quarter-reference.csv @@ -6,7 +6,7 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Cores","Average Memory Reserved Weighted By Wall Hours (Bytes)" -1,4271551495.1406 +1,4271565613.2953 "2 - 3",4299793957.9116 "4 - 7",17179869184.0000 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Year-reference.csv index 1f23036387..f21563ddc8 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/aggregate-Year-reference.csv @@ -6,7 +6,7 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Cores","Average Memory Reserved Weighted By Wall Hours (Bytes)" -1,4271551495.1406 +1,4271565613.2953 "2 - 3",4299793957.9116 "4 - 7",17179869184.0000 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Day-reference.csv index d5c956ca7c..649e82dc13 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Day-reference.csv @@ -7,7 +7,7 @@ start,end --------- Day,"[1] Average Memory Reserved Weighted By Wall Hours (Bytes)","[2 - 3] Average Memory Reserved Weighted By Wall Hours (Bytes)","[4 - 7] Average Memory Reserved Weighted By Wall Hours (Bytes)" 2018-04-18,4293426987.3516,5367761422.0088,17179869184.0000 -2018-04-19,4266696434.1096,4294967296.0000,0 +2018-04-19,4266737750.5513,4294967296.0000,0 2018-04-20,4294967296.0000,4294967296.0000,0 2018-04-21,4294967296.0000,4294967296.0000,0 2018-04-22,4294967296.0000,4294967296.0000,0 diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Month-reference.csv index 5b28c121eb..cdcd4bccce 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[1] Average Memory Reserved Weighted By Wall Hours (Bytes)","[2 - 3] Average Memory Reserved Weighted By Wall Hours (Bytes)","[4 - 7] Average Memory Reserved Weighted By Wall Hours (Bytes)" -2018-04,4271551495.1406,4299793957.9116,17179869184.0000 +2018-04,4271565613.2953,4299793957.9116,17179869184.0000 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv index 0ce69349bb..b56428f4be 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[1] Average Memory Reserved Weighted By Wall Hours (Bytes)","[2 - 3] Average Memory Reserved Weighted By Wall Hours (Bytes)","[4 - 7] Average Memory Reserved Weighted By Wall Hours (Bytes)" -"2018 Q2",4271551495.1406,4299793957.9116,17179869184.0000 +"2018 Q2",4271565613.2953,4299793957.9116,17179869184.0000 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Year-reference.csv index 5e0e271e1b..a26ba9d14e 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_avg_memory_reserved/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[1] Average Memory Reserved Weighted By Wall Hours (Bytes)","[2 - 3] Average Memory Reserved Weighted By Wall Hours (Bytes)","[4 - 7] Average Memory Reserved Weighted By Wall Hours (Bytes)" -2018,4271551495.1406,4299793957.9116,17179869184.0000 +2018,4271565613.2953,4299793957.9116,17179869184.0000 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Day-reference.csv index 7522010ae9..7260c46ab6 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Day-reference.csv @@ -6,7 +6,7 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Cores","Core Hours: Total" -1,913.0200 +1,911.2767 "2 - 3",419.7089 "4 - 7",423.1789 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Month-reference.csv index 7522010ae9..7260c46ab6 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Month-reference.csv @@ -6,7 +6,7 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Cores","Core Hours: Total" -1,913.0200 +1,911.2767 "2 - 3",419.7089 "4 - 7",423.1789 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Quarter-reference.csv index 7522010ae9..7260c46ab6 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Quarter-reference.csv @@ -6,7 +6,7 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Cores","Core Hours: Total" -1,913.0200 +1,911.2767 "2 - 3",419.7089 "4 - 7",423.1789 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Year-reference.csv index 7522010ae9..7260c46ab6 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/aggregate-Year-reference.csv @@ -6,7 +6,7 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Cores","Core Hours: Total" -1,913.0200 +1,911.2767 "2 - 3",419.7089 "4 - 7",423.1789 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/timeseries-Month-reference.csv index b945a63385..9aa8c4aa43 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[1] Core Hours: Total","[2 - 3] Core Hours: Total","[4 - 7] Core Hours: Total" -2018-04,913.0200,419.7089,423.1789 +2018-04,911.2767,419.7089,423.1789 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/timeseries-Quarter-reference.csv index 8fb383779c..fb48f54bd0 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[1] Core Hours: Total","[2 - 3] Core Hours: Total","[4 - 7] Core Hours: Total" -"2018 Q2",913.0200,419.7089,423.1789 +"2018 Q2",911.2767,419.7089,423.1789 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/timeseries-Year-reference.csv index 8d04bf6e53..60a140dc2a 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_core_time/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[1] Core Hours: Total","[2 - 3] Core Hours: Total","[4 - 7] Core Hours: Total" -2018,913.0200,419.7089,423.1789 +2018,911.2767,419.7089,423.1789 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Day-reference.csv index a94d59e304..b832bbbfee 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Day-reference.csv @@ -6,7 +6,7 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Cores","Wall Hours: Total" -1,913.0200 +1,911.2767 "2 - 3",209.8544 "4 - 7",105.7947 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Month-reference.csv index a94d59e304..b832bbbfee 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Month-reference.csv @@ -6,7 +6,7 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Cores","Wall Hours: Total" -1,913.0200 +1,911.2767 "2 - 3",209.8544 "4 - 7",105.7947 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Quarter-reference.csv index a94d59e304..b832bbbfee 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Quarter-reference.csv @@ -6,7 +6,7 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Cores","Wall Hours: Total" -1,913.0200 +1,911.2767 "2 - 3",209.8544 "4 - 7",105.7947 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Year-reference.csv index a94d59e304..b832bbbfee 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/aggregate-Year-reference.csv @@ -6,7 +6,7 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Cores","Wall Hours: Total" -1,913.0200 +1,911.2767 "2 - 3",209.8544 "4 - 7",105.7947 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/timeseries-Month-reference.csv index 3a50a21614..1b0fd20c1d 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[1] Wall Hours: Total","[2 - 3] Wall Hours: Total","[4 - 7] Wall Hours: Total" -2018-04,913.0200,209.8544,105.7947 +2018-04,911.2767,209.8544,105.7947 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/timeseries-Quarter-reference.csv index 3e59f4ad01..a141e08928 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[1] Wall Hours: Total","[2 - 3] Wall Hours: Total","[4 - 7] Wall Hours: Total" -"2018 Q2",913.0200,209.8544,105.7947 +"2018 Q2",911.2767,209.8544,105.7947 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/timeseries-Year-reference.csv index a091ba0a43..c90b7b4c28 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size/cloud_wall_time/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[1] Wall Hours: Total","[2 - 3] Wall Hours: Total","[4 - 7] Wall Hours: Total" -2018,913.0200,209.8544,105.7947 +2018,911.2767,209.8544,105.7947 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Day-reference.csv index a6b3d012e1..5013755c57 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Day-reference.csv @@ -8,6 +8,6 @@ start,end "VM Size: Memory","Average Cores Reserved Weighted By Wall Hours" "8 - 16 GB",4.0000 "4 - 8 GB",2.0000 -"2 - 4 GB",1.1878 +"2 - 4 GB",1.1881 "< 2 GB",1.0000 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Month-reference.csv index a6b3d012e1..5013755c57 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Month-reference.csv @@ -8,6 +8,6 @@ start,end "VM Size: Memory","Average Cores Reserved Weighted By Wall Hours" "8 - 16 GB",4.0000 "4 - 8 GB",2.0000 -"2 - 4 GB",1.1878 +"2 - 4 GB",1.1881 "< 2 GB",1.0000 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv index a6b3d012e1..5013755c57 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Quarter-reference.csv @@ -8,6 +8,6 @@ start,end "VM Size: Memory","Average Cores Reserved Weighted By Wall Hours" "8 - 16 GB",4.0000 "4 - 8 GB",2.0000 -"2 - 4 GB",1.1878 +"2 - 4 GB",1.1881 "< 2 GB",1.0000 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Year-reference.csv index a6b3d012e1..5013755c57 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/aggregate-Year-reference.csv @@ -8,6 +8,6 @@ start,end "VM Size: Memory","Average Cores Reserved Weighted By Wall Hours" "8 - 16 GB",4.0000 "4 - 8 GB",2.0000 -"2 - 4 GB",1.1878 +"2 - 4 GB",1.1881 "< 2 GB",1.0000 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Day-reference.csv index 47aceb86ee..322ce4b67a 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Day-reference.csv @@ -7,7 +7,7 @@ start,end --------- Day,"[8 - 16 GB] Average Cores Reserved Weighted By Wall Hours","[4 - 8 GB] Average Cores Reserved Weighted By Wall Hours","[2 - 4 GB] Average Cores Reserved Weighted By Wall Hours","[< 2 GB] Average Cores Reserved Weighted By Wall Hours" 2018-04-18,4.0000,2.0000,1.0213,1.0000 -2018-04-19,0,0,1.1283,1.0000 +2018-04-19,0,0,1.1301,1.0000 2018-04-20,0,0,1.3285,0 2018-04-21,0,0,1.2500,0 2018-04-22,0,0,1.2500,0 diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Month-reference.csv index 52baa95882..0a147430f7 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[8 - 16 GB] Average Cores Reserved Weighted By Wall Hours","[4 - 8 GB] Average Cores Reserved Weighted By Wall Hours","[2 - 4 GB] Average Cores Reserved Weighted By Wall Hours","[< 2 GB] Average Cores Reserved Weighted By Wall Hours" -2018-04,4.0000,2.0000,1.1878,1.0000 +2018-04,4.0000,2.0000,1.1881,1.0000 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv index 1cbfa8d92a..68e98ced7c 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[8 - 16 GB] Average Cores Reserved Weighted By Wall Hours","[4 - 8 GB] Average Cores Reserved Weighted By Wall Hours","[2 - 4 GB] Average Cores Reserved Weighted By Wall Hours","[< 2 GB] Average Cores Reserved Weighted By Wall Hours" -"2018 Q2",4.0000,2.0000,1.1878,1.0000 +"2018 Q2",4.0000,2.0000,1.1881,1.0000 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Year-reference.csv index f7ad9fdd6b..19565a8174 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_avg_cores_reserved/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[8 - 16 GB] Average Cores Reserved Weighted By Wall Hours","[4 - 8 GB] Average Cores Reserved Weighted By Wall Hours","[2 - 4 GB] Average Cores Reserved Weighted By Wall Hours","[< 2 GB] Average Cores Reserved Weighted By Wall Hours" -2018,4.0000,2.0000,1.1878,1.0000 +2018,4.0000,2.0000,1.1881,1.0000 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Day-reference.csv index 2ee63c451e..9472879ce7 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Day-reference.csv @@ -6,8 +6,8 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Memory","Core Hours: Total" -"2 - 4 GB",1325.6203 +"2 - 4 GB",1323.8936 "8 - 16 GB",423.1789 -"< 2 GB",6.6369 +"< 2 GB",6.6203 "4 - 8 GB",0.4717 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Month-reference.csv index 2ee63c451e..9472879ce7 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Month-reference.csv @@ -6,8 +6,8 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Memory","Core Hours: Total" -"2 - 4 GB",1325.6203 +"2 - 4 GB",1323.8936 "8 - 16 GB",423.1789 -"< 2 GB",6.6369 +"< 2 GB",6.6203 "4 - 8 GB",0.4717 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Quarter-reference.csv index 2ee63c451e..9472879ce7 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Quarter-reference.csv @@ -6,8 +6,8 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Memory","Core Hours: Total" -"2 - 4 GB",1325.6203 +"2 - 4 GB",1323.8936 "8 - 16 GB",423.1789 -"< 2 GB",6.6369 +"< 2 GB",6.6203 "4 - 8 GB",0.4717 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Year-reference.csv index 2ee63c451e..9472879ce7 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/aggregate-Year-reference.csv @@ -6,8 +6,8 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Memory","Core Hours: Total" -"2 - 4 GB",1325.6203 +"2 - 4 GB",1323.8936 "8 - 16 GB",423.1789 -"< 2 GB",6.6369 +"< 2 GB",6.6203 "4 - 8 GB",0.4717 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/timeseries-Month-reference.csv index a867820813..af887def75 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[2 - 4 GB] Core Hours: Total","[8 - 16 GB] Core Hours: Total","[< 2 GB] Core Hours: Total","[4 - 8 GB] Core Hours: Total" -2018-04,1325.6203,423.1789,6.6369,0.4717 +2018-04,1323.8936,423.1789,6.6203,0.4717 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/timeseries-Quarter-reference.csv index 91422513d0..eeca219d02 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[2 - 4 GB] Core Hours: Total","[8 - 16 GB] Core Hours: Total","[< 2 GB] Core Hours: Total","[4 - 8 GB] Core Hours: Total" -"2018 Q2",1325.6203,423.1789,6.6369,0.4717 +"2018 Q2",1323.8936,423.1789,6.6203,0.4717 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/timeseries-Year-reference.csv index 588b771594..89749bcc8b 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_core_time/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[2 - 4 GB] Core Hours: Total","[8 - 16 GB] Core Hours: Total","[< 2 GB] Core Hours: Total","[4 - 8 GB] Core Hours: Total" -2018,1325.6203,423.1789,6.6369,0.4717 +2018,1323.8936,423.1789,6.6203,0.4717 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Day-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Day-reference.csv index 3832d938e8..d0a99873a2 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Day-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Day-reference.csv @@ -6,8 +6,8 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Memory","Wall Hours: Total" -"2 - 4 GB",1116.0017 +"2 - 4 GB",1114.2750 "8 - 16 GB",105.7947 -"< 2 GB",6.6369 +"< 2 GB",6.6203 "4 - 8 GB",0.2358 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Month-reference.csv index 3832d938e8..d0a99873a2 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Month-reference.csv @@ -6,8 +6,8 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Memory","Wall Hours: Total" -"2 - 4 GB",1116.0017 +"2 - 4 GB",1114.2750 "8 - 16 GB",105.7947 -"< 2 GB",6.6369 +"< 2 GB",6.6203 "4 - 8 GB",0.2358 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Quarter-reference.csv index 3832d938e8..d0a99873a2 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Quarter-reference.csv @@ -6,8 +6,8 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Memory","Wall Hours: Total" -"2 - 4 GB",1116.0017 +"2 - 4 GB",1114.2750 "8 - 16 GB",105.7947 -"< 2 GB",6.6369 +"< 2 GB",6.6203 "4 - 8 GB",0.2358 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Year-reference.csv index 3832d938e8..d0a99873a2 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/aggregate-Year-reference.csv @@ -6,8 +6,8 @@ start,end 2018-04-18,2018-04-30 --------- "VM Size: Memory","Wall Hours: Total" -"2 - 4 GB",1116.0017 +"2 - 4 GB",1114.2750 "8 - 16 GB",105.7947 -"< 2 GB",6.6369 +"< 2 GB",6.6203 "4 - 8 GB",0.2358 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/timeseries-Month-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/timeseries-Month-reference.csv index e3dcd5c3e3..174a1aa891 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/timeseries-Month-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/timeseries-Month-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Month,"[2 - 4 GB] Wall Hours: Total","[8 - 16 GB] Wall Hours: Total","[< 2 GB] Wall Hours: Total","[4 - 8 GB] Wall Hours: Total" -2018-04,1116.0017,105.7947,6.6369,0.2358 +2018-04,1114.2750,105.7947,6.6203,0.2358 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/timeseries-Quarter-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/timeseries-Quarter-reference.csv index 7302de982a..5fe8d615d0 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/timeseries-Quarter-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/timeseries-Quarter-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Quarter,"[2 - 4 GB] Wall Hours: Total","[8 - 16 GB] Wall Hours: Total","[< 2 GB] Wall Hours: Total","[4 - 8 GB] Wall Hours: Total" -"2018 Q2",1116.0017,105.7947,6.6369,0.2358 +"2018 Q2",1114.2750,105.7947,6.6203,0.2358 --------- diff --git a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/timeseries-Year-reference.csv b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/timeseries-Year-reference.csv index de9d57bdce..41d6be7192 100644 --- a/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/timeseries-Year-reference.csv +++ b/tests/artifacts/xdmod-test-artifacts/xdmod/regression/current/expected/reference/Cloud/vm_size_memory/cloud_wall_time/timeseries-Year-reference.csv @@ -6,5 +6,5 @@ start,end 2018-04-18,2018-04-30 --------- Year,"[2 - 4 GB] Wall Hours: Total","[8 - 16 GB] Wall Hours: Total","[< 2 GB] Wall Hours: Total","[4 - 8 GB] Wall Hours: Total" -2018,1116.0017,105.7947,6.6369,0.2358 +2018,1114.2750,105.7947,6.6203,0.2358 --------- From 79781465578501e852013e3722928c4ddcc6316f Mon Sep 17 00:00:00 2001 From: Steven Gallo Date: Wed, 6 Feb 2019 14:34:02 -0500 Subject: [PATCH 004/217] Throw Exception if lockfile could not be obtained --- classes/ETL/LockFile.php | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/classes/ETL/LockFile.php b/classes/ETL/LockFile.php index 6d13668f22..fee6b270b4 100644 --- a/classes/ETL/LockFile.php +++ b/classes/ETL/LockFile.php @@ -128,7 +128,10 @@ protected function generateLockfileName($pid = null) * * @param array $actionList A list of action names to be executed by this ETL process. * - * @return boolean TRUE if the lock was generated, FALSE otherwise. + * @return boolean TRUE if the lock was generated. + * + * @throws Exception If a process is already running with overlapping actions. + * @throws Exception If the lock could not be obtained. * ------------------------------------------------------------------------------------------ */ @@ -140,10 +143,9 @@ public function lock(array $actionList) if ( false === ($dh = opendir($this->lockDir)) ) { $error = error_get_last(); - $this->logger->warning( + $this->logAndThrowException( sprintf("Error opening lock directory '%s': %s", $this->lockDir, $error['message']) ); - return false; } // Cleanup any lockfiles not associated with a running process @@ -188,10 +190,9 @@ public function lock(array $actionList) if ( false === ($fp = @fopen($lockFile, 'w')) ) { $error = error_get_last(); - $this->logger->warning( + $this->logAndThrowException( sprintf("Error creating lock file '%s': %s", $lockFile, $error['message']) ); - return false; } // Obtain an advisory lock. This advisory will be memoved if the process dies or From c904af02acdb2e9c745a6bb7ce369938a94b775d Mon Sep 17 00:00:00 2001 From: Ben Plessinger Date: Thu, 7 Feb 2019 10:01:08 -0500 Subject: [PATCH 005/217] Fix some indexes in the cloud (#794) --- configuration/etl/etl_tables.d/cloud_common/event.json | 1 - 1 file changed, 1 deletion(-) diff --git a/configuration/etl/etl_tables.d/cloud_common/event.json b/configuration/etl/etl_tables.d/cloud_common/event.json index cea3d35738..4f5584fcf8 100644 --- a/configuration/etl/etl_tables.d/cloud_common/event.json +++ b/configuration/etl/etl_tables.d/cloud_common/event.json @@ -63,7 +63,6 @@ "name": "PRIMARY", "columns": [ "resource_id", - "event_id", "instance_id", "event_time_utc", "event_type_id", From c464faa12ae08e374506a2cbc892dd65a476751a Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 27 Feb 2019 09:02:36 -0500 Subject: [PATCH 006/217] Add batch export requests table --- configuration/etl/etl.d/xdb.json | 5 +- .../xdb/batch-export-requests.json | 120 ++++++++++++++++++ 2 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 configuration/etl/etl_tables.d/xdb/batch-export-requests.json diff --git a/configuration/etl/etl.d/xdb.json b/configuration/etl/etl.d/xdb.json index 37a41d5d9e..69a0732191 100644 --- a/configuration/etl/etl.d/xdb.json +++ b/configuration/etl/etl.d/xdb.json @@ -33,7 +33,7 @@ "class": "ManageTables", "namespace": "ETL\\Maintenance", "options_class": "MaintenanceOptions", - "# order-matters": "Because foreign key constraints exist between user-roles and users the order matters", + "# order-matters": "Because of foreign key constraints", "definition_file_list": [ "xdb/account-requests.json", "xdb/api-keys.json", @@ -54,7 +54,8 @@ "xdb/user-role-parameters.json", "xdb/user-roles.json", "xdb/user-types.json", - "xdb/version-check.json" + "xdb/version-check.json", + "xdb/batch-export-requests.json" ] }, { diff --git a/configuration/etl/etl_tables.d/xdb/batch-export-requests.json b/configuration/etl/etl_tables.d/xdb/batch-export-requests.json new file mode 100644 index 0000000000..4aa7bbae11 --- /dev/null +++ b/configuration/etl/etl_tables.d/xdb/batch-export-requests.json @@ -0,0 +1,120 @@ +{ + "table_definition": { + "name": "batch_export_requests", + "comment": "Data warehouse batch export requests.", + "engine": "InnoDB", + "columns": [ + { + "name": "id", + "type": "int(11)", + "nullable": false, + "extra": "auto_increment" + }, + { + "name": "user_id", + "type": "int(11)", + "nullable": false, + "comment": "References the user that requested the export." + }, + { + "name": "realm", + "type": "varchar(255)", + "nullable": false, + "comment": "The realm from which data will be exported." + }, + { + "name": "start_date", + "type": "date", + "nullable": false, + "comment": "Start date for the date range of the data that will be exported." + }, + { + "name": "end_date", + "type": "date", + "nullable": false, + "comment": "End date for the date range of the data that will be exported." + }, + { + "name": "export_file_format", + "type": "enum('CSV','JSON')", + "nullable": false, + "comment": "File format that will be used to store the exported data." + }, + { + "name": "requested_datetime", + "type": "datetime", + "nullable": false, + "comment": "Date and time the export request was created." + }, + { + "name": "export_succeeded", + "type": "tinyint(1)", + "nullable": true, + "comment": "True if the export was a success, false if not, null if the request has not yet been processed." + }, + { + "name": "export_created_datetime", + "type": "datetime", + "nullable": true, + "comment": "Date and time the export data was generated." + }, + { + "name": "downloaded_datetime", + "type": "datetime", + "nullable": true, + "comment": "Date and time of the first download of the exported data." + }, + { + "name": "last_modified", + "type": "timestamp", + "nullable": false, + "default": "CURRENT_TIMESTAMP", + "extra": "ON UPDATE CURRENT_TIMESTAMP" + } + ], + "indexes": [ + { + "name": "PRIMARY", + "columns": [ + "id" + ], + "is_unique": true + }, + { + "name": "idx_user_id", + "columns": [ + "user_id" + ] + }, + { + "name": "idx_requested_datetime", + "columns": [ + "requested_datetime" + ] + } + ], + "foreign_key_constraints": [ + { + "name": "fk_user_id", + "columns": [ + "user_id" + ], + "referenced_table": "Users", + "referenced_columns": [ + "id" + ], + "on_delete": "CASCADE" + } + ], + "triggers": [ + { + "schema": "moddb", + "table": "batch_export_requests", + "name": "batch_export_requests_before_insert", + "time": "BEFORE", + "event": "INSERT", + "body": "SET NEW.requested_datetime = NOW();" + } + ] + } +} From b3391463604e764245d617d330ef8cef994cd328 Mon Sep 17 00:00:00 2001 From: jsperhac Date: Fri, 12 Apr 2019 16:52:15 -0400 Subject: [PATCH 007/217] Stubs for Export batch script classes. --- classes/DataWarehouse/Export/FileHandler.php | 0 classes/DataWarehouse/Export/MailHandler.php | 0 classes/DataWarehouse/Export/QueryHandler.php | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 classes/DataWarehouse/Export/FileHandler.php create mode 100644 classes/DataWarehouse/Export/MailHandler.php create mode 100644 classes/DataWarehouse/Export/QueryHandler.php diff --git a/classes/DataWarehouse/Export/FileHandler.php b/classes/DataWarehouse/Export/FileHandler.php new file mode 100644 index 0000000000..e69de29bb2 diff --git a/classes/DataWarehouse/Export/MailHandler.php b/classes/DataWarehouse/Export/MailHandler.php new file mode 100644 index 0000000000..e69de29bb2 diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php new file mode 100644 index 0000000000..e69de29bb2 From ed40bdd225b81508be138a2e5325a2d2725d2010 Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Fri, 19 Apr 2019 16:17:38 -0400 Subject: [PATCH 008/217] Current early state of QueryHandler class for batch export script. --- classes/DataWarehouse/Export/QueryHandler.php | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index e69de29bb2..f3b3b8f6ec 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -0,0 +1,101 @@ +execute($sql); + + return $count; + } + + // Return details of all export requests presently in Submitted state. + public static function listSubmittedRecords() + { + + $sql = "SELECT id, realm, start_date, end_date + FROM batch_export_requests + WHERE export_succeeded IS NULL"; + $results = \DataWarehouse::connect()->query($sql); + + return $results; + } + + // Return details of all export requests made by specified user. + public static function listRequestsForUser($user_id) + { + // TODO: bind your user_id as appropriate please. + $sql = "SELECT id, realm, start_date, end_date, + export_succeeded, expires_datetime, + export_created_datetime, + FROM batch_export_requests + WHERE user_id=".$user_id. + "ORDER BY id"; + $results = \DataWarehouse::connect()->query($sql); + + // TODO: Logic to return neat table of records and states + // (or in another function that handles that part?) + return $results; + } +} + From 00ccd7ea6b74fe3f47ab9aa5e2fa7d6b8d757d8c Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Mon, 29 Apr 2019 10:28:17 -0400 Subject: [PATCH 009/217] Current status of class to handle querying for export. --- classes/DataWarehouse/Export/QueryHandler.php | 54 ++++++++++++++++--- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index f3b3b8f6ec..9e5b0882b2 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -5,54 +5,92 @@ * * TODO: Notes and choices: evaluate these: * make this class a singleton--does it make sense? + * or do I need to have multiple instances--and who instantiates them? + * say I'm using it for a rest endpoint--what I do need to know? * use the DataWarehouse.php (DataWarehouse class) to issue queries? - * note that it's looking rather long in the tooth. There may be a more modern way. + * No, use CCR\DB itself, which will give us PDODB. + * (note that it's looking rather long in the tooth. There may be a more modern way.) * deciding against e.g. making the state transitions reflected by classes. Too much. * think about binding variables and safe access... - * ========================================================================================== + * determine current testing practices and write tests to them + * these need to be component tests + * + * timestamp and current datetime: always use the database's value + * ========================================================================================== */ namespace DataWarehouse\Export; use Exception; -use DataWarehouse; // TODO: DB access here! or?? +use CCR\DB; class QueryHandler { + // Fetch the database handle + $pdo = DB::factory('database'); + - // singleton class should be appropriate here + // TODO verify: singleton class should be appropriate here private function __construct() { } /* transition between request record states */ - public static function createRequestRecord(realm, startDate, endDate) + // create request record for specified export request + public static function createRequestRecord($realm, $startDate, $endDate) { // TODO + $sql = "INSERT INTO batch_export_requests + (requested_datetime, user_id, realm, start_date, end_date) + VALUES + ()"; + + // TODO: return the id } // transition specified export request to Failed state from Submitted. public static function submittedToFailed($id) { - $sql = "UPDATE batch_export_requests SET export_succeeded=0 + $sql = "UPDATE batch_export_requests + SET export_succeeded=0 WHERE id=$id"; // TODO } // transition specified export request to Available state from Submitted. - public static function submittedToAvailable() + public static function submittedToAvailable($id) { + // read export retention duration from config file + $expires_in_days = xd_utilities\getConfiguration('data_warehouse_export','retention_duration'); + + // Nah, don't: + // either here, or below... + //$sqlDate = "SELECT CAST(NOW() as DATETIME"; + // select it + //$currentDatetime = + + $sql = "UPDATE batch_export_requests + SET + export_created=CAST(NOW() as DATETIME), + export_expires=DATE_ADD(CAST(NOW() as DATETIME), INTERVAL $expires_in_days DAY), + export_succeeded=1 + WHERE id=$id"; // TODO } // transition specified export request to Expired state from Available. - public static function availableToExpired() + // Note that when the table grows large we don't want to do date comparisons... + public static function availableToExpired($id) { + // TODO: create the field in the DB. // no-op // TODO: really? + $sql = "UPDATE batch_export_requests + SET export_expired=1 + WHERE id=$id"; } /* list request records states */ From 2b6617e4f71ae6b15e054141b25f56357537beea Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Mon, 29 Apr 2019 13:13:50 -0400 Subject: [PATCH 010/217] Being modern, we are now binding variables... --- classes/DataWarehouse/Export/QueryHandler.php | 115 +++++++++++------- 1 file changed, 71 insertions(+), 44 deletions(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index 9e5b0882b2..2b40155110 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -4,14 +4,14 @@ * Class governing database access by Data Warehouse Export batch script * * TODO: Notes and choices: evaluate these: - * make this class a singleton--does it make sense? + * Perhaps no: make this class a singleton--does it make sense? * or do I need to have multiple instances--and who instantiates them? * say I'm using it for a rest endpoint--what I do need to know? - * use the DataWarehouse.php (DataWarehouse class) to issue queries? - * No, use CCR\DB itself, which will give us PDODB. - * (note that it's looking rather long in the tooth. There may be a more modern way.) + * For database access and to issue queries: + * use CCR\DB namespace, which contains PDODB class that we want. * deciding against e.g. making the state transitions reflected by classes. Too much. - * think about binding variables and safe access... + * think about and safe access... + * do I need to do more to secure vars coming in * determine current testing practices and write tests to them * these need to be component tests * @@ -26,114 +26,141 @@ class QueryHandler { - // Fetch the database handle - $pdo = DB::factory('database'); + private $pdo; // populated in the constructor - // TODO verify: singleton class should be appropriate here private function __construct() { + // Fetch the database handle + $pdo = DB::factory('database'); } /* transition between request record states */ // create request record for specified export request - public static function createRequestRecord($realm, $startDate, $endDate) + public function createRequestRecord($userId, $realm, $startDate, $endDate) { - // TODO $sql = "INSERT INTO batch_export_requests (requested_datetime, user_id, realm, start_date, end_date) VALUES - ()"; + (NOW(), :user_id, :realm, :start_date, :end_date)"; - // TODO: return the id + $params = array('user_id' => $userId, + 'realm' => $realm + 'start_date' => $startDate, + 'end_date' => $endDate); + + // return the id for the inserted record + $id = $this->pdo->insert($sql); + + return($id); } // transition specified export request to Failed state from Submitted. - public static function submittedToFailed($id) + public function submittedToFailed($id) { $sql = "UPDATE batch_export_requests SET export_succeeded=0 - WHERE id=$id"; - // TODO + WHERE id=:id"; + + $params = array('id' => $id); + // Return count of affected rows--should be 1. + $result = $this->pdo->execute($sql, $params); + + return($result); } // transition specified export request to Available state from Submitted. - public static function submittedToAvailable($id) + // All time is current time as relative to database. + public function submittedToAvailable($id) { - // read export retention duration from config file + // read export retention duration from config file. Value is stored in days. $expires_in_days = xd_utilities\getConfiguration('data_warehouse_export','retention_duration'); - // Nah, don't: - // either here, or below... - //$sqlDate = "SELECT CAST(NOW() as DATETIME"; - // select it - //$currentDatetime = - $sql = "UPDATE batch_export_requests SET export_created=CAST(NOW() as DATETIME), - export_expires=DATE_ADD(CAST(NOW() as DATETIME), INTERVAL $expires_in_days DAY), + export_expires=DATE_ADD(CAST(NOW() as DATETIME), INTERVAL :expires_in_days DAY), export_succeeded=1 - WHERE id=$id"; - // TODO + WHERE id=:id"; + $params = array('expires_in_days' => $expires_in_days, + 'id' => $id); + + // Return count of affected rows--should be 1. + $result = $this->pdo->execute($sql, $params); + + return($result); } // transition specified export request to Expired state from Available. // Note that when the table grows large we don't want to do date comparisons... - public static function availableToExpired($id) + // Therefore we should create an export_expired flag on the db table. + public function availableToExpired($id) { // TODO: create the field in the DB. - // no-op - // TODO: really? + $sql = "UPDATE batch_export_requests SET export_expired=1 - WHERE id=$id"; + WHERE id=:id"; + + $params = array('id' => $id); + + // Return count of affected rows--should be 1. + $result = $this->pdo->execute($sql, $params); + + return($result); } /* list request records states */ // Return count of export requests presently in Submitted state. - public static function countSubmittedRecords() + public function countSubmittedRecords() { $sql = "SELECT COUNT(id) FROM batch_export_requests WHERE export_succeeded IS NULL"; - // TODO, you meant the PDODB execute() function here. Decide. - $count = \DataWarehouse::connect()->execute($sql); - return $count; + // Return count of rows. + $result = $this->pdo->execute($sql); + + return($result); } // Return details of all export requests presently in Submitted state. - public static function listSubmittedRecords() + public function listSubmittedRecords() { - $sql = "SELECT id, realm, start_date, end_date FROM batch_export_requests WHERE export_succeeded IS NULL"; - $results = \DataWarehouse::connect()->query($sql); - return $results; + // Return query results. + $result = $this->pdo->query($sql); + + // TODO: Logic to return neat table of records and states + // (or in another function that handles that part?) + return($result); } // Return details of all export requests made by specified user. - public static function listRequestsForUser($user_id) + public function listRequestsForUser($user_id) { - // TODO: bind your user_id as appropriate please. $sql = "SELECT id, realm, start_date, end_date, export_succeeded, expires_datetime, export_created_datetime, FROM batch_export_requests - WHERE user_id=".$user_id. - "ORDER BY id"; - $results = \DataWarehouse::connect()->query($sql); + WHERE user_id=:user_id + ORDER BY id"; + + $params = array('user_id' => $user_id); + + // Return query results. + $result = $this->pdo->query($sql, $params); // TODO: Logic to return neat table of records and states // (or in another function that handles that part?) - return $results; + return $result; } } From ad244d1f4bdc1d5cbd9be2c298d0c7b0f7f659c5 Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Mon, 29 Apr 2019 13:38:55 -0400 Subject: [PATCH 011/217] Strings like realm name need to be quoted... --- classes/DataWarehouse/Export/QueryHandler.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index 2b40155110..61e31d1112 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -46,7 +46,7 @@ public function createRequestRecord($userId, $realm, $startDate, $endDate) (NOW(), :user_id, :realm, :start_date, :end_date)"; $params = array('user_id' => $userId, - 'realm' => $realm + 'realm' => $this->pdo->quote($realm), 'start_date' => $startDate, 'end_date' => $endDate); From 7dbd26f38e9f31d2434387aad38d7e4a36edf543 Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Mon, 29 Apr 2019 13:48:00 -0400 Subject: [PATCH 012/217] Pass your array of params, please. --- classes/DataWarehouse/Export/QueryHandler.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index 61e31d1112..4978bac348 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -51,7 +51,7 @@ public function createRequestRecord($userId, $realm, $startDate, $endDate) 'end_date' => $endDate); // return the id for the inserted record - $id = $this->pdo->insert($sql); + $id = $this->pdo->insert($sql, $params); return($id); } From c6ac461393d7ecb1b971db32de36b16034527968 Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Tue, 30 Apr 2019 09:42:22 -0400 Subject: [PATCH 013/217] Public constructor. --- classes/DataWarehouse/Export/QueryHandler.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index 4978bac348..fd09e0784c 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -29,7 +29,7 @@ class QueryHandler private $pdo; // populated in the constructor - private function __construct() + function __construct() { // Fetch the database handle $pdo = DB::factory('database'); From 0124c169791854e2f797ee2ea8183b108cf953b9 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 1 May 2019 13:10:10 -0400 Subject: [PATCH 014/217] Add expiration columns --- .../etl/etl_tables.d/xdb/batch-export-requests.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/configuration/etl/etl_tables.d/xdb/batch-export-requests.json b/configuration/etl/etl_tables.d/xdb/batch-export-requests.json index 4aa7bbae11..ad6c015ce0 100644 --- a/configuration/etl/etl_tables.d/xdb/batch-export-requests.json +++ b/configuration/etl/etl_tables.d/xdb/batch-export-requests.json @@ -64,6 +64,19 @@ "nullable": true, "comment": "Date and time of the first download of the exported data." }, + { + "name": "export_expires_datetime", + "type": "datetime", + "nullable": true, + "comment": "Date and time the export data will expire." + }, + { + "name": "export_expired", + "type": "tinyint(1)", + "nullable": false, + "default": 0, + "comment": "True if the export has expired, false if not." + }, { "name": "last_modified", "type": "timestamp", From b204ac3cb1f24822a714f8d3dd4cff3fbfa7f023 Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Tue, 30 Apr 2019 09:57:22 -0400 Subject: [PATCH 015/217] Added component tests for the DB Export query handler class. Rebased to xdmod8.5. --- classes/DataWarehouse/Export/QueryHandler.php | 26 +-- configuration/portal_settings.ini | 9 + tests/component/lib/Export/ExportDBTest.php | 178 ++++++++++++++++++ tests/component/phpunit.xml.dist | 5 + 4 files changed, 207 insertions(+), 11 deletions(-) create mode 100644 tests/component/lib/Export/ExportDBTest.php diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index fd09e0784c..ac5be518f5 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -32,7 +32,7 @@ class QueryHandler function __construct() { // Fetch the database handle - $pdo = DB::factory('database'); + $this->pdo = DB::factory('database'); } /* transition between request record states */ @@ -68,7 +68,7 @@ public function submittedToFailed($id) // Return count of affected rows--should be 1. $result = $this->pdo->execute($sql, $params); - return($result); + return(count($result)==1); } // transition specified export request to Available state from Submitted. @@ -76,12 +76,12 @@ public function submittedToFailed($id) public function submittedToAvailable($id) { // read export retention duration from config file. Value is stored in days. - $expires_in_days = xd_utilities\getConfiguration('data_warehouse_export','retention_duration'); + $expires_in_days = \xd_utilities\getConfiguration('data_warehouse_export','retention_duration'); $sql = "UPDATE batch_export_requests SET - export_created=CAST(NOW() as DATETIME), - export_expires=DATE_ADD(CAST(NOW() as DATETIME), INTERVAL :expires_in_days DAY), + export_created_datetime=CAST(NOW() as DATETIME), + export_expires_datetime=DATE_ADD(CAST(NOW() as DATETIME), INTERVAL :expires_in_days DAY), export_succeeded=1 WHERE id=:id"; @@ -91,7 +91,7 @@ public function submittedToAvailable($id) // Return count of affected rows--should be 1. $result = $this->pdo->execute($sql, $params); - return($result); + return(count($result)==1); } // transition specified export request to Expired state from Available. @@ -110,7 +110,7 @@ public function availableToExpired($id) // Return count of affected rows--should be 1. $result = $this->pdo->execute($sql, $params); - return($result); + return(count($result)==1); } /* list request records states */ @@ -123,9 +123,13 @@ public function countSubmittedRecords() WHERE export_succeeded IS NULL"; // Return count of rows. - $result = $this->pdo->execute($sql); + try { + $result = $this->pdo->query($sql); + return($result[0]['COUNT(id)']); - return($result); + } catch (Exception $e) { + return $e; + } } // Return details of all export requests presently in Submitted state. @@ -147,8 +151,8 @@ public function listSubmittedRecords() public function listRequestsForUser($user_id) { $sql = "SELECT id, realm, start_date, end_date, - export_succeeded, expires_datetime, - export_created_datetime, + export_succeeded, export_expires_datetime, + export_created_datetime FROM batch_export_requests WHERE user_id=:user_id ORDER BY id"; diff --git a/configuration/portal_settings.ini b/configuration/portal_settings.ini index 8a12317618..298c1ec635 100644 --- a/configuration/portal_settings.ini +++ b/configuration/portal_settings.ini @@ -145,3 +145,12 @@ database = "mod_hpcdb" [slurm] sacct = "sacct" + +; configuration for XDMoD data warehouse export functionality +; +[data_warehouse_export] +; Exported data files will be stored in this directory +export_directory = "/path/to/export/directory" +; Length of time in days that files will be retained before automatic deletion. +retention_duration = 30 + diff --git a/tests/component/lib/Export/ExportDBTest.php b/tests/component/lib/Export/ExportDBTest.php new file mode 100644 index 0000000000..55729f6508 --- /dev/null +++ b/tests/component/lib/Export/ExportDBTest.php @@ -0,0 +1,178 @@ +countSubmittedRecords(); + $this->assertNotNull($submittedCount); + $this->assertTrue($submittedCount>=0); + } + + private function findSubmittedRecord() + { + $query = new QueryHandler(); + + // Find or create a record in submitted status to transition + $maxSubmitted = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where export_succeeded IS NULL')[0]['id']; + if ($maxSubmitted == NULL) { + $maxSubmitted = $query->createRequestRecord(self::$userId, 'Jobs', '2017-01-01','2017-08-01'); + } + return($maxSubmitted); + } + + public function testNewRecordCreation() + { + $query = new QueryHandler(); + + // find the count + $initialCount = $query->countSubmittedRecords(); + + // add new record and verify + $requestId = $query->createRequestRecord(self::$userId, 'Jobs', '2019-01-01','2019-03-01'); + $this->assertNotNull($requestId); + + // determine final count + $finalCount = $query->countSubmittedRecords(); + + // verify final count + $this->assertTrue($finalCount-$initialCount==1); + } + + public function testSubmittedToFailed() + { + $query = new QueryHandler(); + + // Find or create a record in submitted status to transition + $maxSubmitted = $this->findSubmittedRecord(); + + // Transition record + $result = $query->submittedToFailed($maxSubmitted); + $test = static::$dbh->query("SELECT export_succeeded FROM batch_export_requests where id=:id", + array('id'=>$maxSubmitted))[0]['export_succeeded']; + + // Assert that: + // Exactly one record was transitioned + $this->assertTrue($result==1); + + // That record is marked export_succeeded=FALSE + $this->assertEquals($test, 0); + } + + // TODO + // FAILS: needs export_expires_datetime column + public function testSubmittedToAvailable() + { + $query = new QueryHandler(); + + // Find or create a record in submitted status to transition + $maxSubmitted = $this->findSubmittedRecord(); + + $result = $query->submittedToAvailable($maxSubmitted); + $test = static::$dbh->query("SELECT export_succeeded FROM batch_export_requests where id=:id", + array('id'=>$maxSubmitted))[0]['export_succeeded']; + + // Assert that: + // Exactly one record was transitioned + $this->assertTrue($result==1); + + // That record is marked export_succeeded=TRUE + $this->assertEquals($test, 1); + } + + // TODO + // FAILS: needs export_expires_datetime + public function testAvailableToExpired() + { + $query = new QueryHandler(); + + // Find or create a record in available status to transition + $maxAvailable = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where export_succeeded IS TRUE')[0]['id']; + if ($maxAvailable == NULL) { + $maxSubmitted = $this->findSubmittedRecord(); + $maxAvailable = $query->submittedToAvailable($maxSubmitted); + } + + $result = $query->availableToExpired($maxAvailable); + $test = static::$dbh->query("SELECT export_expired FROM batch_export_requests where id=:id", + array('id'=>$maxAvailable))[0]['export_expired']; + + // Assert that: + // Exactly one record was transitioned + $this->assertTrue($result==1); + + // That record is marked export_expired=TRUE + $this->assertEquals($test, 1); + } + + public function testSubmittedRecordFieldList() + { + $query = new QueryHandler(); + + // Expect these keys from the associative array + $expectedKeys = array( + 'id', + 'realm', + 'start_date', + 'end_date' + ); + + $actual = $query->listSubmittedRecords(); + + if (count($actual) > 0) { + + // check that you get the same fields back from the query... + $this->assertEquals($expectedKeys, array_keys($actual[0])); + } + } + + // TODO + // FAILS: needs export_expires_datetime + public function testUserRecordFieldList() + { + $query = new QueryHandler(); + + // Expect these keys from the associative array + $expectedKeys = array( + 'id', + 'realm', + 'start_date', + 'end_date', + 'export_succeeded', + 'export_expires_datetime', + 'export_created_datetime' + ); + + // Requests via this user have been created as part of these tests + $actual = $query->listRequestsForUser(self::$userId); + + if (count($actual) > 0) { + + // check that you get the same fields back from the query... + $this->assertEquals($expectedKeys, array_keys($actual[0])); + } + } + + public static function setUpBeforeClass() + { + static::$dbh = DB::factory('database'); + static::$maxId = static::$dbh->query('SELECT COALESCE(MAX(id), 0) AS id FROM batch_export_requests')[0]['id']; + } + + public static function tearDownAfterClass() + { + static::$dbh->execute('DELETE FROM batch_export_requests WHERE id > :id', array('id' => static::$maxId)); + } +} diff --git a/tests/component/phpunit.xml.dist b/tests/component/phpunit.xml.dist index 210d3b5f8c..e1a733180e 100644 --- a/tests/component/phpunit.xml.dist +++ b/tests/component/phpunit.xml.dist @@ -18,6 +18,11 @@ stopOnSkipped="false" testSuiteLoaderClass="PHPUnit_Runner_StandardTestSuiteLoader" verbose="true"> + + lib/Export + lib/ETL + lib/Roles + lib/Roles lib/ETL From f776e8723215138f0b747f601e3c1f0c13893613 Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Thu, 2 May 2019 15:10:01 -0400 Subject: [PATCH 016/217] Modified component tests for DB Export query handler to accommodate expiration column names. --- classes/DataWarehouse/Export/QueryHandler.php | 10 ++-- tests/component/lib/Export/ExportDBTest.php | 46 +++++++++++++++---- tests/component/runtests.sh | 2 + 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index ac5be518f5..d1e5992783 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -15,6 +15,8 @@ * determine current testing practices and write tests to them * these need to be component tests * + * TODO, possibly: return list of ids for records that need to be marked 'export_expired' + * * timestamp and current datetime: always use the database's value * ========================================================================================== */ @@ -99,8 +101,6 @@ public function submittedToAvailable($id) // Therefore we should create an export_expired flag on the db table. public function availableToExpired($id) { - // TODO: create the field in the DB. - $sql = "UPDATE batch_export_requests SET export_expired=1 WHERE id=:id"; @@ -151,8 +151,10 @@ public function listSubmittedRecords() public function listRequestsForUser($user_id) { $sql = "SELECT id, realm, start_date, end_date, - export_succeeded, export_expires_datetime, - export_created_datetime + export_succeeded, + export_expired, + export_expires_datetime, + export_created_datetime FROM batch_export_requests WHERE user_id=:user_id ORDER BY id"; diff --git a/tests/component/lib/Export/ExportDBTest.php b/tests/component/lib/Export/ExportDBTest.php index 55729f6508..6d8248a033 100644 --- a/tests/component/lib/Export/ExportDBTest.php +++ b/tests/component/lib/Export/ExportDBTest.php @@ -8,10 +8,14 @@ class ExportDBTest extends BaseTest + // Test access to batch_export_requests table, via QueryHandler class. + // Clean up table to initial state following testing. + // $debug variable, if TRUE, enables print of output from public functions, { static $dbh = null; static $maxId = null; private static $userId = 34; // Tom Furlani + private static $debug = FALSE; public function testCountSubmitted() { @@ -19,6 +23,9 @@ public function testCountSubmitted() $submittedCount = $query->countSubmittedRecords(); $this->assertNotNull($submittedCount); $this->assertTrue($submittedCount>=0); + + // debug + if (self::$debug) print("\n".__FUNCTION__.": submittedRecords=$submittedCount\n"); } private function findSubmittedRecord() @@ -30,9 +37,13 @@ private function findSubmittedRecord() if ($maxSubmitted == NULL) { $maxSubmitted = $query->createRequestRecord(self::$userId, 'Jobs', '2017-01-01','2017-08-01'); } + + if (self::$debug) print("\n".__FUNCTION__.": maxSubmitted ID=$maxSubmitted\n"); + return($maxSubmitted); } + // Create two new records to enable testing of transition to Available, and transition to Expired: public function testNewRecordCreation() { $query = new QueryHandler(); @@ -44,11 +55,21 @@ public function testNewRecordCreation() $requestId = $query->createRequestRecord(self::$userId, 'Jobs', '2019-01-01','2019-03-01'); $this->assertNotNull($requestId); + // add another new record and verify + $requestId2 = $query->createRequestRecord(self::$userId, 'Accounts', '2016-12-01','2017-01-01'); + + $this->assertNotNull($requestId2 ); + $this->assertTrue($requestId2-$requestId==1); + // determine final count $finalCount = $query->countSubmittedRecords(); // verify final count - $this->assertTrue($finalCount-$initialCount==1); + // should have added 2 records. + $this->assertTrue($finalCount-$initialCount==2); + + // debug + if (self::$debug) print("\n".__FUNCTION__.": initialCount=$initialCount finalCount=$finalCount requestId=$requestId requestId2=$requestId2\n"); } public function testSubmittedToFailed() @@ -69,10 +90,11 @@ public function testSubmittedToFailed() // That record is marked export_succeeded=FALSE $this->assertEquals($test, 0); + + // debug + if (self::$debug) print("\n".__FUNCTION__.": transitioned Id=$maxSubmitted\n"); } - // TODO - // FAILS: needs export_expires_datetime column public function testSubmittedToAvailable() { $query = new QueryHandler(); @@ -90,16 +112,19 @@ public function testSubmittedToAvailable() // That record is marked export_succeeded=TRUE $this->assertEquals($test, 1); + + // debug + if (self::$debug) print("\n".__FUNCTION__.": transitioned Id=$maxSubmitted\n"); } - // TODO - // FAILS: needs export_expires_datetime public function testAvailableToExpired() { $query = new QueryHandler(); // Find or create a record in available status to transition - $maxAvailable = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where export_succeeded IS TRUE')[0]['id']; + $maxAvailable = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where + export_succeeded IS TRUE + AND export_expired IS FALSE')[0]['id']; if ($maxAvailable == NULL) { $maxSubmitted = $this->findSubmittedRecord(); $maxAvailable = $query->submittedToAvailable($maxSubmitted); @@ -115,6 +140,9 @@ public function testAvailableToExpired() // That record is marked export_expired=TRUE $this->assertEquals($test, 1); + + // debug + if (self::$debug) print("\n".__FUNCTION__.": transitioned Id=$maxAvailable\n"); } public function testSubmittedRecordFieldList() @@ -138,8 +166,6 @@ public function testSubmittedRecordFieldList() } } - // TODO - // FAILS: needs export_expires_datetime public function testUserRecordFieldList() { $query = new QueryHandler(); @@ -151,6 +177,7 @@ public function testUserRecordFieldList() 'start_date', 'end_date', 'export_succeeded', + 'export_expired', 'export_expires_datetime', 'export_created_datetime' ); @@ -165,14 +192,17 @@ public function testUserRecordFieldList() } } + // determine initial max id to enable cleanup after testing public static function setUpBeforeClass() { static::$dbh = DB::factory('database'); static::$maxId = static::$dbh->query('SELECT COALESCE(MAX(id), 0) AS id FROM batch_export_requests')[0]['id']; } + // Reset the table to where it started public static function tearDownAfterClass() { static::$dbh->execute('DELETE FROM batch_export_requests WHERE id > :id', array('id' => static::$maxId)); } + } diff --git a/tests/component/runtests.sh b/tests/component/runtests.sh index 64e3f6b94e..c3d96223b9 100755 --- a/tests/component/runtests.sh +++ b/tests/component/runtests.sh @@ -55,6 +55,8 @@ if [ 0 -ne ${#INCLUDE_GROUPS[@]} ]; then INCLUDE_GROUP_OPTION="--group "$(implode_array , ${INCLUDE_ONLY_GROUPS[@]}) fi +$phpunit ${PHPUNITARGS} --testsuite=Export -v $EXCLUDE_GROUP_OPTION $INCLUDE_GROUP_OPTION + # This test suite runs everything in lib/Roles $phpunit ${PHPUNITARGS} --testsuite=Roles -v $EXCLUDE_GROUP_OPTION $INCLUDE_GROUP_OPTION From dfbe59bf02a41f0c4229516310230b2adec957b2 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 14 May 2019 13:10:06 -0400 Subject: [PATCH 017/217] Add migration and test --- .../etl.d/xdmod-migration-8_1_2-8_5_0.json | 18 ++++++ .../lib/Database/DataWarehouseExportTest.php | 61 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 tests/integration/lib/Database/DataWarehouseExportTest.php diff --git a/configuration/etl/etl.d/xdmod-migration-8_1_2-8_5_0.json b/configuration/etl/etl.d/xdmod-migration-8_1_2-8_5_0.json index 8b4d497864..61e7f66a37 100644 --- a/configuration/etl/etl.d/xdmod-migration-8_1_2-8_5_0.json +++ b/configuration/etl/etl.d/xdmod-migration-8_1_2-8_5_0.json @@ -19,6 +19,24 @@ "schema": "modw_aggregates" } } + }, + { + "name": "update-moddb-tables", + "description": "Update moddb tables", + "namespace": "ETL\\Maintenance", + "class": "ManageTables", + "options_class": "MaintenanceOptions", + "definition_file_list": [ + "xdb/batch-export-requests.json" + ], + "endpoints": { + "destination": { + "type": "mysql", + "name": "XDMoD Database", + "config": "database", + "schema": "moddb" + } + } } ] } diff --git a/tests/integration/lib/Database/DataWarehouseExportTest.php b/tests/integration/lib/Database/DataWarehouseExportTest.php new file mode 100644 index 0000000000..800be67a38 --- /dev/null +++ b/tests/integration/lib/Database/DataWarehouseExportTest.php @@ -0,0 +1,61 @@ +db = DB::factory('database'); + $this->dbHelper = MySQLHelper::factory($this->db); + } + + /** + * Test that the table used by the data warehouse export exists. + */ + public function testTableExists() + { + $this->assertTrue( + $this->dbHelper->tableExists(self::EXPORT_REQUEST_TABLE_NAME), + sprintf('Table `%s` exists', self::EXPORT_REQUEST_TABLE_NAME) + ); + } + + /** + * Test that the table used by the data warehouse export is empty. + * + * @depends testTableExists + */ + public function testTableEmpty() + { + list($row) = $this->db->query( + sprintf( + 'SELECT COUNT(*) AS count FROM `%s`', + self::EXPORT_REQUEST_TABLE_NAME + ) + ); + $this->assertEquals( + 0, + $row['count'], + sprintf('Table `%s` is empty', self::EXPORT_REQUEST_TABLE_NAME) + ); + } +} From b82071ff0384a713e24cd4b4627f6c9edfdea132 Mon Sep 17 00:00:00 2001 From: Rudra Chakraborty Date: Tue, 2 Apr 2019 13:16:37 -0400 Subject: [PATCH 018/217] thing --- .../gui/js/modules/data_export/ExportPanel.js | 1966 +++++++++++++++++ 1 file changed, 1966 insertions(+) create mode 100644 html/gui/js/modules/data_export/ExportPanel.js diff --git a/html/gui/js/modules/data_export/ExportPanel.js b/html/gui/js/modules/data_export/ExportPanel.js new file mode 100644 index 0000000000..349dad3bd5 --- /dev/null +++ b/html/gui/js/modules/data_export/ExportPanel.js @@ -0,0 +1,1966 @@ +/* eslint no-underscore-dangle: [ + "error", + { + "allow" : [ + "_getPath", + "_processViewRequest", + "_copy", + "_createHistoryEntry", + "_createHistoryToken", + "_createHistoryTokenFromArray", + "_find", + "_fromArray", + "_generateURL", + "_generateView", + "_getParams", + "_makeRequest", + "_panelActivation", + "_performLoad", + "_truncatePath", + "_updateHistoryFromPanel", + "_upsertSearch" + ] + } +] */ + +// TODO: Move this someplace else, just here for testing... +if (!String.prototype.trim) { + String.prototype.trim = function () { + return this.replace(/^\s+|\s+$/g, ''); + }; +} + +var exceptionhandler = function (proxy, type, action, exception, response) { + switch (response.status) { + case 403: + case 500: + var details = Ext.decode(response.responseText); + Ext.Msg.alert("Error " + response.status + " " + response.statusText, details.message); + break; + case 401: + // Do nothing + break; + default: + Ext.Msg.alert(response.status + ' ' + response.statusText, response.responseText); + } +}; + +/* + * JobViewer panel + * @author Joe White + * @date 2014-09-23 + * + */ +XDMoD.Module.JobViewer = Ext.extend(XDMoD.PortalModule, { + INSTANCE: null, + + // PORTAL MODULE PROPERTIES =============================================== + title: 'Data Export', + module_id: 'data_export', + + // PORTAL MODULE TOOLBAR CONFIG =========================================== + usesToolbar: false, + + // PROPERTIES ============================================================= + token: XDMoD.REST.token, /*NOTE: This is populated via PHP. So will this render only once? */ + timeSeriesURL: '/rest/supremm/explorer/hctimeseries/', + optionWhiteList: ['host'], + storePropertyWhiteList: ['jobid'], + + children_ids: ['nodeid', 'cpuid'], + + child_to_parent: { + nodeid: 'tsid', + cpuid: 'tsid' + }, + + tabpanel_id: 'info_display', + analyticsContainerId: 'analytics_container', + treeLoaded: false, + parameters: ['realm', 'recordid', 'jobid', 'jobid', 'infoid', 'tsid'], + rest: { + warehouse: 'warehouse' + }, + + // DATA STORES ============================================================ + timeseriesstore: null, + memusedstore: null, + simdinsstore: null, + membwstore: null, + lnetstore: null, + ib_lnetstore: null, + accountdatastore: null, + acctstore: null, + mdatastore: null, + jobrecordstore: null, + store: null, + + /** + * Find the active sub tab under the job tabs + * @returns the component or false if non found + */ + getActiveJobSubPanel: function () { + var activeJobPanels = Ext.getCmp(this.tabpanel_id).getActiveTab().findByType('tabpanel'); + if (activeJobPanels.length < 1) { + return false; + } + return activeJobPanels[0].getActiveTab(); + }, + + /** + * This tabs constructor, here we take care to setup everything this tab + * will need throughout it's lifecycle. + */ + initComponent: function () { + // ROUTE: the unused toolbar events to a special no-opt function. + // Just to be sure that it doesn't go somewhere it's not + // supposed to. + this.on('role_selection_change', this.noOpt); + this.on('duration_change', this.noOpt); + + this.addEvents( + 'record_loaded', + 'data_account_loaded', + 'data_acct_loaded', + 'data_mdata_loaded', + 'data_jobrecord_loaded', + 'data_store_loaded' + ); + + // SETUP: the components for this tab and add them to this tabs 'items' + // property. + Ext.apply(this, { + items: this.setupComponents(), + customOrder: this.getToolbarConfig() + }); + + // Make sure to call the superclasses initComponent ( constructor ) + XDMoD.Module.JobViewer.superclass.initComponent.apply(this, arguments); + + this.loading = false; + + /* + * The timezone setting for highcharts is a global option that applies + * to all charts in the browser window. Individual charts in the job viewer + * can change the timezone as appropriate. The job viewer tab resets the + * timezone on deactivate to avoid impacting the highcharts plots in + * other tabs in the interface. The timezone settings are cached in the + * tab so that when it is activated again they are restored. + */ + this.cachedHighChartTimezone = null; + + }, // initComponent + + /** + * Apply implements an immutable merge (ie. returns a new object containing + * the properties of both objects.) of two javascript objects, + * lhs ( left hand side) and rhs ( right hand side). A property that exists + * in both lhs and rhs will default to the value of rhs. + * + * @param {object} lhs Left hand side of the merge. + * @param {object} rhs Right hand side of the merge. + * + **/ + apply: function (lhs, rhs) { + if (typeof lhs === 'object' && typeof rhs === 'object') { + var results = {}; + for (var property in lhs) { + if (lhs.hasOwnProperty(property)) { + results[property] = lhs[property]; + } + } + for (property in rhs) { + if (rhs.hasOwnProperty(property)) { + var rhsExists = rhs[property] !== undefined + && rhs[property] !== null; + if (rhsExists) { + results[property] = rhs[property]; + } + } + } + return results; + } + return lhs; + },// apply + + /** + * Helper function that handles setting up this components toolbar. + */ + getToolbarConfig: function () { + var self = this; + + var searchPanel = new XDMoD.Module.JobViewer.SearchPanel({ + id: 'job-viewer-search-panel', + jobViewer: this, + token: self.token + }); + + self.searchHistoryPanel.relayEvents(searchPanel, ['reload_root']); + self.searchHistoryPanel.relayEvents(this, ['reload_root']); + searchPanel.relayEvents(self, ['edit_search']); + + self.searchWindow = new Ext.Window({ + id: 'search-window', + closable: true, + closeAction: 'hide', + modal: true, + title: 'Search', + layout: 'fit', + resizable: true, + boxMaxHeight: 641, + boxMaxWidth: 1014, + items: [searchPanel] + }); + searchPanel.relayEvents(self.searchWindow, ['show', 'hide', 'move']); + + var searchButton = new Ext.Button({ + text: 'Search', + iconCls: 'search', + tooltip: 'Search for some subset of jobs', + handler: function (b) { + self.searchWindow.show(); + } + }); + + return [ + searchButton, + { + item: ' ', + separator: false + }, + XDMoD.ToolbarItem.EXPORT_MENU, + XDMoD.ToolbarItem.PRINT_BUTTON + ]; + }, + + /** + * Build the components that will make up this tabs UI. Return an array of the top level components for display in a + * 'border' layout ( note: one of the returned components *must* have 'region: center' as a property ). + * + * @returns {Array} of components to display as this tabs UI. + */ + setupComponents: function () { + var self = this; + + self.sortMode = 'age'; + + // SEARCH FORM ( CHILD OF BASIC SEARCH / NAVIGATION ) ================== + self.searchHistoryPanel = new XDMoD.Module.JobViewer.SearchHistoryPanel({ + id: 'jobviewer_search_history_panel', + jobViewer: this, + region: 'center', + listeners: { + data_loaded: function (data) { + self.treeLoaded = true; + if (self.historyEventWaiting) { + self.createHistoryCallback.call(self.createHistoryCallbackScope, self.createHistoryCallbackData); + } + } + } + }); + + self.treeSorter = new Ext.tree.TreeSorter(self.searchHistoryPanel, { + folderSort: false, + dir: "asc", + sortType: function(value, node) { + if (node.attributes.dtype == 'recordid') { + if (self.sortMode == 'age') { + return 9007199254740991 - parseInt(node.attributes.recordid, 10); + } else { + return node.attributes.text; + } + } else { + return node.attributes[node.attributes.dtype]; + } + } + }); + + // NAVIGATION ( PARENT WESTERN PANEL ) ================================= + var searchHistory = new Ext.Panel({ + region: 'west', + title: 'Search History', + collapsible: true, + collapsed: false, + layout: 'border', + collapseFirst: false, + pinned: false, + plugins: new Ext.ux.collapsedPanelTitlePlugin('Navigation'), + width: 250, + items: [ + self.searchHistoryPanel + ], + listeners: { + collapse: function (p) { + + }, + expand: function (p) { + if (p.pinned) { + p.getTool('pin').hide(); + p.getTool('unpin').show(); + } else { + p.getTool('pin').show(); + p.getTool('unpin').hide(); + } + } + }, + tools: [{ + id: 'pin', + qtip: 'Prevent auto hiding of the Search History', + hidden: false, + handler: function (ev, toolEl, p, tc) { + p.pinned = true; + if (p.collapsed) { + p.expand(false); + } + p.getTool('pin').hide(); + p.getTool('unpin').show(); + } + }, { + id: 'unpin', + qtip: 'Allow auto hiding of the Search History', + hidden: true, + handler: function (ev, toolEl, p, tc) { + p.pinned = false; + p.getTool('pin').show(); + p.getTool('unpin').hide(); + } + }] + }); // navbar ========================================================== + + var tabPanel = new Ext.TabPanel({ + id: 'info_display', + enableTabScroll: true, + region: 'center' + }); + + var assistPanel = new CCR.xdmod.ui.AssistPanel({ + region: 'center', + border: false, + headerText: 'No job is selected for viewing', + subHeaderText: 'Please refer to the instructions below:', + graphic: 'gui/images/data_export_instructions.png', + userManualRef: 'job+viewer' + }); + + var viewPanel = new Ext.Panel({ + id: 'info_display_container', + frame: false, + layout: 'card', + border: false, + activeItem: 0, + region: 'center', + items: [assistPanel, tabPanel] + }); + + // RETURN: an array of the top level components. To be used as the + // 'items' for another component with a border layout. + return new Array( + searchHistory,// WEST + viewPanel // CENTER + ); + }, // setupComponents + + /** + * Helper function that formats the provided val based on the provided units. + * + * @param val + * @param units + * @returns {*} + */ + formatData: function (val, units) { + switch (units) { + case "TODO": + case "": + case null: + return val; + break; + case "seconds": + return XDMoD.utils.format.humanTime(val); + break; + case "boolean": + return (val == 1) ? "True" : "False"; + break; + case "packets": + case "messages": + return Math.ceil(val); + break; + case "ratio": + case "1": + return XDMoD.utils.format.convertToSiPrefix(val, '', 3); + break; + case 'bytes': + case 'B': + case 'B/s': + return XDMoD.utils.format.convertToBinaryPrefix(val, units, 4); + break; + case 'kilobyte': + return XDMoD.utils.format.convertToBinaryPrefix(val * 1024.0, 'byte', 4); + break; + case 'megabyte': + return XDMoD.utils.format.convertToBinaryPrefix(val * 1024.0 * 1024.0, 'byte', 4); + break; + default: + return XDMoD.utils.format.convertToSiPrefix(val, units, 4); + break; + } + }, // formatData + + /** + * This is a marker function that is used to explicitly indicate that + * certain events are to take no action. + */ + noOpt: function () { + /** NO-OPT, what did you expect? **/ + }, // noOpt + + /** + * Helper function that retrieves the requested parameter from the provided + * source string via the provided name. + * + * @param name that will be used when looking for the the parameter in + * source. + * @param source that will be used to search for parameter 'name'. + * @returns {String} an empty string if not found, else the value of the + * parameter. + */ + getParameterByName: function (name, source) { + name = name.replace(/[\[]/, "\\[").replace(/[\]]/, "\\]"); + var regex = new RegExp("[\\?&]" + name + "=([^&#]*)"), + results = regex.exec(source); + return results === null + ? "" + : decodeURIComponent(results[1].replace(/\+/g, " ")); + }, // getParameterByName + + /** + * Generate a URL based on the provided base path. This URL will include + * the required XDMoD REST token. + * + * @param {String} base + * @param {Array} path + * @returns {String} + * @private + */ + _generateURL: function (base, path) { + + var isType = CCR.isType; + var encode = CCR.encode; + if (isType(base, CCR.Types.String) && isType(path, CCR.Types.Array)) { + var params = this._getParams(path); + var encoded = encode(params); + var result = base + '?' + encoded + '&token=' + XDMoD.REST.token; + return result; + } + return base; + }, // _generateURL + + /** + * Helper function that will generate a new view or informational tab based + * on the 'attributes.type' property. + * + * @param {Object} attributes + * @param {Array} path + * @param {String} title + * @param {Ext.tree.TreeNode} parent + * @return {*} + * @private + */ + _generateView: function (attributes, path, title, parent) { + var self = this; + var exists = CCR.exists; + + var dtype = attributes['dtype']; + var id = attributes[dtype]; + var jobId = attributes['jobid']; + + var uniqueId = [dtype, id].join('_'); + + var nestedId = [uniqueId, jobId, 'nested'].join('_'); + var textId = [uniqueId, jobId, 'text'].join('_'); + var kvId = [uniqueId, jobId, 'kv'].join('_'); + var kvStoreId = [uniqueId, jobId, 'kv', 'store'].join('_'); + var metricsId = [uniqueId, jobId, 'metrics'].join('_'); + var chartId = [uniqueId, jobId, 'chart'].join('_'); + + var valueColumnId = [uniqueId, 'column', 'value'].join('_'); + var nestedValueColumnId = [uniqueId, 'nested', 'column', 'value'].join('_'); + + var type = attributes.type; + var base = attributes.url; + + var tab; + var url = self._generateURL(base, path); + switch (type) { + case 'nested': + tab = new XDMoD.Module.JobViewer.NestedViewPanel({ + updateHistory: true, + preferWindowPath: true, + id: nestedId, + dtypes: [], + dtype: dtype, + dtypeValue: id, + jobId: jobId, + path: path, + title: title, + dataUrl: url, + autoExpandColumn: nestedValueColumnId, + columns: [ + {"header": "Key", "dataIndex": "key", mapping: "key", "width": 250}, + {"header": "Value", "dataIndex": "value", mapping: "value", id: nestedValueColumnId} + ] + + }); + break; + case 'utf8-text': + tab = new Ext.Panel({ + id: textId, + dtypes: [], + dtype: dtype, + dtypeValue: id, + jobId: jobId, + path: path, + closable: false, + updateHistory: true, + preferWindowPath: true, + title: title, + flex: 1, + layout: 'fit', + + items: [ + { + id: 'jobscript', + html: 'Loading', + autoScroll: true, + layout: 'fit', + url: url, + listeners: { + afterrender: function (panel) { + this._performLoad(this.url, panel); + } + }, + _performLoad: function (url, panel) { + Ext.Ajax.request({ + url: url, + success: function (response) { + if (response && response.responseText) { + var exists = CCR.exists; + + var json = JSON.parse(response.responseText); + var success = exists(json) && exists(json.success) && json.success; + var data = json.data || []; + var hasData = exists(data) && data.length > 0; + + if (success && hasData) { + panel.update('
' + json.data[0].value + '
'); + } else if (success && !hasData) { + panel.update('
 No Data Retrieved.
'); + } else if (!success) { + panel.update('
 An error occurred while attempting to perform the requested operation.
'); + } + } + } + }); + } + } + ] + + }); + break; + case 'metrics': + case 'keyvaluedata': + tab = new Ext.grid.GridPanel({ + dtypes: [], + title: title, + layout: 'fit', + dtype: dtype, + dtypeValue: id, + jobId: jobId, + path: path, + id: kvId, + closable: false, + updateHistory: true, + preferWindowPath: true, + height: '100%', + width: '100%', + store: new Ext.data.GroupingStore({ + id: kvStoreId, + url: url, + autoLoad: true, + proxy: new Ext.data.HttpProxy({ + api: { + read: { + url: url, + method: 'GET' + } + } + }), + reader: new Ext.data.JsonReader({ + root: 'data', + successProperty: 'success', + fields: [ + {name: 'key', mapping: 'key', type: 'string'}, + {name: 'value', mapping: 'value', type: 'string'}, + {name: 'full_value', mapping: 'value', type: 'string'}, + {name: 'units', mapping: 'units', type: 'string'}, + {name: 'group', mapping: 'group', type: 'string'}, + {name: 'help', mapping: 'help', type: 'string'}, + {name: 'documentation', mapping: 'documentation', type: 'string'} + ] + }), + listeners: { + /** + * Fires after the records have been loaded. + * + * @param {Ext.data.Store} store + * @param {Array} records + * @param {Object} options + */ + load: function (store, records, options) { + var exists = CCR.exists; + + for (var i = 0; i < records.length; i++) { + var record = records[i]; + + var value = record.get('value'); + var units = record.get('units'); + var fullValue = self.formatData(value, units, 4); + record.set('full_value', fullValue); + } + } + }, + groupField: 'group' + }), + columnLines: true, + flex: 2, + autoExpandColumn: valueColumnId, + columns: [ + { + "header": "Key", + "width": 250, + "sortable": true, + "dataIndex": "key", + renderer: function (value, metadata, record, rowIndex, colIndex, store) { + var help = record.get('documentation'); + metadata.attr = 'ext:qtip="' + help + '"'; + return value; + } + }, + { + "header": "Value", + "width": 150, + "sortable": true, + "dataIndex": "full_value", + id: valueColumnId, + renderer: {fn: self.renderFullValue, scope: self}, + editor: new Ext.form.TextField({ + allowBlank: false + }) + }, + { + "header": "Category", + "hidden": true, + "dataIndex": "group" + } + ], + view: new Ext.grid.GroupingView({ + forceFit:true, + groupTextTpl: '{text} ({[values.rs.length]} {[values.rs.length > 1 ? "Items" : "Item"]})' + }) + }); + break; + case 'detailedmetrics': + var renderValueWithUnits = function(value, node) { + if (typeof value === 'undefined') { + return ''; + } + if (node.unit) { + return self.formatData(value, node.unit); + } + return value; + }; + tab = new Ext.ux.tree.TreeGrid({ + id: metricsId, + dtypes: [], + dtype: dtype, + dtypeValue: id, + jobId: jobId, + path: path, + closable: false, + updateHistory: true, + preferWindowPath: true, + title: title, + autoScroll: true, + columnResize: false, + enableDD: true, + columns: [ + { + header: "Device", + dataIndex: "name", + width: 225, + tpl: new Ext.XTemplate('{name:this.render}', { + render: function(value, node) { + if (node.documentation) { + return '' + value + ''; + } + return value; + } + }) + }, + { + header: "Average", + dataIndex: "avg", + width: 125, + tpl: new Ext.XTemplate('{avg:this.render}', { + render: renderValueWithUnits + }) + }, + {header: "Count", dataIndex: "cnt", "width": 65}, + {header: "Standard Dev.", dataIndex: "std", "width": 125}, + { + header: "Median", + dataIndex: "med", + width: 125, + tpl: new Ext.XTemplate('{med:this.render}', { + render: renderValueWithUnits + }) + }, + {header: "Skew", dataIndex: "skw", "width": 125}, + { + header: "Minimum", + dataIndex: "min", + width: 125, + tpl: new Ext.XTemplate('{min:this.render}', { + render: renderValueWithUnits + }) + }, + { + header: "Maximum", + dataIndex: "max", + width: 125, + tpl: new Ext.XTemplate('{max:this.render}', { + render: renderValueWithUnits + }) + }, + {header: "Coefficient of variation", dataIndex: "cov", "width": 125}, + {header: "Kurtosis", dataIndex: "krt", "width": 125} + ], + dataUrl: url + }); + break; + case 'ganttchart': + tab = new XDMoD.Module.JobViewer.GanttChart({ + id: chartId, + title: title, + url: base, + baseParams: this._getParams(path), + historyToken: '#' + this.module_id + '?' + this._createHistoryTokenFromArray(path), + path: path, + dtypes: [], + dtype: dtype, + dtypeValue: id + }); + break; + case 'timeline': + case 'timeseries': + var tsid = this._find('tsid', 'tsid', path); + if (exists(tsid)) { + title = this._find('text', 'tsid', this.currentNode); + dtype = this._find('dtype', 'tsid', this.currentNode); + id = this._find(dtype, 'tsid', this.currentNode); + } + tab = new XDMoD.Module.JobViewer.ChartPanel({ + id: chartId, + dtypes: [], + dtype: dtype, + dtypeValue: id, + jobId: jobId, + path: path, + closable: false, + preferWindowPath: true, + updateHistory: true, + title: title, + layout: 'fit', + jobViewer: self, + jobTab: parent, + baseUrl: base, + store: new Ext.data.JsonStore({ + proxy: new Ext.data.HttpProxy({url: url}), + autoLoad: true, + root: 'data', + fields: ["series", "schema"] + }) + }); + break; + case 'analytics': + Ext.Ajax.request({ + dtype: dtype, + dtypeValue: id, + url: url, + method: 'GET', + success: function (response) { + var data = JSON.parse(response.responseText); + var success = exists(data) && exists(data.success) && data.success; + if (success) { + parent.fireEvent('update_analytics', data.data, true); + } + }, + failure: function (data) { + } + }); + break; + default: + break; + } + if (exists(tab)) { + tab.addListener('activate', self._panelActivation, self); + tab.helptext = { + title: tab.title, + documentation: attributes.documentation + }; + } + return tab; + }, // _generateView + + listeners: { + + /** + * Fired when this tab is either clicked on or activated. + * + * @param panel + **/ + activate: function () { + if (!this.loadMask) { + this.getExportMenu().setDisabled(true); + this.getPrintButton().setDisabled(true); + this.loadMask = new Ext.LoadMask(this.id); + } + Highcharts.setOptions({ global: { timezone: this.cachedHighChartTimezone } }); + + if (this.clearing) { + return; + } + + var token = CCR.tokenize(document.location.hash); + var params = Ext.urlDecode(token.params); + + if (params.job) { + this.loadMask.show(); + this.fireEvent('create_history_entry', Ext.decode(window.atob(params.job))); + return; + } + + if (!params.realm) { + return; + } + + if (params.action) { + this.loadMask.show(); + this.fireEvent('run_search_action', params); + return; + } + + this.loadMask.hide(); + + var selectionModel = this.searchHistoryPanel.getSelectionModel(); + + var path = this._getPath(token.raw); + var isSelected = this.compareNodePath(this.currentNode, path) && selectionModel && CCR.exists(selectionModel.getSelectedNode()); + + if (!isSelected) { + this.searchHistoryPanel.fireEvent('expand_node', path); + return; + } + + if (params.recordid && params.jobid) { + this.getExportMenu().setDisabled(!params.tsid); + this.getPrintButton().setDisabled(!params.tsid); + if (params.infoid) { + this.fireEvent('process_view_node', path); + } else { + this.fireEvent('process_job_node', path, this._processViewRequest); + } + } + }, // activate + + deactivate: function() { + this.cachedHighChartTimezone = Highcharts.getOptions().global.timezone; + Highcharts.setOptions({global: {timezone: null}}); + }, + + /** + * Takes care of clearing the whole informational display area. + * This includes the job tab panel and the analytics container. + * Also, since we just removed the whole display area, go ahead and + * reload the search history root. This will also ensure that our + * History Token is in sync with what the user is currently viewing, + * which is nothing. + */ + clear_display: function () { + this.clearing = true; + + var tabs = Ext.getCmp(this.tabpanel_id); + var analytics = Ext.getCmp(this.analyticsContainerId); + + // IF: the analytics is showing then hide it. + if (analytics && !analytics.hidden) analytics.hide(); + + // REMOVE: all of the tabs. + tabs.removeAll(); + + // FORCE: the container panel to re-lay itself out. + tabs.ownerCt.doLayout(false, true); + + this.fireEvent('reload_root'); + + this.clearing = false; + }, // clear_display + + export_option_selected: function (exportParams) { + var chartPanel = this.getActiveJobSubPanel(); + if (chartPanel) { + chartPanel.fireEvent('export_option_selected', exportParams); + } + }, + + print_clicked: function () { + var chartPanel = this.getActiveJobSubPanel(); + if (chartPanel) { + chartPanel.fireEvent('print_clicked'); + } + }, + + /** + * Process the given path for a job node history event. + * + * @param {Array} path + * @param {Boolean} isSelected + */ + process_job_node: function (path, callback) { + var self = this; + var exists = CCR.exists; + var isType = CCR.isType; + + var hasCurrentNode = exists(this.currentNode); + var hasAttributes = hasCurrentNode && exists(this.currentNode.attributes); + + if (!hasCurrentNode) { + console.log('No node... crying now ;-('); + } else if (hasCurrentNode && hasAttributes) { + + var tabs = Ext.getCmp(this.tabpanel_id); + + var jobId = this._find('jobid', 'jobid', path); + var title = this._find('text', 'jobid', this.currentNode); + + var jobPath = this._truncatePath('jobid', path); + + if (exists(jobId)) { + + jobId = parseInt(jobId); + + var found = tabs.find('jobId', jobId); + if (isType(found, CCR.Types.Array) && found.length > 0) { + var tab = found[0]; + tabs.activate(tab); + } else { + + var jobTab = new XDMoD.Module.JobViewer.JobPanel({ + itemId: 'jobid_' + jobId.toString(), + jobViewer: self, + jobId: jobId, + title: title, + path: jobPath, + listeners: { + beforeshow: function() { + // Set the tab panel to be active since a new tab is to be added. + Ext.getCmp('info_display_container').getLayout().setActiveItem(1); + }, + destroy: function () { + if (Ext.getCmp('info_display').items.length < 1) { + // All tabs have been destroyed. Set the assist image to be active + Ext.getCmp('info_display_container').getLayout().setActiveItem(0); + } + } + } + }); + + tabs.add(jobTab); + tabs.activate(jobTab); + + var base = this._find('url', 'jobid', this.currentNode) || this.searchHistoryPanel.url; + + var params = this._getParams(jobPath); + var encoded = CCR.encode(params); + var url = base + '?' + encoded + '&token=' + XDMoD.REST.token; + Ext.Ajax.request({ + url: url, + method: 'GET', + success: function (response) { + + var data = JSON.parse(response.responseText); + var success = exists(data) && exists(data.success) && data.success; + + if (success) { + var views = data.results; + + var jobTabs = jobTab.getComponent('job_tabs'); + + var tab; + for (var i = 0; i < views.length; i++) { + var view = views[i]; + view['jobid'] = jobId; + + var dtype = view['dtype']; + var id = view[dtype]; + + + var jobPath = self._copy(path, [], true); + jobPath[jobPath.length] = {dtype: dtype, value: id}; + + var currentViews = exists(view.text) ? jobTabs.find('title', view.text) : null; + var viewsExists = exists(currentViews) && exists(currentViews.length) && currentViews.length > 0; + + if (!viewsExists) { + tab = self._generateView(view, jobPath, view.text, jobTab); + if (exists(tab)) jobTabs.add(tab); + } + } + } + if (isType(callback, CCR.Types.Function)) callback.apply(self); + } + }) + } + } + + + } + }, // process_job_node + + /** + * Process the given path for the view_node history event. + * + * @param {Array} path + * @param {Boolean} isSelected + */ + process_view_node: function (path) { + var exists = CCR.exists; + var isType = CCR.isType; + var self = this; + + var hasCurrentNode = exists(this.currentNode); + var hasAttributes = hasCurrentNode && exists(this.currentNode.attributes); + + if (!hasCurrentNode) { + // TODO: write code to populate the currentNode. + console.log('No node... crying now :('); + } else if (hasCurrentNode && hasAttributes) { + + // RETRIEVE: The currentNodes attributes. + var attributes = this.currentNode.attributes; + + var jobId = this._find('jobid', 'jobid', this.currentNode); + jobId = isType(jobId, CCR.Types.Number) ? jobId : isType(jobId, CCR.Types.String) ? parseInt(jobId) : 0; + + var dType = attributes.dtype; + if (!attributes.jobid) attributes.jobid = jobId; + + // RETRIEVE: the tabs component + var tabs = Ext.getCmp(this.tabpanel_id); + + // RETRIEVE: The job tab if it exists. + var found = tabs.find('jobId', jobId); + var jobTab = isType(found, CCR.Types.Array) && found.length > 0 ? found[0] : null; + + /** + * Helper function that does the bulk of the work for + * processing a view node request. + */ + var processView = function () { + var toInt = CCR.toInt; + var currentPath = self._getPath(document.location.hash); + + var jobId = toInt(self._find('jobid', 'jobid', currentPath)); + var title = self._find('text', dType, self.currentNode); + + // RETRIEVE: the tabs component + var tabs = Ext.getCmp(self.tabpanel_id); + + // RETRIEVE: The job tab if it exists. + var jobTab = self._fromArray(tabs.find('jobId', jobId), 0); + + if (exists(jobTab)) { + + var currentlyActive = tabs.activeTab && tabs.activeTab.jobId + ? tabs.activeTab.jobId === jobTab.jobId + : false; + + if (!currentlyActive) { + jobTab.revert = false; + tabs.setActiveTab(jobTab); + return; + } + + // RETRIEVE: the tab panel that holds the informational tabs. + var infoTabs = jobTab.getComponent('job_tabs'); + + if (exists(infoTabs)) { + var isChild = self.children_ids.indexOf(dType) >= 0; + var infoDType = isChild ? self.child_to_parent[dType] : dType; + var infoValue = self._find(infoDType, infoDType, self.currentNode); + + var view = exists(title) ? infoTabs.findBy(function (component, container) { + var dTypes = component.dtypes; + var compDType = component.dtype; + var compValue = component.dtypeValue; + + return ((dTypes && dTypes.indexOf(infoDType) >= 0) || (compDType === infoDType)) && (infoValue === compValue); + }) : null; + var viewExists = exists(view) && exists(view.length) && view.length > 0; + + // CREATE: the view tab. and add it to the jobs' tabs. + if (!viewExists) { + + var tab = self._generateView(attributes, path, title, jobTab); + + // IF: we ended up with a tab being created then go ahead + // and add it and activate it. + if (exists(tab)) { + infoTabs.add(tab); + + infoTabs.activate(tab); + } + } else { + var toActivate = view[0]; + toActivate.dtypes[dType] = attributes[dType]; + + var reload = Ext.encode(toActivate.path) !== Ext.encode(currentPath); + + toActivate.path = currentPath; + toActivate.revert = true; + + infoTabs.setActiveTab(toActivate); + toActivate.fireEvent('activate', toActivate, reload); + + } + } + } + }; // processView + + + if (!exists(jobTab)) { + var jobPath = this._truncatePath('jobid', path); + this.fireEvent('process_job_node', jobPath, processView); + } else { + processView.apply(this); + } + } + }, // process_view_node + + /** + * Process a search deletion request by realm. + * + * @param realm of searches that is to be deleted. + */ + search_delete_by_realm: function (realm) { + var self = this; + var isType = CCR.isType; + var exists = CCR.exists; + if (isType(realm, CCR.Types.String)) { + Ext.MessageBox.confirm('Delete All Saved Searches?', 'Do you want to delete all saved searches for the realm: ' + realm + ' ?', + function (btn) { + if (btn === 'ok' || btn === 'yes') { + Ext.Ajax.request({ + /*'/rest/datawarehouse/search/history?realm=' + realm + '&token=' + XDMoD.REST.token,*/ + url: XDMoD.REST.url + '/' + self.rest.warehouse + '/search/history?realm=' + realm + '&token=' + XDMoD.REST.token, + method: 'DELETE', + success: function (response) { + var data = JSON.parse(response.responseText); + var success = exists(data) && exists(data.success) && data.success; + if (success) { + self.fireEvent('clear_display'); + var current = Ext.History.getToken(); + if (isType(current, CCR.Types.String)) { + var currentToken = CCR.tokenize(current); + var token = currentToken.tab + '?realm=' + realm; + Ext.History.add(token); + } else if (isType(current, CCR.Types.Object)) { + var token = current.tab + '?realm=' + realm; + Ext.History.add(token); + } + } else { + Ext.MessageBox.show({ + title: 'Deletion Error', + msg: 'There was an error removing all searches for the realm: [' + realm + '].', + icon: Ext.MessageBox.ERROR, + buttons: Ext.MessageBox.OK + }); + } + }, + failure: function (response) { + Ext.MessageBox.show({ + title: 'Deletion Error', + msg: 'There was an error removing all searches for the realm: [' + realm + '].', + icon: Ext.MessageBox.ERROR, + buttons: Ext.MessageBox.OK + }); + } + }); + } else { + Ext.MessageBox.alert('Ok', 'All your searches are belong to you.'); + } + }); + } + }, // search_delete_by_realm + + /** + * Process a search deletion request for a given node. + * + * @param {Ext.tree.TreeNode} node on which to process the deletion request. + */ + search_delete_by_node: function (node) { + var self = this; + var isType = CCR.isType; + var exists = CCR.exists; + var encode = CCR.encode; + if (isType(node, CCR.Types.Object)) { + var title = node.text || node.attributes ? node.attributes.text : undefined; + Ext.MessageBox.confirm('Delete All Saved Searches?', 'Do you want to delete the search: ' + title + ' ?', + function (text) { + if (text === 'ok' || text === 'yes') { + var recordId = node.attributes.recordid !== undefined ? node.attributes.recordid : null; + var fragment = recordId !== null ? '/search/history/' + recordId : '/search/history'; + var path = self._getPath(node); + /*'/rest/datawarehouse/search/history'*/ + var url = self._generateURL(XDMoD.REST.url + '/' + self.rest.warehouse + fragment, path); + Ext.Ajax.request({ + url: url, + method: 'DELETE', + success: function (response) { + var data = JSON.parse(response.responseText); + var success = exists(data) && exists(data.success) && data.success; + if (success) { + self.fireEvent('clear_display'); + var current = Ext.History.getToken(); + var path = self._getPath(node); + if (path && path.length && path.length > 0) delete path[path.length - 1]; + var params = self._getParams(path); + var encoded = encode(params); + if (isType(current, CCR.Types.String)) { + var currentToken = CCR.tokenize(current); + var token = currentToken.tab + '?' + encoded; + Ext.History.add(token); + } else if (isType(current, CCR.Types.Object)) { + var token = current.tab + '?' + encoded; + Ext.History.add(token); + } + } else { + Ext.MessageBox.show({ + title: 'Deletion Error', + msg: 'There was an error removing search: [' + title + '].', + icon: Ext.MessageBox.ERROR, + buttons: Ext.MessageBox.OK + }); + } + }, + failure: function (response) { + Ext.MessageBox.show({ + title: 'Deletion Error', + msg: 'There was an error removing search: [' + title + '].', + icon: Ext.MessageBox.ERROR, + buttons: Ext.MessageBox.OK + }); + } + }) + } + } + ); + + } + }, // search_delete_by_node + + + /** + * Attempt to create a history entry. This will queue the work to be + * done if the Search History Tree has no + * + * @param data + */ + create_history_entry: function (data) { + if (this.treeLoaded) { + this._createHistoryEntry(data); + } else { + this.historyEventWaiting = true; + this.createHistoryCallback = this._createHistoryEntry; + this.createHistoryCallbackData = data; + this.createHistoryCallbackScope = this; + } + }, + + /** + * Run a job search and save the first result in the search history + */ + run_search_action: function (searchparams) { + var self = this; + + Ext.Ajax.request({ + url: XDMoD.REST.url + '/' + this.rest.warehouse + '/search/jobs', + method: 'GET', + params: { + token: XDMoD.REST.token, + realm: searchparams.realm, + params: JSON.stringify(searchparams) + }, + success: function (response) { + var data = JSON.parse(response.responseText); + if (data.success === false || data.totalCount < 1) { + Ext.Msg.show({ + title: 'No results', + msg: 'No jobs were found that meet the requested search parameters.', + buttons: Ext.Msg.OK, + fn: Ext.History.add(self.module_id + '?realm=' + searchparams.realm), + icon: Ext.MessageBox.INFO + }); + return; + } + var historyEntry = { + title: searchparams.title || 'Linked Search', + realm: searchparams.realm, + text: data.results[0].text, + job_id: data.results[0].jobid, + local_job_id: data.results[0].local_job_id + }; + self.fireEvent('create_history_entry', historyEntry); + }, + failure: function (response) { + var message; + try { + var result = JSON.parse(response.responseText); + if (result.message) { + message = result.message; + } + } catch (e) { + message = 'Error processing request'; + } + + Ext.Msg.show({ + title: 'Error ' + response.status + ' ' + response.statusText, + msg: message, + buttons: Ext.Msg.OK, + fn: Ext.History.add(self.module_id + '?realm=' + searchparams.realm), + icon: Ext.MessageBox.ERROR + }); + } + }); + }, + + /** + * Update the sort order for the nodes in the search history tree. + * Does nothing if the current sort order is same as requested + * + * @param the requested sort order + */ + update_tree_sort: function(sortMode) { + if (this.sortMode != sortMode) { + this.sortMode = sortMode; + this.searchHistoryPanel.getRootNode().reload(function() { + this.currentNode = this.searchHistoryPanel.getRootNode(); + this.fireEvent('activate'); + }, this); + } + }, + + /** + * Process a node selected event. This results in the History Token + * ( document.location.hash ) being updated to reflect the node + * that is currently selected. Also ensure that the node selected is + * recorded as the current node. + * + * @param node that has been selected. + */ + node_selected: function (node) { + var exists = CCR.exists; + + this.currentNode = node; + + var token = this._createHistoryToken(node); + + var raw = Ext.History.getToken(); + var current = typeof raw === 'string' ? CCR.tokenize(raw) : raw; + + if (current.params === token) { + this.fireEvent('activate'); + } else { + if (exists(token)) Ext.History.add(this.module_id + '?' + token, false); + } + }, // node_selected + + }, // listeners =========================================================== + + /** + * Handles rendering the 'full_value' column of the 'keyvaluepair' grid. + * + * @param {Object} value + * @param {Object} metadata + * @param {Ext.data.Record} record + * @param {Number} rowIndex + * @param {Number} colIndex + * @param {Ext.data.Store} store + */ + renderFullValue: function (value, metadata, record, rowIndex, colIndex, store) { + return value; + }, // renderFullValue + + /** + * Create a history token from the provided `node`. + * + * @param {Ext.tree.TreeNode} node + * @returns {null|String} + * @private + */ + _createHistoryToken: function (node) { + var exists = CCR.exists; + if (exists(node) && exists(node.attributes)) { + + var results = []; + for (var next = node; next.parentNode !== null; next = next.parentNode) { + var key = next.attributes['dtype']; + var value = exists(key) ? next.attributes[key] : null; + if (exists(value)) results.push(key + '=' + value); + } + + return results.reverse().join('&'); + } + return null; + }, // _createHistoryToken + + /** + * Create a history token from the provided array of objects which provide + * minimally: + * { + * dtype: string, + * value: * + * } + * + * @param data + * @returns {*} + * @private + */ + _createHistoryTokenFromArray: function (data) { + var exists = CCR.exists; + if (exists(data) && exists(data.length) && data.length > 0) { + + var results = []; + for (var i = 0; i < data.length; i++) { + var next = data[i]; + var key = next['dtype']; + var value = next['value']; + if (exists(key) && exists(value)) results.push(key + '=' + value); + } + var reverse = results && results.length > 0 && results[0].indexOf('realm') < 0; + if (reverse) results.reverse(); + return results.join('&'); + } + return null; + }, // _createHistoryTokenFromArray + + /** + * Truncate the path at the first object that has the provided 'property'. + * + * @param {String} dtype + * @param {Array} path + * @returns {Array} + * @private + */ + _truncatePath: function (dtype, path) { + var isType = CCR.isType; + + if (!isType(dtype, CCR.Types.String)) return path; + if (!isType(path, CCR.Types.Array)) return path; + var results = []; + for (var i = 0; i < path.length; i++) { + var entry = path[i]; + if (entry.hasOwnProperty('dtype') && entry['dtype'] !== dtype) { + results.push(entry); + } else if (entry['dtype'] === dtype) { + results.push(entry); + break; + } + } + return results; + },// _truncatePath + + /** + * Attempt to find ( return ) the provided property of the provided node, + * or one of it's children, iff the node or child has the provided dtype. + * + * @param property to be found. + * @param dtype that will be used to qualify the node. + * @param node the root node to be used in the search. + * @returns {*} the value of the property, if found. + * @private + */ + _find: function (property, dtype, node) { + var isType = CCR.isType; + var exists = CCR.exists; + + if (!isType(property, CCR.Types.String)) return undefined; + if (!exists(node)) return undefined; + + var hasDtype = isType(dtype, CCR.Types.String); + var isObject = isType(node, CCR.Types.Object); + var isArray = isType(node, CCR.Types.Array); + + if (isObject) { + for (var current = node; exists(current); current = current.parentNode) { + var attributes = current.attributes || {}; + if (hasDtype && attributes.hasOwnProperty('dtype') && attributes.dtype === dtype && attributes.hasOwnProperty(property)) { + return attributes[property]; + } else if (!hasDtype && attributes.hasOwnProperty(property)) { + return attributes[property]; + } + } + } else if (isArray) { + for (var i = 0; i < node.length; i++) { + var attributes = node[i]; + if (hasDtype && attributes.hasOwnProperty('dtype') && attributes.dtype === dtype && attributes.hasOwnProperty('value')) { + return attributes['value']; + } else if (!hasDtype && attributes.hasOwnProperty(property)) { + return attributes[property]; + } + } + } + return null; + }, // _find + + /** + * Retrieve the 'path' values from the provided tree node or window hash. + * + * @param {String|Ext.tree.TreeNode} node + * @returns {Array} + * @private + */ + _getPath: function (node, keys) { + var exists = CCR.exists; + var isType = CCR.isType; + var results = []; + if (isType(node, CCR.Types.Object)) { + for (; exists(node); node = node.parentNode) { + var attributes = node.attributes || []; + var dtype = attributes['dtype']; + var value = exists(dtype) ? attributes[dtype] : undefined; + if (exists(value)) results.push({dtype: dtype, value: value}); + } + return results.reverse(); + } else if (isType(node, CCR.Types.String)) { + + var results = []; + + var token = CCR.tokenize(node); + var params = token && token.params && token.params.split ? token.params.split('&') : []; + for (var i = 0; i < params.length; i++) { + var param = params[i].split('='); + var key = param[0]; + var value = param[1]; + + var keyIndex = !exists(keys) || keys.indexOf(key); + if (keyIndex >= 0) { + results.push({dtype: key, value: value}); + } + } + return results; + } + }, // _getPath + + /** + * Accepts an Array of objects of the correct format and returns a query parameter style string. + * + * + * path should be provided in the following format: + * [ + * { dtype: string, value: string }, + * .... + * ] + * + * @param {Array} path + * @return {Object} in the form { dtype: value, dtype: value, ... } + * @private + */ + _getParams: function (path) { + var exists = CCR.exists; + + if (exists(path)) { + var results = {}; + for (var i = 0; i < path.length; i++) { + var part = path[i]; + var key = exists(part) && exists(part.dtype) ? part.dtype : null; + var value = exists(part) && exists(part.value) ? part.value : null; + if (exists(key) && exists(value)) results[key] = value; + } + return results; + } + return {} + }, //_getParams + + /** + * + * @param panel + * @private + */ + _panelActivation: function (panel) { + if (panel.updateHistory) this._updateHistoryFromPanel(panel); + }, // _panelActivation + + /** + * Attempt to update the History Token from a panel being activated. This + * captures the user scenario of clicking on an informational tab and not + * on a Search History Node. + * + * @param panel + * @private + */ + _updateHistoryFromPanel: function (panel) { + var token; + if (panel.path) { + token = '#' + this.module_id + '?' + this._createHistoryTokenFromArray(panel.path); + } else { + + var path = this._getPath(window.location.hash); + + var dtype = panel.dtype; + var value = panel.dtypeValue; + + path = this._truncatePath(dtype, path); + + var dtypeFound = false; + var dtypeValuesDiffer = true; + for (var i = 0; i < path.length; i++) { + var entry = path[i]; + if (entry.dtype === dtype) { + dtypeFound = true; + + dtypeValuesDiffer = entry.value != value; + entry.value = value; + break; + } + } + + if (!dtypeFound) { + path.push({ + dtype: dtype, + value: value + }); + token = '#' + this.module_id + '?' + this._createHistoryTokenFromArray(path); + } else if (dtypeValuesDiffer) { + token = '#' + this.module_id + '?' + this._createHistoryTokenFromArray(path); + } + } + Ext.History.add(token, true); + }, // _updateHistoryFromPanel + + /** + * Copy 'from' array -> 'to' array. If the append flag is set then it + * appends, else if overwrites. + * + * @param {Array} from + * @param {Array} to + * @param {Boolean} append if true then appends, else overwrites. + * @private + */ + _copy: function (from, to, append) { + var exists = CCR.exists; + var isType = CCR.isType; + + if (!isType(from, CCR.Types.Array)) throw new Error('Can only copy from arrays.'); + if (!isType(to, CCR.Types.Array)) return new Error('Can only copy to arrays.') + append = exists(append); + + for (var i = 0; i < from.length; i++) { + if (append) { + to.push(from[i]); + } else if (i <= to.length) { + to[i] = from[i]; + } + } + return to; + }, // _copy + + /** + * Attempt to process a request to show the currentNodes view. This occurs + * after a job_node has been processed ( expanded ). + * + * @private + */ + _processViewRequest: function () { + var isType = CCR.isType; + var exists = CCR.exists; + + var jobId = this._find('jobid', 'jobid', this.currentNode); + var title = this._find('text', 'jobid', this.currentNode); + + jobId = isType(jobId, CCR.Types.Number) ? jobId : isType(jobId, CCR.Types.String) ? parseInt(jobId) : 0; + + // RETRIEVE: the tabs component + var tabs = Ext.getCmp(this.tabpanel_id); + + // RETRIEVE: The job tab if it exists. + var found = tabs.find('jobId', jobId); + + var jobTab = isType(found, CCR.Types.Array) && found.length > 0 ? found[0] : null; + if (exists(jobTab)) { + + var currentlyActive = tabs.activeTab && tabs.activeTab.jobId + ? tabs.activeTab.jobId === jobTab.jobId + : false; + + if (!currentlyActive) { + tabs.setActiveTab(jobTab); + } + + var jobTabs = jobTab.getComponent('job_tabs'); + if (exists(jobTabs)) { + var hasTabs = jobTabs.items.length > 0; + if (hasTabs) { + jobTabs.activate(jobTabs.items.get(0)); + } + } + } + }, // _processViewRequest + + /** + * Replace the property found in the paths' array of objects + * with the provided value if found. + * + * @param {String} property + * @param {*} value + * @param {Array} path + * @private + */ + _replace: function (property, value, path) { + var isType = CCR.isType; + if (!isType(path, CCR.Types.Array)) return; + for (var i = 0; i < path.length; i++) { + var entry = path[i]; + if (entry.dtype === property) { + entry.value = value; + return; + } + } + return; + }, // _replace + + + /** + * Retrieve the value at the index provided from 'data'. This is done in a + * null-safe / index-safe way. + * + * @param {Array} data + * @param {Number} index + * @returns {*} + * @private + */ + _fromArray: function (data, index) { + var isType = CCR.isType; + if (!isType(data, CCR.Types.Array) || (!isType(index, CCR.Types.Number) & index < 0)) return undefined; + if (index >= data.length) return undefined; + + return data[index]; + }, // _fromArray + + /** + * Compare the given search history tree node with the provided path array. + * The path encoding from the _getPath function. + * + * @param Ext.tree.TreeNode node + * @param Array path + * @returns true if the path array matches the tree node, false otherwise + */ + compareNodePath: function (node, path) { + var i; + var np; + for (np = node, i = path.length - 1; np && np.attributes && np.attributes.dtype; np = np.parentNode, --i) { + if (i < 0) { + return false; + } + if (path[i].dtype !== np.attributes.dtype || path[i].value !== String(np.attributes[np.attributes.dtype])) { + return false; + } + } + return i === -1; + }, + + /** + * Attempt to create a history entry ( node located in the Search History + * Panel ) from the provided options. This occurs when interpreting the + * results of a History Token activation from Metric Explorer. + * + * @param options + * @private + */ + _createHistoryEntry: function (options) { + var self = this; + var searchTitle = options.title; + var realm = options.realm; + var jobTitle = options.text; + var jobId = options.job_id; + var jobLocalId = options.local_job_id; + + var jobData = { + resource: "", + name: "", + jobid: jobId, + text: jobTitle, + dtype: 'jobid', + local_job_id: jobLocalId + }; + + var searchPromise = this._makeRequest( + 'GET', + XDMoD.REST.url + '/' + this.rest.warehouse + '/search/history', + null, + { + realm: realm, + title: searchTitle, + token: XDMoD.REST.token + } + ); + + searchPromise.then(function (results) { + var data = results.data; + var recordId = data.recordid; + var jobs = data.results || [ + jobData + ]; + var jobFound = false; + for (var i = 0; i < jobs.length; i++) { + var job = jobs[i]; + if (job.jobid == jobId) { + jobFound = true; + } + } + if (!jobFound) jobs.push(jobData); + var upsertPromise = self._upsertSearch(realm, searchTitle, recordId, jobs, null); + upsertPromise.then(function (data) { + var realmNode = self.searchHistoryPanel.root.findChild('text', realm); + var path = self._getPath(realmNode); + recordId = recordId || data.results.recordid; + path.push({ + dtype: 'recordid', + value: recordId + }); + path.push({ + dtype: 'jobid', + value: jobId + }); + self.fireEvent('reload_root', path); + self.historyEventWaiting = false; + })['catch'](function (response) { + var message = response.msg || response.message || 'Updating the provided search.'; + CCR.error('Error', 'Unable to complete the requested operation: ' + message); + }); + })['catch'](function (response) { + // It wasn't found, so we need to create it. + var upsertPromise = self._upsertSearch(realm, searchTitle, null, [jobData], null); + upsertPromise.then(function (data) { + var realmNode = self.searchHistoryPanel.root.findChild('text', realm); + var path = self._getPath(realmNode); + var recordId = recordId || data.results.recordid; + path.push({ + dtype: 'recordid', + value: recordId + }); + path.push({ + dtype: 'jobid', + value: jobId + }); + self.fireEvent('reload_root', path); + self.historyEventWaiting = false; + }); + /* var message = response.msg || response.message || 'Retrieving search info.'; + CCR.error('Error', 'Unable to complete the requested operation: ' + message);*/ + }); + }, // _createHistoryEntry + + /** + * Attempts to execute an 'upsert' ( either an update or an insert depending + * on the context ) of a 'search' which is identified by the provided + * properties. Note that the results will be provided via a Promise so + * the caller will need to call: + * + * _upsertSearch(realm, title, id, jobs).then( function(results) { + * ... processing logic goes here ... + * }).catch( function(errorResponse) { + * ... error logic goes here ... + * }); + * + * @param {String} realm in which this search took place + * @param {String} title to give the search. + * @param {String} id optional, if provided, then this will be used as the 'recordid' + * which will indicate an update. + * @param {Object} jobs that should be associated with this search. + * @param {Array} searchTerms optional, that should be included with this search. + * @returns {*|Promise.} + * @private + */ + _upsertSearch: function (realm, title, id, jobs, searchTerms) { + + var url = XDMoD.REST.url + '/' + this.rest.warehouse + '/search/history?realm=' + realm + '&token=' + XDMoD.REST.token; + searchTerms = searchTerms || {}; + var params = { + 'data': JSON.stringify( + { + "text": title, + "searchterms": searchTerms, + "results": jobs + }) + }; + if (CCR.exists(id)) params['recordid'] = id; + + return this._makeRequest('POST', url, null, params); + }, // _upsertSearch + + /** + * Helper function that wraps making an AJAX call via Exts' Ext.Ajax.request + * method in a Promise. It also expects the response to be JSON and that it + * it will have a 'success' root property. This is for the proper handling + * of the resolve, reject methods. If the method returns anything other than + * a '200' status code then the reject ( catch ) method will be called. Also + * , if the root property 'success' does not exist or if it does exist but + * is false then the reject ( catch ) method will be called as well. In all + * other cases the resolve ( then ) method will be called. + * + * @param {String} method to use when making the request: GET|PUT|POST + * |DELETE|PATCH + * @param {String} url to use when making the request. + * @param {Object} data optional, will be included as the request options + * 'data' property. + * @param {Object} params optional, will be included as the request options + * 'params' property. + * @returns {Promise} that can be used to complete the requested request + * @private + */ + _makeRequest: function (method, url, data, params) { + return new RSVP.Promise(function (resolve, reject) { + var options = { + method: method, + url: url, + success: function (response) { + var data = JSON.parse(response.responseText); + var success = CCR.exists(data) && CCR.exists(data.success) && data.success; + if (success) { + resolve(data); + } else { + reject(response); + } + }, + failure: function (response) { + reject(response); + } + }; + + if (CCR.exists(data)) options['data'] = data; + if (CCR.exists(params)) options['params'] = params; + Ext.Ajax.request(options); + }); + } // _makeRequest +}) +; //XDMoD.Module.JobViewer From f91a33a18895bb12d069cb1f796da13189a24cc8 Mon Sep 17 00:00:00 2001 From: Rudra Chakraborty Date: Tue, 2 Apr 2019 15:53:39 -0400 Subject: [PATCH 019/217] more stuff --- configuration/roles.json | 9 +++++++++ html/gui/js/CCR.js | 3 ++- .../data_export/{ExportPanel.js => DataExport.js} | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) rename html/gui/js/modules/data_export/{ExportPanel.js => DataExport.js} (99%) diff --git a/configuration/roles.json b/configuration/roles.json index e8ec94d2a7..87dc613238 100644 --- a/configuration/roles.json +++ b/configuration/roles.json @@ -56,6 +56,15 @@ "javascriptReference": "CCR.xdmod.ui.aboutXD", "userManualSectionName": "About", "tooltip": "" + }, + { + "name": "data_export", + "title": "Data Exporter", + "position": 1100, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "userManualSectionName": "Data Exporter", + "tooltip": "" } ], "query_descripters": [ diff --git a/html/gui/js/CCR.js b/html/gui/js/CCR.js index 3b8f0960f5..a9ea736222 100644 --- a/html/gui/js/CCR.js +++ b/html/gui/js/CCR.js @@ -580,7 +580,8 @@ CCR.xdmod.reporting.dirtyState = false; CCR.xdmod.catalog = { metric_explorer: {}, - report_generator: {} + report_generator: {}, + data_export: {} }; CCR.xdmod.ui.invertColor = function (hexTripletColor) { diff --git a/html/gui/js/modules/data_export/ExportPanel.js b/html/gui/js/modules/data_export/DataExport.js similarity index 99% rename from html/gui/js/modules/data_export/ExportPanel.js rename to html/gui/js/modules/data_export/DataExport.js index 349dad3bd5..3866615108 100644 --- a/html/gui/js/modules/data_export/ExportPanel.js +++ b/html/gui/js/modules/data_export/DataExport.js @@ -51,7 +51,7 @@ var exceptionhandler = function (proxy, type, action, exception, response) { * @date 2014-09-23 * */ -XDMoD.Module.JobViewer = Ext.extend(XDMoD.PortalModule, { +XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { INSTANCE: null, // PORTAL MODULE PROPERTIES =============================================== From afcba76fca65ebb18a15379e36b8aca1133c154a Mon Sep 17 00:00:00 2001 From: Rudra Chakraborty Date: Tue, 2 Apr 2019 16:13:40 -0400 Subject: [PATCH 020/217] things --- configuration/roles.json | 4 +- html/gui/js/modules/DataExport.js | 177 ++ html/gui/js/modules/data_export/DataExport.js | 1966 ----------------- 3 files changed, 179 insertions(+), 1968 deletions(-) create mode 100644 html/gui/js/modules/DataExport.js delete mode 100644 html/gui/js/modules/data_export/DataExport.js diff --git a/configuration/roles.json b/configuration/roles.json index 87dc613238..1d7ece559e 100644 --- a/configuration/roles.json +++ b/configuration/roles.json @@ -59,11 +59,11 @@ }, { "name": "data_export", - "title": "Data Exporter", + "title": "Data Export", "position": 1100, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", - "userManualSectionName": "Data Exporter", + "userManualSectionName": "Data Export", "tooltip": "" } ], diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js new file mode 100644 index 0000000000..665283572b --- /dev/null +++ b/html/gui/js/modules/DataExport.js @@ -0,0 +1,177 @@ +XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { + + title: 'Data Export', // <-- rename this + + module_id: 'data_export', // <-- rename this (see the section on REPORT CHECKBOX for how to name this) + + usesToolbar: true, + + toolbarItems: { + + durationSelector: true, + exportMenu: true, + printButton: true, + reportCheckbox: true + + }, + + // ------------------------------------------------------------------ + + initComponent: function () { + + this.on('role_selection_change', function(config) { + + /* + + Fired upon selecting an entry from the Role menu + + 'config' represents the configuration of the selected menu entry + + The properties which will be of use are the following: + + config.text (the label of the selected item) + config.value (the internal value associated with the selected item) + + */ + + });//role_selection_change + + // ============================================== + + this.on('duration_change', function(config) { + + /* + + Fired upon any of the following actions: + - Selecting a duration preset + - Selecting an aggregation unit + - Pressing ENTER to confirm an updated (and valid) date in the Start or End field + - Pressing the Refresh button + + 'config' is an object with the following details: + + { + aggregation_unit: ______, + preset: _____, + start_date: YYYY-MM-DD, + end_date: YYYY-MM-DD + } + + */ + + });//duration_change + + // ============================================== + + this.on('export_option_selected', function (config){ + + /* + + Fired upon selecting an entry from the Export menu + + 'config' represents the export parameters associated with the selected + entry, and is an object with the following details: + + { + format: ____, + width: ____, + height: ____, + inline: ____, + scale: ____, + show_title: ____, + } + + */ + + });//export_option_selected + + // ============================================== + + this.on('print_clicked', function() { + + /* + + Fired upon clicking the Print button + + */ + + });//print_clicked + + // ============================================== + + /* + + REPORT CHECKBOX + + Setting the check state of the 'Available For Report' checkbox is handled manually: + + self.getReportCheckbox().storeChartArguments(chart_args, title, subtitle, start_date, end_date, included_in_report); + + When the above call is made, the included_in_report value (either 'y' or 'n') determines whether the 'Available For Report' + checkbox is checked or not. When the user manually checks the 'Available For Report' checkbox, the chart arguments cached + via the last call to storeChartArguments() will be used. + + It is important that the module_id set in the configuration to this XDMoD.PortalModule subclass be unique among all other + XDMoD.PortalModule subclasses. The reporting layer relies on distinct module_id(s) to function properly. + + NOTE: The value of module_id needs to match the base name of the corresponding controller used to serve up the chart data. + + e.g. If your module consults 'controllers/abc.php' to get its chart data, and you want to add that chart data to a report, + the module_id needs to be named 'abc' + + */ + + // ============================================== + + var mainArea = new Ext.Panel({ + + region: 'center', + html: 'This is my new module' + + });//mainArea + + // ============================================== + + var customToolbarComponent = new Ext.Button({ + + text: 'Custom' + + });//customToolbarComponent + + // ============================================== + + Ext.apply(this, { + + /* + + If custom components are to be placed in the toolbar, you can specify where they are to be placed, relative + to the components available to you by default. Simply define a 'customOrder' property, which is an array + representing the order in which the components are to be placed/arranged. + + customOrder: [ + + XDMoD.ToolbarItem.DURATION_SELECTOR, + XDMoD.ToolbarItem.EXPORT_MENU, + customToolbarComponent, + XDMoD.ToolbarItem.PRINT_BUTTON, + XDMoD.ToolbarItem.REPORT_CHECKBOX + + ], + + The top-down ordering of the components in the 'customOrder' array corresponds to the left-right ordering of + the components in the top toolbar. In this example, the 'Custom' button (specific to this module) is placed + to the right of the Export menu and to the left of the Print button. + + */ + + items: [ + mainArea + ] + + });//Ext.apply + + XDMoD.Module.DataExport.superclass.initComponent.apply(this, arguments); + + },//initComponent + + });//XDMoD.Module.DataExport \ No newline at end of file diff --git a/html/gui/js/modules/data_export/DataExport.js b/html/gui/js/modules/data_export/DataExport.js deleted file mode 100644 index 3866615108..0000000000 --- a/html/gui/js/modules/data_export/DataExport.js +++ /dev/null @@ -1,1966 +0,0 @@ -/* eslint no-underscore-dangle: [ - "error", - { - "allow" : [ - "_getPath", - "_processViewRequest", - "_copy", - "_createHistoryEntry", - "_createHistoryToken", - "_createHistoryTokenFromArray", - "_find", - "_fromArray", - "_generateURL", - "_generateView", - "_getParams", - "_makeRequest", - "_panelActivation", - "_performLoad", - "_truncatePath", - "_updateHistoryFromPanel", - "_upsertSearch" - ] - } -] */ - -// TODO: Move this someplace else, just here for testing... -if (!String.prototype.trim) { - String.prototype.trim = function () { - return this.replace(/^\s+|\s+$/g, ''); - }; -} - -var exceptionhandler = function (proxy, type, action, exception, response) { - switch (response.status) { - case 403: - case 500: - var details = Ext.decode(response.responseText); - Ext.Msg.alert("Error " + response.status + " " + response.statusText, details.message); - break; - case 401: - // Do nothing - break; - default: - Ext.Msg.alert(response.status + ' ' + response.statusText, response.responseText); - } -}; - -/* - * JobViewer panel - * @author Joe White - * @date 2014-09-23 - * - */ -XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { - INSTANCE: null, - - // PORTAL MODULE PROPERTIES =============================================== - title: 'Data Export', - module_id: 'data_export', - - // PORTAL MODULE TOOLBAR CONFIG =========================================== - usesToolbar: false, - - // PROPERTIES ============================================================= - token: XDMoD.REST.token, /*NOTE: This is populated via PHP. So will this render only once? */ - timeSeriesURL: '/rest/supremm/explorer/hctimeseries/', - optionWhiteList: ['host'], - storePropertyWhiteList: ['jobid'], - - children_ids: ['nodeid', 'cpuid'], - - child_to_parent: { - nodeid: 'tsid', - cpuid: 'tsid' - }, - - tabpanel_id: 'info_display', - analyticsContainerId: 'analytics_container', - treeLoaded: false, - parameters: ['realm', 'recordid', 'jobid', 'jobid', 'infoid', 'tsid'], - rest: { - warehouse: 'warehouse' - }, - - // DATA STORES ============================================================ - timeseriesstore: null, - memusedstore: null, - simdinsstore: null, - membwstore: null, - lnetstore: null, - ib_lnetstore: null, - accountdatastore: null, - acctstore: null, - mdatastore: null, - jobrecordstore: null, - store: null, - - /** - * Find the active sub tab under the job tabs - * @returns the component or false if non found - */ - getActiveJobSubPanel: function () { - var activeJobPanels = Ext.getCmp(this.tabpanel_id).getActiveTab().findByType('tabpanel'); - if (activeJobPanels.length < 1) { - return false; - } - return activeJobPanels[0].getActiveTab(); - }, - - /** - * This tabs constructor, here we take care to setup everything this tab - * will need throughout it's lifecycle. - */ - initComponent: function () { - // ROUTE: the unused toolbar events to a special no-opt function. - // Just to be sure that it doesn't go somewhere it's not - // supposed to. - this.on('role_selection_change', this.noOpt); - this.on('duration_change', this.noOpt); - - this.addEvents( - 'record_loaded', - 'data_account_loaded', - 'data_acct_loaded', - 'data_mdata_loaded', - 'data_jobrecord_loaded', - 'data_store_loaded' - ); - - // SETUP: the components for this tab and add them to this tabs 'items' - // property. - Ext.apply(this, { - items: this.setupComponents(), - customOrder: this.getToolbarConfig() - }); - - // Make sure to call the superclasses initComponent ( constructor ) - XDMoD.Module.JobViewer.superclass.initComponent.apply(this, arguments); - - this.loading = false; - - /* - * The timezone setting for highcharts is a global option that applies - * to all charts in the browser window. Individual charts in the job viewer - * can change the timezone as appropriate. The job viewer tab resets the - * timezone on deactivate to avoid impacting the highcharts plots in - * other tabs in the interface. The timezone settings are cached in the - * tab so that when it is activated again they are restored. - */ - this.cachedHighChartTimezone = null; - - }, // initComponent - - /** - * Apply implements an immutable merge (ie. returns a new object containing - * the properties of both objects.) of two javascript objects, - * lhs ( left hand side) and rhs ( right hand side). A property that exists - * in both lhs and rhs will default to the value of rhs. - * - * @param {object} lhs Left hand side of the merge. - * @param {object} rhs Right hand side of the merge. - * - **/ - apply: function (lhs, rhs) { - if (typeof lhs === 'object' && typeof rhs === 'object') { - var results = {}; - for (var property in lhs) { - if (lhs.hasOwnProperty(property)) { - results[property] = lhs[property]; - } - } - for (property in rhs) { - if (rhs.hasOwnProperty(property)) { - var rhsExists = rhs[property] !== undefined - && rhs[property] !== null; - if (rhsExists) { - results[property] = rhs[property]; - } - } - } - return results; - } - return lhs; - },// apply - - /** - * Helper function that handles setting up this components toolbar. - */ - getToolbarConfig: function () { - var self = this; - - var searchPanel = new XDMoD.Module.JobViewer.SearchPanel({ - id: 'job-viewer-search-panel', - jobViewer: this, - token: self.token - }); - - self.searchHistoryPanel.relayEvents(searchPanel, ['reload_root']); - self.searchHistoryPanel.relayEvents(this, ['reload_root']); - searchPanel.relayEvents(self, ['edit_search']); - - self.searchWindow = new Ext.Window({ - id: 'search-window', - closable: true, - closeAction: 'hide', - modal: true, - title: 'Search', - layout: 'fit', - resizable: true, - boxMaxHeight: 641, - boxMaxWidth: 1014, - items: [searchPanel] - }); - searchPanel.relayEvents(self.searchWindow, ['show', 'hide', 'move']); - - var searchButton = new Ext.Button({ - text: 'Search', - iconCls: 'search', - tooltip: 'Search for some subset of jobs', - handler: function (b) { - self.searchWindow.show(); - } - }); - - return [ - searchButton, - { - item: ' ', - separator: false - }, - XDMoD.ToolbarItem.EXPORT_MENU, - XDMoD.ToolbarItem.PRINT_BUTTON - ]; - }, - - /** - * Build the components that will make up this tabs UI. Return an array of the top level components for display in a - * 'border' layout ( note: one of the returned components *must* have 'region: center' as a property ). - * - * @returns {Array} of components to display as this tabs UI. - */ - setupComponents: function () { - var self = this; - - self.sortMode = 'age'; - - // SEARCH FORM ( CHILD OF BASIC SEARCH / NAVIGATION ) ================== - self.searchHistoryPanel = new XDMoD.Module.JobViewer.SearchHistoryPanel({ - id: 'jobviewer_search_history_panel', - jobViewer: this, - region: 'center', - listeners: { - data_loaded: function (data) { - self.treeLoaded = true; - if (self.historyEventWaiting) { - self.createHistoryCallback.call(self.createHistoryCallbackScope, self.createHistoryCallbackData); - } - } - } - }); - - self.treeSorter = new Ext.tree.TreeSorter(self.searchHistoryPanel, { - folderSort: false, - dir: "asc", - sortType: function(value, node) { - if (node.attributes.dtype == 'recordid') { - if (self.sortMode == 'age') { - return 9007199254740991 - parseInt(node.attributes.recordid, 10); - } else { - return node.attributes.text; - } - } else { - return node.attributes[node.attributes.dtype]; - } - } - }); - - // NAVIGATION ( PARENT WESTERN PANEL ) ================================= - var searchHistory = new Ext.Panel({ - region: 'west', - title: 'Search History', - collapsible: true, - collapsed: false, - layout: 'border', - collapseFirst: false, - pinned: false, - plugins: new Ext.ux.collapsedPanelTitlePlugin('Navigation'), - width: 250, - items: [ - self.searchHistoryPanel - ], - listeners: { - collapse: function (p) { - - }, - expand: function (p) { - if (p.pinned) { - p.getTool('pin').hide(); - p.getTool('unpin').show(); - } else { - p.getTool('pin').show(); - p.getTool('unpin').hide(); - } - } - }, - tools: [{ - id: 'pin', - qtip: 'Prevent auto hiding of the Search History', - hidden: false, - handler: function (ev, toolEl, p, tc) { - p.pinned = true; - if (p.collapsed) { - p.expand(false); - } - p.getTool('pin').hide(); - p.getTool('unpin').show(); - } - }, { - id: 'unpin', - qtip: 'Allow auto hiding of the Search History', - hidden: true, - handler: function (ev, toolEl, p, tc) { - p.pinned = false; - p.getTool('pin').show(); - p.getTool('unpin').hide(); - } - }] - }); // navbar ========================================================== - - var tabPanel = new Ext.TabPanel({ - id: 'info_display', - enableTabScroll: true, - region: 'center' - }); - - var assistPanel = new CCR.xdmod.ui.AssistPanel({ - region: 'center', - border: false, - headerText: 'No job is selected for viewing', - subHeaderText: 'Please refer to the instructions below:', - graphic: 'gui/images/data_export_instructions.png', - userManualRef: 'job+viewer' - }); - - var viewPanel = new Ext.Panel({ - id: 'info_display_container', - frame: false, - layout: 'card', - border: false, - activeItem: 0, - region: 'center', - items: [assistPanel, tabPanel] - }); - - // RETURN: an array of the top level components. To be used as the - // 'items' for another component with a border layout. - return new Array( - searchHistory,// WEST - viewPanel // CENTER - ); - }, // setupComponents - - /** - * Helper function that formats the provided val based on the provided units. - * - * @param val - * @param units - * @returns {*} - */ - formatData: function (val, units) { - switch (units) { - case "TODO": - case "": - case null: - return val; - break; - case "seconds": - return XDMoD.utils.format.humanTime(val); - break; - case "boolean": - return (val == 1) ? "True" : "False"; - break; - case "packets": - case "messages": - return Math.ceil(val); - break; - case "ratio": - case "1": - return XDMoD.utils.format.convertToSiPrefix(val, '', 3); - break; - case 'bytes': - case 'B': - case 'B/s': - return XDMoD.utils.format.convertToBinaryPrefix(val, units, 4); - break; - case 'kilobyte': - return XDMoD.utils.format.convertToBinaryPrefix(val * 1024.0, 'byte', 4); - break; - case 'megabyte': - return XDMoD.utils.format.convertToBinaryPrefix(val * 1024.0 * 1024.0, 'byte', 4); - break; - default: - return XDMoD.utils.format.convertToSiPrefix(val, units, 4); - break; - } - }, // formatData - - /** - * This is a marker function that is used to explicitly indicate that - * certain events are to take no action. - */ - noOpt: function () { - /** NO-OPT, what did you expect? **/ - }, // noOpt - - /** - * Helper function that retrieves the requested parameter from the provided - * source string via the provided name. - * - * @param name that will be used when looking for the the parameter in - * source. - * @param source that will be used to search for parameter 'name'. - * @returns {String} an empty string if not found, else the value of the - * parameter. - */ - getParameterByName: function (name, source) { - name = name.replace(/[\[]/, "\\[").replace(/[\]]/, "\\]"); - var regex = new RegExp("[\\?&]" + name + "=([^&#]*)"), - results = regex.exec(source); - return results === null - ? "" - : decodeURIComponent(results[1].replace(/\+/g, " ")); - }, // getParameterByName - - /** - * Generate a URL based on the provided base path. This URL will include - * the required XDMoD REST token. - * - * @param {String} base - * @param {Array} path - * @returns {String} - * @private - */ - _generateURL: function (base, path) { - - var isType = CCR.isType; - var encode = CCR.encode; - if (isType(base, CCR.Types.String) && isType(path, CCR.Types.Array)) { - var params = this._getParams(path); - var encoded = encode(params); - var result = base + '?' + encoded + '&token=' + XDMoD.REST.token; - return result; - } - return base; - }, // _generateURL - - /** - * Helper function that will generate a new view or informational tab based - * on the 'attributes.type' property. - * - * @param {Object} attributes - * @param {Array} path - * @param {String} title - * @param {Ext.tree.TreeNode} parent - * @return {*} - * @private - */ - _generateView: function (attributes, path, title, parent) { - var self = this; - var exists = CCR.exists; - - var dtype = attributes['dtype']; - var id = attributes[dtype]; - var jobId = attributes['jobid']; - - var uniqueId = [dtype, id].join('_'); - - var nestedId = [uniqueId, jobId, 'nested'].join('_'); - var textId = [uniqueId, jobId, 'text'].join('_'); - var kvId = [uniqueId, jobId, 'kv'].join('_'); - var kvStoreId = [uniqueId, jobId, 'kv', 'store'].join('_'); - var metricsId = [uniqueId, jobId, 'metrics'].join('_'); - var chartId = [uniqueId, jobId, 'chart'].join('_'); - - var valueColumnId = [uniqueId, 'column', 'value'].join('_'); - var nestedValueColumnId = [uniqueId, 'nested', 'column', 'value'].join('_'); - - var type = attributes.type; - var base = attributes.url; - - var tab; - var url = self._generateURL(base, path); - switch (type) { - case 'nested': - tab = new XDMoD.Module.JobViewer.NestedViewPanel({ - updateHistory: true, - preferWindowPath: true, - id: nestedId, - dtypes: [], - dtype: dtype, - dtypeValue: id, - jobId: jobId, - path: path, - title: title, - dataUrl: url, - autoExpandColumn: nestedValueColumnId, - columns: [ - {"header": "Key", "dataIndex": "key", mapping: "key", "width": 250}, - {"header": "Value", "dataIndex": "value", mapping: "value", id: nestedValueColumnId} - ] - - }); - break; - case 'utf8-text': - tab = new Ext.Panel({ - id: textId, - dtypes: [], - dtype: dtype, - dtypeValue: id, - jobId: jobId, - path: path, - closable: false, - updateHistory: true, - preferWindowPath: true, - title: title, - flex: 1, - layout: 'fit', - - items: [ - { - id: 'jobscript', - html: 'Loading', - autoScroll: true, - layout: 'fit', - url: url, - listeners: { - afterrender: function (panel) { - this._performLoad(this.url, panel); - } - }, - _performLoad: function (url, panel) { - Ext.Ajax.request({ - url: url, - success: function (response) { - if (response && response.responseText) { - var exists = CCR.exists; - - var json = JSON.parse(response.responseText); - var success = exists(json) && exists(json.success) && json.success; - var data = json.data || []; - var hasData = exists(data) && data.length > 0; - - if (success && hasData) { - panel.update('
' + json.data[0].value + '
'); - } else if (success && !hasData) { - panel.update('
 No Data Retrieved.
'); - } else if (!success) { - panel.update('
 An error occurred while attempting to perform the requested operation.
'); - } - } - } - }); - } - } - ] - - }); - break; - case 'metrics': - case 'keyvaluedata': - tab = new Ext.grid.GridPanel({ - dtypes: [], - title: title, - layout: 'fit', - dtype: dtype, - dtypeValue: id, - jobId: jobId, - path: path, - id: kvId, - closable: false, - updateHistory: true, - preferWindowPath: true, - height: '100%', - width: '100%', - store: new Ext.data.GroupingStore({ - id: kvStoreId, - url: url, - autoLoad: true, - proxy: new Ext.data.HttpProxy({ - api: { - read: { - url: url, - method: 'GET' - } - } - }), - reader: new Ext.data.JsonReader({ - root: 'data', - successProperty: 'success', - fields: [ - {name: 'key', mapping: 'key', type: 'string'}, - {name: 'value', mapping: 'value', type: 'string'}, - {name: 'full_value', mapping: 'value', type: 'string'}, - {name: 'units', mapping: 'units', type: 'string'}, - {name: 'group', mapping: 'group', type: 'string'}, - {name: 'help', mapping: 'help', type: 'string'}, - {name: 'documentation', mapping: 'documentation', type: 'string'} - ] - }), - listeners: { - /** - * Fires after the records have been loaded. - * - * @param {Ext.data.Store} store - * @param {Array} records - * @param {Object} options - */ - load: function (store, records, options) { - var exists = CCR.exists; - - for (var i = 0; i < records.length; i++) { - var record = records[i]; - - var value = record.get('value'); - var units = record.get('units'); - var fullValue = self.formatData(value, units, 4); - record.set('full_value', fullValue); - } - } - }, - groupField: 'group' - }), - columnLines: true, - flex: 2, - autoExpandColumn: valueColumnId, - columns: [ - { - "header": "Key", - "width": 250, - "sortable": true, - "dataIndex": "key", - renderer: function (value, metadata, record, rowIndex, colIndex, store) { - var help = record.get('documentation'); - metadata.attr = 'ext:qtip="' + help + '"'; - return value; - } - }, - { - "header": "Value", - "width": 150, - "sortable": true, - "dataIndex": "full_value", - id: valueColumnId, - renderer: {fn: self.renderFullValue, scope: self}, - editor: new Ext.form.TextField({ - allowBlank: false - }) - }, - { - "header": "Category", - "hidden": true, - "dataIndex": "group" - } - ], - view: new Ext.grid.GroupingView({ - forceFit:true, - groupTextTpl: '{text} ({[values.rs.length]} {[values.rs.length > 1 ? "Items" : "Item"]})' - }) - }); - break; - case 'detailedmetrics': - var renderValueWithUnits = function(value, node) { - if (typeof value === 'undefined') { - return ''; - } - if (node.unit) { - return self.formatData(value, node.unit); - } - return value; - }; - tab = new Ext.ux.tree.TreeGrid({ - id: metricsId, - dtypes: [], - dtype: dtype, - dtypeValue: id, - jobId: jobId, - path: path, - closable: false, - updateHistory: true, - preferWindowPath: true, - title: title, - autoScroll: true, - columnResize: false, - enableDD: true, - columns: [ - { - header: "Device", - dataIndex: "name", - width: 225, - tpl: new Ext.XTemplate('{name:this.render}', { - render: function(value, node) { - if (node.documentation) { - return '' + value + ''; - } - return value; - } - }) - }, - { - header: "Average", - dataIndex: "avg", - width: 125, - tpl: new Ext.XTemplate('{avg:this.render}', { - render: renderValueWithUnits - }) - }, - {header: "Count", dataIndex: "cnt", "width": 65}, - {header: "Standard Dev.", dataIndex: "std", "width": 125}, - { - header: "Median", - dataIndex: "med", - width: 125, - tpl: new Ext.XTemplate('{med:this.render}', { - render: renderValueWithUnits - }) - }, - {header: "Skew", dataIndex: "skw", "width": 125}, - { - header: "Minimum", - dataIndex: "min", - width: 125, - tpl: new Ext.XTemplate('{min:this.render}', { - render: renderValueWithUnits - }) - }, - { - header: "Maximum", - dataIndex: "max", - width: 125, - tpl: new Ext.XTemplate('{max:this.render}', { - render: renderValueWithUnits - }) - }, - {header: "Coefficient of variation", dataIndex: "cov", "width": 125}, - {header: "Kurtosis", dataIndex: "krt", "width": 125} - ], - dataUrl: url - }); - break; - case 'ganttchart': - tab = new XDMoD.Module.JobViewer.GanttChart({ - id: chartId, - title: title, - url: base, - baseParams: this._getParams(path), - historyToken: '#' + this.module_id + '?' + this._createHistoryTokenFromArray(path), - path: path, - dtypes: [], - dtype: dtype, - dtypeValue: id - }); - break; - case 'timeline': - case 'timeseries': - var tsid = this._find('tsid', 'tsid', path); - if (exists(tsid)) { - title = this._find('text', 'tsid', this.currentNode); - dtype = this._find('dtype', 'tsid', this.currentNode); - id = this._find(dtype, 'tsid', this.currentNode); - } - tab = new XDMoD.Module.JobViewer.ChartPanel({ - id: chartId, - dtypes: [], - dtype: dtype, - dtypeValue: id, - jobId: jobId, - path: path, - closable: false, - preferWindowPath: true, - updateHistory: true, - title: title, - layout: 'fit', - jobViewer: self, - jobTab: parent, - baseUrl: base, - store: new Ext.data.JsonStore({ - proxy: new Ext.data.HttpProxy({url: url}), - autoLoad: true, - root: 'data', - fields: ["series", "schema"] - }) - }); - break; - case 'analytics': - Ext.Ajax.request({ - dtype: dtype, - dtypeValue: id, - url: url, - method: 'GET', - success: function (response) { - var data = JSON.parse(response.responseText); - var success = exists(data) && exists(data.success) && data.success; - if (success) { - parent.fireEvent('update_analytics', data.data, true); - } - }, - failure: function (data) { - } - }); - break; - default: - break; - } - if (exists(tab)) { - tab.addListener('activate', self._panelActivation, self); - tab.helptext = { - title: tab.title, - documentation: attributes.documentation - }; - } - return tab; - }, // _generateView - - listeners: { - - /** - * Fired when this tab is either clicked on or activated. - * - * @param panel - **/ - activate: function () { - if (!this.loadMask) { - this.getExportMenu().setDisabled(true); - this.getPrintButton().setDisabled(true); - this.loadMask = new Ext.LoadMask(this.id); - } - Highcharts.setOptions({ global: { timezone: this.cachedHighChartTimezone } }); - - if (this.clearing) { - return; - } - - var token = CCR.tokenize(document.location.hash); - var params = Ext.urlDecode(token.params); - - if (params.job) { - this.loadMask.show(); - this.fireEvent('create_history_entry', Ext.decode(window.atob(params.job))); - return; - } - - if (!params.realm) { - return; - } - - if (params.action) { - this.loadMask.show(); - this.fireEvent('run_search_action', params); - return; - } - - this.loadMask.hide(); - - var selectionModel = this.searchHistoryPanel.getSelectionModel(); - - var path = this._getPath(token.raw); - var isSelected = this.compareNodePath(this.currentNode, path) && selectionModel && CCR.exists(selectionModel.getSelectedNode()); - - if (!isSelected) { - this.searchHistoryPanel.fireEvent('expand_node', path); - return; - } - - if (params.recordid && params.jobid) { - this.getExportMenu().setDisabled(!params.tsid); - this.getPrintButton().setDisabled(!params.tsid); - if (params.infoid) { - this.fireEvent('process_view_node', path); - } else { - this.fireEvent('process_job_node', path, this._processViewRequest); - } - } - }, // activate - - deactivate: function() { - this.cachedHighChartTimezone = Highcharts.getOptions().global.timezone; - Highcharts.setOptions({global: {timezone: null}}); - }, - - /** - * Takes care of clearing the whole informational display area. - * This includes the job tab panel and the analytics container. - * Also, since we just removed the whole display area, go ahead and - * reload the search history root. This will also ensure that our - * History Token is in sync with what the user is currently viewing, - * which is nothing. - */ - clear_display: function () { - this.clearing = true; - - var tabs = Ext.getCmp(this.tabpanel_id); - var analytics = Ext.getCmp(this.analyticsContainerId); - - // IF: the analytics is showing then hide it. - if (analytics && !analytics.hidden) analytics.hide(); - - // REMOVE: all of the tabs. - tabs.removeAll(); - - // FORCE: the container panel to re-lay itself out. - tabs.ownerCt.doLayout(false, true); - - this.fireEvent('reload_root'); - - this.clearing = false; - }, // clear_display - - export_option_selected: function (exportParams) { - var chartPanel = this.getActiveJobSubPanel(); - if (chartPanel) { - chartPanel.fireEvent('export_option_selected', exportParams); - } - }, - - print_clicked: function () { - var chartPanel = this.getActiveJobSubPanel(); - if (chartPanel) { - chartPanel.fireEvent('print_clicked'); - } - }, - - /** - * Process the given path for a job node history event. - * - * @param {Array} path - * @param {Boolean} isSelected - */ - process_job_node: function (path, callback) { - var self = this; - var exists = CCR.exists; - var isType = CCR.isType; - - var hasCurrentNode = exists(this.currentNode); - var hasAttributes = hasCurrentNode && exists(this.currentNode.attributes); - - if (!hasCurrentNode) { - console.log('No node... crying now ;-('); - } else if (hasCurrentNode && hasAttributes) { - - var tabs = Ext.getCmp(this.tabpanel_id); - - var jobId = this._find('jobid', 'jobid', path); - var title = this._find('text', 'jobid', this.currentNode); - - var jobPath = this._truncatePath('jobid', path); - - if (exists(jobId)) { - - jobId = parseInt(jobId); - - var found = tabs.find('jobId', jobId); - if (isType(found, CCR.Types.Array) && found.length > 0) { - var tab = found[0]; - tabs.activate(tab); - } else { - - var jobTab = new XDMoD.Module.JobViewer.JobPanel({ - itemId: 'jobid_' + jobId.toString(), - jobViewer: self, - jobId: jobId, - title: title, - path: jobPath, - listeners: { - beforeshow: function() { - // Set the tab panel to be active since a new tab is to be added. - Ext.getCmp('info_display_container').getLayout().setActiveItem(1); - }, - destroy: function () { - if (Ext.getCmp('info_display').items.length < 1) { - // All tabs have been destroyed. Set the assist image to be active - Ext.getCmp('info_display_container').getLayout().setActiveItem(0); - } - } - } - }); - - tabs.add(jobTab); - tabs.activate(jobTab); - - var base = this._find('url', 'jobid', this.currentNode) || this.searchHistoryPanel.url; - - var params = this._getParams(jobPath); - var encoded = CCR.encode(params); - var url = base + '?' + encoded + '&token=' + XDMoD.REST.token; - Ext.Ajax.request({ - url: url, - method: 'GET', - success: function (response) { - - var data = JSON.parse(response.responseText); - var success = exists(data) && exists(data.success) && data.success; - - if (success) { - var views = data.results; - - var jobTabs = jobTab.getComponent('job_tabs'); - - var tab; - for (var i = 0; i < views.length; i++) { - var view = views[i]; - view['jobid'] = jobId; - - var dtype = view['dtype']; - var id = view[dtype]; - - - var jobPath = self._copy(path, [], true); - jobPath[jobPath.length] = {dtype: dtype, value: id}; - - var currentViews = exists(view.text) ? jobTabs.find('title', view.text) : null; - var viewsExists = exists(currentViews) && exists(currentViews.length) && currentViews.length > 0; - - if (!viewsExists) { - tab = self._generateView(view, jobPath, view.text, jobTab); - if (exists(tab)) jobTabs.add(tab); - } - } - } - if (isType(callback, CCR.Types.Function)) callback.apply(self); - } - }) - } - } - - - } - }, // process_job_node - - /** - * Process the given path for the view_node history event. - * - * @param {Array} path - * @param {Boolean} isSelected - */ - process_view_node: function (path) { - var exists = CCR.exists; - var isType = CCR.isType; - var self = this; - - var hasCurrentNode = exists(this.currentNode); - var hasAttributes = hasCurrentNode && exists(this.currentNode.attributes); - - if (!hasCurrentNode) { - // TODO: write code to populate the currentNode. - console.log('No node... crying now :('); - } else if (hasCurrentNode && hasAttributes) { - - // RETRIEVE: The currentNodes attributes. - var attributes = this.currentNode.attributes; - - var jobId = this._find('jobid', 'jobid', this.currentNode); - jobId = isType(jobId, CCR.Types.Number) ? jobId : isType(jobId, CCR.Types.String) ? parseInt(jobId) : 0; - - var dType = attributes.dtype; - if (!attributes.jobid) attributes.jobid = jobId; - - // RETRIEVE: the tabs component - var tabs = Ext.getCmp(this.tabpanel_id); - - // RETRIEVE: The job tab if it exists. - var found = tabs.find('jobId', jobId); - var jobTab = isType(found, CCR.Types.Array) && found.length > 0 ? found[0] : null; - - /** - * Helper function that does the bulk of the work for - * processing a view node request. - */ - var processView = function () { - var toInt = CCR.toInt; - var currentPath = self._getPath(document.location.hash); - - var jobId = toInt(self._find('jobid', 'jobid', currentPath)); - var title = self._find('text', dType, self.currentNode); - - // RETRIEVE: the tabs component - var tabs = Ext.getCmp(self.tabpanel_id); - - // RETRIEVE: The job tab if it exists. - var jobTab = self._fromArray(tabs.find('jobId', jobId), 0); - - if (exists(jobTab)) { - - var currentlyActive = tabs.activeTab && tabs.activeTab.jobId - ? tabs.activeTab.jobId === jobTab.jobId - : false; - - if (!currentlyActive) { - jobTab.revert = false; - tabs.setActiveTab(jobTab); - return; - } - - // RETRIEVE: the tab panel that holds the informational tabs. - var infoTabs = jobTab.getComponent('job_tabs'); - - if (exists(infoTabs)) { - var isChild = self.children_ids.indexOf(dType) >= 0; - var infoDType = isChild ? self.child_to_parent[dType] : dType; - var infoValue = self._find(infoDType, infoDType, self.currentNode); - - var view = exists(title) ? infoTabs.findBy(function (component, container) { - var dTypes = component.dtypes; - var compDType = component.dtype; - var compValue = component.dtypeValue; - - return ((dTypes && dTypes.indexOf(infoDType) >= 0) || (compDType === infoDType)) && (infoValue === compValue); - }) : null; - var viewExists = exists(view) && exists(view.length) && view.length > 0; - - // CREATE: the view tab. and add it to the jobs' tabs. - if (!viewExists) { - - var tab = self._generateView(attributes, path, title, jobTab); - - // IF: we ended up with a tab being created then go ahead - // and add it and activate it. - if (exists(tab)) { - infoTabs.add(tab); - - infoTabs.activate(tab); - } - } else { - var toActivate = view[0]; - toActivate.dtypes[dType] = attributes[dType]; - - var reload = Ext.encode(toActivate.path) !== Ext.encode(currentPath); - - toActivate.path = currentPath; - toActivate.revert = true; - - infoTabs.setActiveTab(toActivate); - toActivate.fireEvent('activate', toActivate, reload); - - } - } - } - }; // processView - - - if (!exists(jobTab)) { - var jobPath = this._truncatePath('jobid', path); - this.fireEvent('process_job_node', jobPath, processView); - } else { - processView.apply(this); - } - } - }, // process_view_node - - /** - * Process a search deletion request by realm. - * - * @param realm of searches that is to be deleted. - */ - search_delete_by_realm: function (realm) { - var self = this; - var isType = CCR.isType; - var exists = CCR.exists; - if (isType(realm, CCR.Types.String)) { - Ext.MessageBox.confirm('Delete All Saved Searches?', 'Do you want to delete all saved searches for the realm: ' + realm + ' ?', - function (btn) { - if (btn === 'ok' || btn === 'yes') { - Ext.Ajax.request({ - /*'/rest/datawarehouse/search/history?realm=' + realm + '&token=' + XDMoD.REST.token,*/ - url: XDMoD.REST.url + '/' + self.rest.warehouse + '/search/history?realm=' + realm + '&token=' + XDMoD.REST.token, - method: 'DELETE', - success: function (response) { - var data = JSON.parse(response.responseText); - var success = exists(data) && exists(data.success) && data.success; - if (success) { - self.fireEvent('clear_display'); - var current = Ext.History.getToken(); - if (isType(current, CCR.Types.String)) { - var currentToken = CCR.tokenize(current); - var token = currentToken.tab + '?realm=' + realm; - Ext.History.add(token); - } else if (isType(current, CCR.Types.Object)) { - var token = current.tab + '?realm=' + realm; - Ext.History.add(token); - } - } else { - Ext.MessageBox.show({ - title: 'Deletion Error', - msg: 'There was an error removing all searches for the realm: [' + realm + '].', - icon: Ext.MessageBox.ERROR, - buttons: Ext.MessageBox.OK - }); - } - }, - failure: function (response) { - Ext.MessageBox.show({ - title: 'Deletion Error', - msg: 'There was an error removing all searches for the realm: [' + realm + '].', - icon: Ext.MessageBox.ERROR, - buttons: Ext.MessageBox.OK - }); - } - }); - } else { - Ext.MessageBox.alert('Ok', 'All your searches are belong to you.'); - } - }); - } - }, // search_delete_by_realm - - /** - * Process a search deletion request for a given node. - * - * @param {Ext.tree.TreeNode} node on which to process the deletion request. - */ - search_delete_by_node: function (node) { - var self = this; - var isType = CCR.isType; - var exists = CCR.exists; - var encode = CCR.encode; - if (isType(node, CCR.Types.Object)) { - var title = node.text || node.attributes ? node.attributes.text : undefined; - Ext.MessageBox.confirm('Delete All Saved Searches?', 'Do you want to delete the search: ' + title + ' ?', - function (text) { - if (text === 'ok' || text === 'yes') { - var recordId = node.attributes.recordid !== undefined ? node.attributes.recordid : null; - var fragment = recordId !== null ? '/search/history/' + recordId : '/search/history'; - var path = self._getPath(node); - /*'/rest/datawarehouse/search/history'*/ - var url = self._generateURL(XDMoD.REST.url + '/' + self.rest.warehouse + fragment, path); - Ext.Ajax.request({ - url: url, - method: 'DELETE', - success: function (response) { - var data = JSON.parse(response.responseText); - var success = exists(data) && exists(data.success) && data.success; - if (success) { - self.fireEvent('clear_display'); - var current = Ext.History.getToken(); - var path = self._getPath(node); - if (path && path.length && path.length > 0) delete path[path.length - 1]; - var params = self._getParams(path); - var encoded = encode(params); - if (isType(current, CCR.Types.String)) { - var currentToken = CCR.tokenize(current); - var token = currentToken.tab + '?' + encoded; - Ext.History.add(token); - } else if (isType(current, CCR.Types.Object)) { - var token = current.tab + '?' + encoded; - Ext.History.add(token); - } - } else { - Ext.MessageBox.show({ - title: 'Deletion Error', - msg: 'There was an error removing search: [' + title + '].', - icon: Ext.MessageBox.ERROR, - buttons: Ext.MessageBox.OK - }); - } - }, - failure: function (response) { - Ext.MessageBox.show({ - title: 'Deletion Error', - msg: 'There was an error removing search: [' + title + '].', - icon: Ext.MessageBox.ERROR, - buttons: Ext.MessageBox.OK - }); - } - }) - } - } - ); - - } - }, // search_delete_by_node - - - /** - * Attempt to create a history entry. This will queue the work to be - * done if the Search History Tree has no - * - * @param data - */ - create_history_entry: function (data) { - if (this.treeLoaded) { - this._createHistoryEntry(data); - } else { - this.historyEventWaiting = true; - this.createHistoryCallback = this._createHistoryEntry; - this.createHistoryCallbackData = data; - this.createHistoryCallbackScope = this; - } - }, - - /** - * Run a job search and save the first result in the search history - */ - run_search_action: function (searchparams) { - var self = this; - - Ext.Ajax.request({ - url: XDMoD.REST.url + '/' + this.rest.warehouse + '/search/jobs', - method: 'GET', - params: { - token: XDMoD.REST.token, - realm: searchparams.realm, - params: JSON.stringify(searchparams) - }, - success: function (response) { - var data = JSON.parse(response.responseText); - if (data.success === false || data.totalCount < 1) { - Ext.Msg.show({ - title: 'No results', - msg: 'No jobs were found that meet the requested search parameters.', - buttons: Ext.Msg.OK, - fn: Ext.History.add(self.module_id + '?realm=' + searchparams.realm), - icon: Ext.MessageBox.INFO - }); - return; - } - var historyEntry = { - title: searchparams.title || 'Linked Search', - realm: searchparams.realm, - text: data.results[0].text, - job_id: data.results[0].jobid, - local_job_id: data.results[0].local_job_id - }; - self.fireEvent('create_history_entry', historyEntry); - }, - failure: function (response) { - var message; - try { - var result = JSON.parse(response.responseText); - if (result.message) { - message = result.message; - } - } catch (e) { - message = 'Error processing request'; - } - - Ext.Msg.show({ - title: 'Error ' + response.status + ' ' + response.statusText, - msg: message, - buttons: Ext.Msg.OK, - fn: Ext.History.add(self.module_id + '?realm=' + searchparams.realm), - icon: Ext.MessageBox.ERROR - }); - } - }); - }, - - /** - * Update the sort order for the nodes in the search history tree. - * Does nothing if the current sort order is same as requested - * - * @param the requested sort order - */ - update_tree_sort: function(sortMode) { - if (this.sortMode != sortMode) { - this.sortMode = sortMode; - this.searchHistoryPanel.getRootNode().reload(function() { - this.currentNode = this.searchHistoryPanel.getRootNode(); - this.fireEvent('activate'); - }, this); - } - }, - - /** - * Process a node selected event. This results in the History Token - * ( document.location.hash ) being updated to reflect the node - * that is currently selected. Also ensure that the node selected is - * recorded as the current node. - * - * @param node that has been selected. - */ - node_selected: function (node) { - var exists = CCR.exists; - - this.currentNode = node; - - var token = this._createHistoryToken(node); - - var raw = Ext.History.getToken(); - var current = typeof raw === 'string' ? CCR.tokenize(raw) : raw; - - if (current.params === token) { - this.fireEvent('activate'); - } else { - if (exists(token)) Ext.History.add(this.module_id + '?' + token, false); - } - }, // node_selected - - }, // listeners =========================================================== - - /** - * Handles rendering the 'full_value' column of the 'keyvaluepair' grid. - * - * @param {Object} value - * @param {Object} metadata - * @param {Ext.data.Record} record - * @param {Number} rowIndex - * @param {Number} colIndex - * @param {Ext.data.Store} store - */ - renderFullValue: function (value, metadata, record, rowIndex, colIndex, store) { - return value; - }, // renderFullValue - - /** - * Create a history token from the provided `node`. - * - * @param {Ext.tree.TreeNode} node - * @returns {null|String} - * @private - */ - _createHistoryToken: function (node) { - var exists = CCR.exists; - if (exists(node) && exists(node.attributes)) { - - var results = []; - for (var next = node; next.parentNode !== null; next = next.parentNode) { - var key = next.attributes['dtype']; - var value = exists(key) ? next.attributes[key] : null; - if (exists(value)) results.push(key + '=' + value); - } - - return results.reverse().join('&'); - } - return null; - }, // _createHistoryToken - - /** - * Create a history token from the provided array of objects which provide - * minimally: - * { - * dtype: string, - * value: * - * } - * - * @param data - * @returns {*} - * @private - */ - _createHistoryTokenFromArray: function (data) { - var exists = CCR.exists; - if (exists(data) && exists(data.length) && data.length > 0) { - - var results = []; - for (var i = 0; i < data.length; i++) { - var next = data[i]; - var key = next['dtype']; - var value = next['value']; - if (exists(key) && exists(value)) results.push(key + '=' + value); - } - var reverse = results && results.length > 0 && results[0].indexOf('realm') < 0; - if (reverse) results.reverse(); - return results.join('&'); - } - return null; - }, // _createHistoryTokenFromArray - - /** - * Truncate the path at the first object that has the provided 'property'. - * - * @param {String} dtype - * @param {Array} path - * @returns {Array} - * @private - */ - _truncatePath: function (dtype, path) { - var isType = CCR.isType; - - if (!isType(dtype, CCR.Types.String)) return path; - if (!isType(path, CCR.Types.Array)) return path; - var results = []; - for (var i = 0; i < path.length; i++) { - var entry = path[i]; - if (entry.hasOwnProperty('dtype') && entry['dtype'] !== dtype) { - results.push(entry); - } else if (entry['dtype'] === dtype) { - results.push(entry); - break; - } - } - return results; - },// _truncatePath - - /** - * Attempt to find ( return ) the provided property of the provided node, - * or one of it's children, iff the node or child has the provided dtype. - * - * @param property to be found. - * @param dtype that will be used to qualify the node. - * @param node the root node to be used in the search. - * @returns {*} the value of the property, if found. - * @private - */ - _find: function (property, dtype, node) { - var isType = CCR.isType; - var exists = CCR.exists; - - if (!isType(property, CCR.Types.String)) return undefined; - if (!exists(node)) return undefined; - - var hasDtype = isType(dtype, CCR.Types.String); - var isObject = isType(node, CCR.Types.Object); - var isArray = isType(node, CCR.Types.Array); - - if (isObject) { - for (var current = node; exists(current); current = current.parentNode) { - var attributes = current.attributes || {}; - if (hasDtype && attributes.hasOwnProperty('dtype') && attributes.dtype === dtype && attributes.hasOwnProperty(property)) { - return attributes[property]; - } else if (!hasDtype && attributes.hasOwnProperty(property)) { - return attributes[property]; - } - } - } else if (isArray) { - for (var i = 0; i < node.length; i++) { - var attributes = node[i]; - if (hasDtype && attributes.hasOwnProperty('dtype') && attributes.dtype === dtype && attributes.hasOwnProperty('value')) { - return attributes['value']; - } else if (!hasDtype && attributes.hasOwnProperty(property)) { - return attributes[property]; - } - } - } - return null; - }, // _find - - /** - * Retrieve the 'path' values from the provided tree node or window hash. - * - * @param {String|Ext.tree.TreeNode} node - * @returns {Array} - * @private - */ - _getPath: function (node, keys) { - var exists = CCR.exists; - var isType = CCR.isType; - var results = []; - if (isType(node, CCR.Types.Object)) { - for (; exists(node); node = node.parentNode) { - var attributes = node.attributes || []; - var dtype = attributes['dtype']; - var value = exists(dtype) ? attributes[dtype] : undefined; - if (exists(value)) results.push({dtype: dtype, value: value}); - } - return results.reverse(); - } else if (isType(node, CCR.Types.String)) { - - var results = []; - - var token = CCR.tokenize(node); - var params = token && token.params && token.params.split ? token.params.split('&') : []; - for (var i = 0; i < params.length; i++) { - var param = params[i].split('='); - var key = param[0]; - var value = param[1]; - - var keyIndex = !exists(keys) || keys.indexOf(key); - if (keyIndex >= 0) { - results.push({dtype: key, value: value}); - } - } - return results; - } - }, // _getPath - - /** - * Accepts an Array of objects of the correct format and returns a query parameter style string. - * - * - * path should be provided in the following format: - * [ - * { dtype: string, value: string }, - * .... - * ] - * - * @param {Array} path - * @return {Object} in the form { dtype: value, dtype: value, ... } - * @private - */ - _getParams: function (path) { - var exists = CCR.exists; - - if (exists(path)) { - var results = {}; - for (var i = 0; i < path.length; i++) { - var part = path[i]; - var key = exists(part) && exists(part.dtype) ? part.dtype : null; - var value = exists(part) && exists(part.value) ? part.value : null; - if (exists(key) && exists(value)) results[key] = value; - } - return results; - } - return {} - }, //_getParams - - /** - * - * @param panel - * @private - */ - _panelActivation: function (panel) { - if (panel.updateHistory) this._updateHistoryFromPanel(panel); - }, // _panelActivation - - /** - * Attempt to update the History Token from a panel being activated. This - * captures the user scenario of clicking on an informational tab and not - * on a Search History Node. - * - * @param panel - * @private - */ - _updateHistoryFromPanel: function (panel) { - var token; - if (panel.path) { - token = '#' + this.module_id + '?' + this._createHistoryTokenFromArray(panel.path); - } else { - - var path = this._getPath(window.location.hash); - - var dtype = panel.dtype; - var value = panel.dtypeValue; - - path = this._truncatePath(dtype, path); - - var dtypeFound = false; - var dtypeValuesDiffer = true; - for (var i = 0; i < path.length; i++) { - var entry = path[i]; - if (entry.dtype === dtype) { - dtypeFound = true; - - dtypeValuesDiffer = entry.value != value; - entry.value = value; - break; - } - } - - if (!dtypeFound) { - path.push({ - dtype: dtype, - value: value - }); - token = '#' + this.module_id + '?' + this._createHistoryTokenFromArray(path); - } else if (dtypeValuesDiffer) { - token = '#' + this.module_id + '?' + this._createHistoryTokenFromArray(path); - } - } - Ext.History.add(token, true); - }, // _updateHistoryFromPanel - - /** - * Copy 'from' array -> 'to' array. If the append flag is set then it - * appends, else if overwrites. - * - * @param {Array} from - * @param {Array} to - * @param {Boolean} append if true then appends, else overwrites. - * @private - */ - _copy: function (from, to, append) { - var exists = CCR.exists; - var isType = CCR.isType; - - if (!isType(from, CCR.Types.Array)) throw new Error('Can only copy from arrays.'); - if (!isType(to, CCR.Types.Array)) return new Error('Can only copy to arrays.') - append = exists(append); - - for (var i = 0; i < from.length; i++) { - if (append) { - to.push(from[i]); - } else if (i <= to.length) { - to[i] = from[i]; - } - } - return to; - }, // _copy - - /** - * Attempt to process a request to show the currentNodes view. This occurs - * after a job_node has been processed ( expanded ). - * - * @private - */ - _processViewRequest: function () { - var isType = CCR.isType; - var exists = CCR.exists; - - var jobId = this._find('jobid', 'jobid', this.currentNode); - var title = this._find('text', 'jobid', this.currentNode); - - jobId = isType(jobId, CCR.Types.Number) ? jobId : isType(jobId, CCR.Types.String) ? parseInt(jobId) : 0; - - // RETRIEVE: the tabs component - var tabs = Ext.getCmp(this.tabpanel_id); - - // RETRIEVE: The job tab if it exists. - var found = tabs.find('jobId', jobId); - - var jobTab = isType(found, CCR.Types.Array) && found.length > 0 ? found[0] : null; - if (exists(jobTab)) { - - var currentlyActive = tabs.activeTab && tabs.activeTab.jobId - ? tabs.activeTab.jobId === jobTab.jobId - : false; - - if (!currentlyActive) { - tabs.setActiveTab(jobTab); - } - - var jobTabs = jobTab.getComponent('job_tabs'); - if (exists(jobTabs)) { - var hasTabs = jobTabs.items.length > 0; - if (hasTabs) { - jobTabs.activate(jobTabs.items.get(0)); - } - } - } - }, // _processViewRequest - - /** - * Replace the property found in the paths' array of objects - * with the provided value if found. - * - * @param {String} property - * @param {*} value - * @param {Array} path - * @private - */ - _replace: function (property, value, path) { - var isType = CCR.isType; - if (!isType(path, CCR.Types.Array)) return; - for (var i = 0; i < path.length; i++) { - var entry = path[i]; - if (entry.dtype === property) { - entry.value = value; - return; - } - } - return; - }, // _replace - - - /** - * Retrieve the value at the index provided from 'data'. This is done in a - * null-safe / index-safe way. - * - * @param {Array} data - * @param {Number} index - * @returns {*} - * @private - */ - _fromArray: function (data, index) { - var isType = CCR.isType; - if (!isType(data, CCR.Types.Array) || (!isType(index, CCR.Types.Number) & index < 0)) return undefined; - if (index >= data.length) return undefined; - - return data[index]; - }, // _fromArray - - /** - * Compare the given search history tree node with the provided path array. - * The path encoding from the _getPath function. - * - * @param Ext.tree.TreeNode node - * @param Array path - * @returns true if the path array matches the tree node, false otherwise - */ - compareNodePath: function (node, path) { - var i; - var np; - for (np = node, i = path.length - 1; np && np.attributes && np.attributes.dtype; np = np.parentNode, --i) { - if (i < 0) { - return false; - } - if (path[i].dtype !== np.attributes.dtype || path[i].value !== String(np.attributes[np.attributes.dtype])) { - return false; - } - } - return i === -1; - }, - - /** - * Attempt to create a history entry ( node located in the Search History - * Panel ) from the provided options. This occurs when interpreting the - * results of a History Token activation from Metric Explorer. - * - * @param options - * @private - */ - _createHistoryEntry: function (options) { - var self = this; - var searchTitle = options.title; - var realm = options.realm; - var jobTitle = options.text; - var jobId = options.job_id; - var jobLocalId = options.local_job_id; - - var jobData = { - resource: "", - name: "", - jobid: jobId, - text: jobTitle, - dtype: 'jobid', - local_job_id: jobLocalId - }; - - var searchPromise = this._makeRequest( - 'GET', - XDMoD.REST.url + '/' + this.rest.warehouse + '/search/history', - null, - { - realm: realm, - title: searchTitle, - token: XDMoD.REST.token - } - ); - - searchPromise.then(function (results) { - var data = results.data; - var recordId = data.recordid; - var jobs = data.results || [ - jobData - ]; - var jobFound = false; - for (var i = 0; i < jobs.length; i++) { - var job = jobs[i]; - if (job.jobid == jobId) { - jobFound = true; - } - } - if (!jobFound) jobs.push(jobData); - var upsertPromise = self._upsertSearch(realm, searchTitle, recordId, jobs, null); - upsertPromise.then(function (data) { - var realmNode = self.searchHistoryPanel.root.findChild('text', realm); - var path = self._getPath(realmNode); - recordId = recordId || data.results.recordid; - path.push({ - dtype: 'recordid', - value: recordId - }); - path.push({ - dtype: 'jobid', - value: jobId - }); - self.fireEvent('reload_root', path); - self.historyEventWaiting = false; - })['catch'](function (response) { - var message = response.msg || response.message || 'Updating the provided search.'; - CCR.error('Error', 'Unable to complete the requested operation: ' + message); - }); - })['catch'](function (response) { - // It wasn't found, so we need to create it. - var upsertPromise = self._upsertSearch(realm, searchTitle, null, [jobData], null); - upsertPromise.then(function (data) { - var realmNode = self.searchHistoryPanel.root.findChild('text', realm); - var path = self._getPath(realmNode); - var recordId = recordId || data.results.recordid; - path.push({ - dtype: 'recordid', - value: recordId - }); - path.push({ - dtype: 'jobid', - value: jobId - }); - self.fireEvent('reload_root', path); - self.historyEventWaiting = false; - }); - /* var message = response.msg || response.message || 'Retrieving search info.'; - CCR.error('Error', 'Unable to complete the requested operation: ' + message);*/ - }); - }, // _createHistoryEntry - - /** - * Attempts to execute an 'upsert' ( either an update or an insert depending - * on the context ) of a 'search' which is identified by the provided - * properties. Note that the results will be provided via a Promise so - * the caller will need to call: - * - * _upsertSearch(realm, title, id, jobs).then( function(results) { - * ... processing logic goes here ... - * }).catch( function(errorResponse) { - * ... error logic goes here ... - * }); - * - * @param {String} realm in which this search took place - * @param {String} title to give the search. - * @param {String} id optional, if provided, then this will be used as the 'recordid' - * which will indicate an update. - * @param {Object} jobs that should be associated with this search. - * @param {Array} searchTerms optional, that should be included with this search. - * @returns {*|Promise.} - * @private - */ - _upsertSearch: function (realm, title, id, jobs, searchTerms) { - - var url = XDMoD.REST.url + '/' + this.rest.warehouse + '/search/history?realm=' + realm + '&token=' + XDMoD.REST.token; - searchTerms = searchTerms || {}; - var params = { - 'data': JSON.stringify( - { - "text": title, - "searchterms": searchTerms, - "results": jobs - }) - }; - if (CCR.exists(id)) params['recordid'] = id; - - return this._makeRequest('POST', url, null, params); - }, // _upsertSearch - - /** - * Helper function that wraps making an AJAX call via Exts' Ext.Ajax.request - * method in a Promise. It also expects the response to be JSON and that it - * it will have a 'success' root property. This is for the proper handling - * of the resolve, reject methods. If the method returns anything other than - * a '200' status code then the reject ( catch ) method will be called. Also - * , if the root property 'success' does not exist or if it does exist but - * is false then the reject ( catch ) method will be called as well. In all - * other cases the resolve ( then ) method will be called. - * - * @param {String} method to use when making the request: GET|PUT|POST - * |DELETE|PATCH - * @param {String} url to use when making the request. - * @param {Object} data optional, will be included as the request options - * 'data' property. - * @param {Object} params optional, will be included as the request options - * 'params' property. - * @returns {Promise} that can be used to complete the requested request - * @private - */ - _makeRequest: function (method, url, data, params) { - return new RSVP.Promise(function (resolve, reject) { - var options = { - method: method, - url: url, - success: function (response) { - var data = JSON.parse(response.responseText); - var success = CCR.exists(data) && CCR.exists(data.success) && data.success; - if (success) { - resolve(data); - } else { - reject(response); - } - }, - failure: function (response) { - reject(response); - } - }; - - if (CCR.exists(data)) options['data'] = data; - if (CCR.exists(params)) options['params'] = params; - Ext.Ajax.request(options); - }); - } // _makeRequest -}) -; //XDMoD.Module.JobViewer From f2a00ae78f816eb333e0aa235417fba04bcce5fc Mon Sep 17 00:00:00 2001 From: Rudra Chakraborty Date: Wed, 3 Apr 2019 15:42:17 -0400 Subject: [PATCH 021/217] thing --- configuration/roles.json | 2 +- html/index.php | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/configuration/roles.json b/configuration/roles.json index 1d7ece559e..2924851608 100644 --- a/configuration/roles.json +++ b/configuration/roles.json @@ -60,7 +60,7 @@ { "name": "data_export", "title": "Data Export", - "position": 1100, + "position": 400, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", "userManualSectionName": "Data Export", diff --git a/html/index.php b/html/index.php index 0e1ded6b32..bb5e3c2e15 100644 --- a/html/index.php +++ b/html/index.php @@ -458,6 +458,7 @@ function ($item) { + From e73b55213618cd83725f4b3f338a87dc36d90891 Mon Sep 17 00:00:00 2001 From: Rudra Chakraborty Date: Fri, 5 Apr 2019 13:53:01 -0400 Subject: [PATCH 022/217] the stuffs --- html/gui/js/modules/DataExport.js | 341 +++++++++++++++--------------- 1 file changed, 166 insertions(+), 175 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 665283572b..93e1fb76d2 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -1,177 +1,168 @@ -XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { - - title: 'Data Export', // <-- rename this - - module_id: 'data_export', // <-- rename this (see the section on REPORT CHECKBOX for how to name this) - - usesToolbar: true, - - toolbarItems: { - - durationSelector: true, - exportMenu: true, - printButton: true, - reportCheckbox: true - - }, - +XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { + + title: 'Data Export', // <-- rename this + + module_id: 'data_export', // <-- rename this (see the section on REPORT CHECKBOX for how to name this) + + usesToolbar: false, + // ------------------------------------------------------------------ - + initComponent: function () { - - this.on('role_selection_change', function(config) { - - /* - - Fired upon selecting an entry from the Role menu - - 'config' represents the configuration of the selected menu entry - - The properties which will be of use are the following: - - config.text (the label of the selected item) - config.value (the internal value associated with the selected item) - - */ - - });//role_selection_change - - // ============================================== - - this.on('duration_change', function(config) { - - /* - - Fired upon any of the following actions: - - Selecting a duration preset - - Selecting an aggregation unit - - Pressing ENTER to confirm an updated (and valid) date in the Start or End field - - Pressing the Refresh button - - 'config' is an object with the following details: - - { - aggregation_unit: ______, - preset: _____, - start_date: YYYY-MM-DD, - end_date: YYYY-MM-DD - } - - */ - - });//duration_change - - // ============================================== - - this.on('export_option_selected', function (config){ - - /* - - Fired upon selecting an entry from the Export menu - - 'config' represents the export parameters associated with the selected - entry, and is an object with the following details: - - { - format: ____, - width: ____, - height: ____, - inline: ____, - scale: ____, - show_title: ____, - } - - */ - - });//export_option_selected - - // ============================================== - - this.on('print_clicked', function() { - - /* - - Fired upon clicking the Print button - - */ - - });//print_clicked - - // ============================================== - - /* - - REPORT CHECKBOX - - Setting the check state of the 'Available For Report' checkbox is handled manually: - - self.getReportCheckbox().storeChartArguments(chart_args, title, subtitle, start_date, end_date, included_in_report); - - When the above call is made, the included_in_report value (either 'y' or 'n') determines whether the 'Available For Report' - checkbox is checked or not. When the user manually checks the 'Available For Report' checkbox, the chart arguments cached - via the last call to storeChartArguments() will be used. - - It is important that the module_id set in the configuration to this XDMoD.PortalModule subclass be unique among all other - XDMoD.PortalModule subclasses. The reporting layer relies on distinct module_id(s) to function properly. - - NOTE: The value of module_id needs to match the base name of the corresponding controller used to serve up the chart data. - - e.g. If your module consults 'controllers/abc.php' to get its chart data, and you want to add that chart data to a report, - the module_id needs to be named 'abc' - - */ - - // ============================================== - - var mainArea = new Ext.Panel({ - - region: 'center', - html: 'This is my new module' - - });//mainArea - - // ============================================== - - var customToolbarComponent = new Ext.Button({ - - text: 'Custom' - - });//customToolbarComponent - - // ============================================== - - Ext.apply(this, { - - /* - - If custom components are to be placed in the toolbar, you can specify where they are to be placed, relative - to the components available to you by default. Simply define a 'customOrder' property, which is an array - representing the order in which the components are to be placed/arranged. - - customOrder: [ - - XDMoD.ToolbarItem.DURATION_SELECTOR, - XDMoD.ToolbarItem.EXPORT_MENU, - customToolbarComponent, - XDMoD.ToolbarItem.PRINT_BUTTON, - XDMoD.ToolbarItem.REPORT_CHECKBOX - - ], - - The top-down ordering of the components in the 'customOrder' array corresponds to the left-right ordering of - the components in the top toolbar. In this example, the 'Custom' button (specific to this module) is placed - to the right of the Export menu and to the left of the Print button. - - */ - - items: [ - mainArea - ] - - });//Ext.apply - - XDMoD.Module.DataExport.superclass.initComponent.apply(this, arguments); - - },//initComponent - - });//XDMoD.Module.DataExport \ No newline at end of file + + this.on('role_selection_change', function (config) { + + /* + + Fired upon selecting an entry from the Role menu + + 'config' represents the configuration of the selected menu entry + + The properties which will be of use are the following: + + config.text (the label of the selected item) + config.value (the internal value associated with the selected item) + + */ + + }); //role_selection_change + + // ============================================== + + this.on('duration_change', function (config) { + + /* + + Fired upon any of the following actions: + - Selecting a duration preset + - Selecting an aggregation unit + - Pressing ENTER to confirm an updated (and valid) date in the Start or End field + - Pressing the Refresh button + + 'config' is an object with the following details: + + { + aggregation_unit: ______, + preset: _____, + start_date: YYYY-MM-DD, + end_date: YYYY-MM-DD + } + + */ + + }); //duration_change + + // ============================================== + + this.on('export_option_selected', function (config) { + + /* + + Fired upon selecting an entry from the Export menu + + 'config' represents the export parameters associated with the selected + entry, and is an object with the following details: + + { + format: ____, + width: ____, + height: ____, + inline: ____, + scale: ____, + show_title: ____, + } + + */ + + }); //export_option_selected + + // ============================================== + + this.on('print_clicked', function () { + + /* + + Fired upon clicking the Print button + + */ + + }); //print_clicked + + // ============================================== + + /* + + REPORT CHECKBOX + + Setting the check state of the 'Available For Report' checkbox is handled manually: + + self.getReportCheckbox().storeChartArguments(chart_args, title, subtitle, start_date, end_date, included_in_report); + + When the above call is made, the included_in_report value (either 'y' or 'n') determines whether the 'Available For Report' + checkbox is checked or not. When the user manually checks the 'Available For Report' checkbox, the chart arguments cached + via the last call to storeChartArguments() will be used. + + It is important that the module_id set in the configuration to this XDMoD.PortalModule subclass be unique among all other + XDMoD.PortalModule subclasses. The reporting layer relies on distinct module_id(s) to function properly. + + NOTE: The value of module_id needs to match the base name of the corresponding controller used to serve up the chart data. + + e.g. If your module consults 'controllers/abc.php' to get its chart data, and you want to add that chart data to a report, + the module_id needs to be named 'abc' + + */ + + // ============================================== + + var mainArea = new Ext.Panel({ + + region: 'center', + html: 'This is my new module' + + }); //mainArea + + // ============================================== + + var customToolbarComponent = new Ext.Button({ + + text: 'Custom' + + }); //customToolbarComponent + + // ============================================== + + Ext.apply(this, { + + /* + + If custom components are to be placed in the toolbar, you can specify where they are to be placed, relative + to the components available to you by default. Simply define a 'customOrder' property, which is an array + representing the order in which the components are to be placed/arranged. + + customOrder: [ + + XDMoD.ToolbarItem.DURATION_SELECTOR, + XDMoD.ToolbarItem.EXPORT_MENU, + customToolbarComponent, + XDMoD.ToolbarItem.PRINT_BUTTON, + XDMoD.ToolbarItem.REPORT_CHECKBOX + + ], + + The top-down ordering of the components in the 'customOrder' array corresponds to the left-right ordering of + the components in the top toolbar. In this example, the 'Custom' button (specific to this module) is placed + to the right of the Export menu and to the left of the Print button. + + */ + + items: [ + mainArea + ] + + }); //Ext.apply + + XDMoD.Module.DataExport.superclass.initComponent.apply(this, arguments); + + }, //initComponent + +}); //XDMoD.Module.DataExport From b79b632c1b267316dafe192032f8d25d16ecafbe Mon Sep 17 00:00:00 2001 From: Rudra Chakraborty Date: Fri, 5 Apr 2019 15:20:45 -0400 Subject: [PATCH 023/217] things --- html/gui/js/modules/DataExport.js | 47 ++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 93e1fb76d2..679de33cd2 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -117,18 +117,54 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { var mainArea = new Ext.Panel({ region: 'center', - html: 'This is my new module' + html: 'Batch Data Warehouse Exporter' }); //mainArea // ============================================== - var customToolbarComponent = new Ext.Button({ + var downloadButton = new Ext.Button({ - text: 'Custom' + text: 'Download' }); //customToolbarComponent + var submitButton = new Ext.Button({ + + text: 'Submit' + + }); //customToolbarComponent + + var downloadOptions = new Ext.FormPanel({ + labelWidth: 75, // label settings here cascade unless overridden + frame: true, + title: 'Export Options', + bodyStyle: 'padding:5px 5px 0', + width: 350, + defaults: { + width: 230 + }, + defaultType: 'textfield', + + items: [{ + fieldLabel: 'Desired Realm', + name: 'first', + allowBlank: false + }, new Ext.form.DateField({ + fieldLabel: 'Start Date', + name: 'startDate', + }),new Ext.form.DateField({ + fieldLabel: 'End Date', + name: 'endDate', + })], + + buttons: [{ + text: 'Save' + }, { + text: 'Cancel' + }] + }); + // ============================================== Ext.apply(this, { @@ -156,7 +192,10 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { */ items: [ - mainArea + mainArea, + downloadOptions, + submitButton, + downloadButton ] }); //Ext.apply From 839922d1dc70d16fa2578dbe0f3cc4f8d7969bd9 Mon Sep 17 00:00:00 2001 From: Rudra Chakraborty Date: Fri, 5 Apr 2019 15:46:50 -0400 Subject: [PATCH 024/217] Thing --- html/gui/js/modules/DataExport.js | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 679de33cd2..aa90ab1f8d 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -117,16 +117,18 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { var mainArea = new Ext.Panel({ region: 'center', - html: 'Batch Data Warehouse Exporter' + html: 'Batch Data Warehouse Exporter', + /*items: [ + downloadOptions, + downloadButton + ]*/ }); //mainArea // ============================================== var downloadButton = new Ext.Button({ - - text: 'Download' - + text: 'Download', }); //customToolbarComponent var submitButton = new Ext.Button({ @@ -147,7 +149,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { defaultType: 'textfield', items: [{ - fieldLabel: 'Desired Realm', + fieldLabel: 'Realm', name: 'first', allowBlank: false }, new Ext.form.DateField({ @@ -156,13 +158,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { }),new Ext.form.DateField({ fieldLabel: 'End Date', name: 'endDate', - })], - - buttons: [{ - text: 'Save' - }, { - text: 'Cancel' - }] + }), submitButton], }); // ============================================== @@ -194,7 +190,6 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { items: [ mainArea, downloadOptions, - submitButton, downloadButton ] From a142e5afb553aa22aa853fe40f525e6349dbe92f Mon Sep 17 00:00:00 2001 From: Rudra Chakraborty Date: Wed, 10 Apr 2019 11:12:30 -0400 Subject: [PATCH 025/217] thing --- html/gui/js/modules/DataExport.js | 2141 ++++++++++++++++++++++++++--- 1 file changed, 1950 insertions(+), 191 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index aa90ab1f8d..7e404b9511 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -1,202 +1,1961 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { - + /** + * The default number of results to retrieve during paging operations. + * + * @var {Number} + */ + _DEFAULT_PAGE_SIZE: 24, title: 'Data Export', // <-- rename this - module_id: 'data_export', // <-- rename this (see the section on REPORT CHECKBOX for how to name this) - usesToolbar: false, - - // ------------------------------------------------------------------ - + _DEFAULT_SEARCH_NAME_SIZE: 2, + + /** + * Default constructor, just used to setup the events to be listened for, + * some sane defaults, as well as the child components. + * properties of note: + * searchStore: local Ext.data.ArrayStore() used to store the search + * terms before submitting as a 'search'. + * realmField: Combobox containing the realms available for selection. + * searchField: Combobox dependent on 'realmField' selection. Provides + * a selection of dimensions available for selection. + * valueField: Combobox dependent on 'searchField' selection. Provides + * a selection of dimension values available for selection. + **/ initComponent: function () { - - this.on('role_selection_change', function (config) { - - /* - - Fired upon selecting an entry from the Role menu - - 'config' represents the configuration of the selected menu entry - - The properties which will be of use are the following: - - config.text (the label of the selected item) - config.value (the internal value associated with the selected item) - - */ - - }); //role_selection_change - - // ============================================== - - this.on('duration_change', function (config) { - - /* - - Fired upon any of the following actions: - - Selecting a duration preset - - Selecting an aggregation unit - - Pressing ENTER to confirm an updated (and valid) date in the Start or End field - - Pressing the Refresh button - - 'config' is an object with the following details: - - { - aggregation_unit: ______, - preset: _____, - start_date: YYYY-MM-DD, - end_date: YYYY-MM-DD - } - - */ - - }); //duration_change - - // ============================================== - - this.on('export_option_selected', function (config) { - - /* - - Fired upon selecting an entry from the Export menu - - 'config' represents the export parameters associated with the selected - entry, and is an object with the following details: - - { - format: ____, - width: ____, - height: ____, - inline: ____, - scale: ____, - show_title: ____, - } - - */ - - }); //export_option_selected - - // ============================================== - - this.on('print_clicked', function () { - - /* - - Fired upon clicking the Print button - - */ - - }); //print_clicked - - // ============================================== - - /* - - REPORT CHECKBOX - - Setting the check state of the 'Available For Report' checkbox is handled manually: - - self.getReportCheckbox().storeChartArguments(chart_args, title, subtitle, start_date, end_date, included_in_report); - - When the above call is made, the included_in_report value (either 'y' or 'n') determines whether the 'Available For Report' - checkbox is checked or not. When the user manually checks the 'Available For Report' checkbox, the chart arguments cached - via the last call to storeChartArguments() will be used. - - It is important that the module_id set in the configuration to this XDMoD.PortalModule subclass be unique among all other - XDMoD.PortalModule subclasses. The reporting layer relies on distinct module_id(s) to function properly. - - NOTE: The value of module_id needs to match the base name of the corresponding controller used to serve up the chart data. - - e.g. If your module consults 'controllers/abc.php' to get its chart data, and you want to add that chart data to a report, - the module_id needs to be named 'abc' - - */ - - // ============================================== - - var mainArea = new Ext.Panel({ - - region: 'center', - html: 'Batch Data Warehouse Exporter', - /*items: [ - downloadOptions, - downloadButton - ]*/ - - }); //mainArea - - // ============================================== - - var downloadButton = new Ext.Button({ - text: 'Download', - }); //customToolbarComponent - - var submitButton = new Ext.Button({ - - text: 'Submit' - - }); //customToolbarComponent - - var downloadOptions = new Ext.FormPanel({ - labelWidth: 75, // label settings here cascade unless overridden - frame: true, - title: 'Export Options', - bodyStyle: 'padding:5px 5px 0', - width: 350, - defaults: { - width: 230 - }, - defaultType: 'textfield', - - items: [{ - fieldLabel: 'Realm', - name: 'first', - allowBlank: false - }, new Ext.form.DateField({ - fieldLabel: 'Start Date', - name: 'startDate', - }),new Ext.form.DateField({ - fieldLabel: 'End Date', - name: 'endDate', - }), submitButton], - }); - - // ============================================== - - Ext.apply(this, { - - /* - - If custom components are to be placed in the toolbar, you can specify where they are to be placed, relative - to the components available to you by default. Simply define a 'customOrder' property, which is an array - representing the order in which the components are to be placed/arranged. - - customOrder: [ - - XDMoD.ToolbarItem.DURATION_SELECTOR, - XDMoD.ToolbarItem.EXPORT_MENU, - customToolbarComponent, - XDMoD.ToolbarItem.PRINT_BUTTON, - XDMoD.ToolbarItem.REPORT_CHECKBOX - + var self = this; + + this.addEvents( + 'add_condition', + 'remove_condition', + 'perform_search', + 'cancel_search', + 'close_search', + 'reset_criteria', + 'realm_selected', + 'field_selected' + ); + + this.resultsStore = this._createResultsStore(); + + this.selected = {}; + this.children = []; + + this.editing = false; + + XDMoD.Module.DataExport.SearchPanel.superclass.initComponent.apply( + Ext.apply(this, { + layout: 'table', + width: 1000, + autoScroll: true, + border: false, + layoutConfig: { + columns: 2 + }, + style: 'background-color: #D0D0D0;', + defaults: { + frame: false, + border: false, + width: 500, + height: 300 + }, + items: self._getItems() + }), arguments + ); + + this.resetResults = function() { + self.resultsStore.loadData({results:[], totalCount: 0}, false); + self.selected = {}; + }; + + }, // initComponent + + listeners: { + + /** + * Fired when this component is first rendered to the page. We use it + * to setup event forwarding of the 'load' event from the search_results + * component to this components bottom toolbar. We do that here because + * it's only at this point that both components are available. + */ + render: function () { + var searchResults = Ext.getCmp('search_results'); + var bbar = searchResults ? searchResults.getBottomToolbar() : null; + if (bbar) { + bbar.relayEvents(this.resultsStore, ['load']); + } + }, // render + + /** + * This is the 'show' event that has been forwarded from the containing + * window. + * @param {Ext.Window} window + **/ + show: function (window) { + var body = Ext.getBody(); + var bodyBox = body.getBox(); + var bodyHeight = bodyBox.height; + var bodyWidth = bodyBox.width; + + var box = window.getBox(); + var x = box.x; + var y = box.y; + var height = box.height; + var width = box.width; + + var adjHeight = Math.min(bodyHeight, height); + var adjWidth = Math.min(bodyWidth, width); + var adjX = x < 0 ? 0 : x; + var adjY = y < 0 ? 0 : y; + + window.setHeight(adjHeight); + window.setWidth(adjWidth); + window.setPosition(adjX, adjY); + + this.shown = true; + if (!this.children) { + this.children = []; + } + + this.fireEvent('validate'); + this.fireEvent('validate_search_criteria'); + if (!this.editing) { + this.ownerCt.setTitle("Search"); + } + }, // show + + /** + * Forwarded 'move' event from this panel's Ext.Window parent. + * This will keep the window from being moved outside of the users + * visible space. + * + * @param {Ext.Window} window the window object that was moved. + * @param {Number} x new x coordinate of the window. + * @param {Number} y new y coordinate of the window. + **/ + move: function(window, x, y) { + var adjX = x < 0 ? 0 : x; + var adjY = y < 0 ? 0 : y; + + if ((adjX === 0 && adjX !== x) || + (adjY === 0 && adjY !== y)) { + window.setPosition(adjX, adjY); + } + }, + + /** + * Forwarded 'hide' event from this panel's Ext.Window parent. + * we capture it here so that we can clean up the UI and have + * it ready for its next use. + **/ + hide: function() { + this.fireEvent('close_search', this, false); + }, + + /** + * + * @param {String} realm + * @param {String} field + * @param {String} fieldDisplay + * @param {String} operator + * @param {String} value + * @param {String} valueId + */ + add_condition: function (realm, field, fieldDisplay, operator, value, valueId) { + var record = new this.searchStore.recordType({ + realm: realm, + field: field, + fieldDisplay: fieldDisplay, + operator: operator, + value: value, + valueId: valueId + }); + this.searchStore.add(record); + }, // add_condition + + /** + * Indicates that this component should remove the provided record from + * it's search store. + * + * @param {Ext.data.Record} record + */ + remove_condition: function (record) { + if (CCR.isType(record, CCR.Types.Object)) { + this.searchStore.remove(record); + } + }, // remove_condition + + /** + * Indicates that this component should cancel the currently active + * search. + * + * @param panel + */ + cancel_search: function (/*panel*/) { + this.ownerCt.hide(); + }, // cancel_search + + /** + * Indicates that the user has requested a search be performed. + * + * @param panel + */ + search_requested: function (panel, searchType, searchParams) { + var self = this; + if(!this.loadMask) { + this.loadMask = new Ext.LoadMask( + panel.getEl(), + { + id: 'job-viewer-search-mask', + store: this.resultsStore + }); + } + + panel.resetResults(); + + this.resultsStore.load({ + params: searchParams, + callback: function (records, options, success) { + + var resultsGrid, saveButton, text, params, prefix; + + if(!success) { + // Store load failure is handled by the exception listener + return; + } + + // Cache the search parameters in the store object for use if the + // search is saved (ExtJs does not provide a native call to do this). + self.resultsStore.searchParams = searchParams; + + resultsGrid = Ext.getCmp('search_results'); + + if(self.resultsStore.getTotalCount() === 0) { + resultsGrid.setDisabled(true); + resultsGrid.getEl().mask(searchType + ' returned zero jobs.', 'x-mask'); + } else { + resultsGrid.getEl().unmask(); + resultsGrid.setDisabled(false); + + if (searchType == 'Lookup') { + // Since the Quick Lookup search is designed to find an exact job, + // automatically select the jobs returned + self.resultsStore.each(function (record) { + record.set('included', true); + text = record.get('text'); + }); + var resource = self._getFieldDisplayValue(Ext.getCmp('basic-resource')); + var localJobId = self._getFieldDisplayValue(Ext.getCmp('basic-localjobid')); + params = [resource, localJobId]; + } else if (searchType == 'Search') { + params = [ + 'search', + self._formatDate(new Date()), + Math.floor((Math.random() * 1024) + 1) + ]; + } + + var searchNameField = Ext.getCmp('job-viewer-search-name'); + var searchName = searchNameField.getValue(); + + if (!searchName) { + if (!params) { + params = JSON.parse(searchParams.params); + } + searchNameField.setValue(self._generateDefaultName(params, prefix)); + searchNameField.focus(true); + } + + self.fireEvent('validate'); + } + } + }); + }, // search_requested + + basic_search_realm_selected: function (realm) { + var basicSearch = Ext.getCmp('job-viewer-search-lookup'); + basicSearch.disable(); + var jobidField = Ext.getCmp('basic-localjobid'); + jobidField.reset(); + var resource = Ext.getCmp('basic-resource'); + resource.store.load({ + params: { + realm: realm + } + }); + }, + + /** + * Event fired when a realm has been selected. + * + * @param realm + */ + realm_selected: function (realm) { + Ext.getCmp('search-add').disable(); + Ext.getCmp('job-viewer-search-search').disable(); + this.searchStore.removeAll(); + this.valueField.store.setBaseParam({ + realm: realm + }); + this.searchField.reset(); + this.searchField.store.load({ + params: { + realm: realm + } + }); + }, + + validate_search_criteria: function() { + var lookupValid, searchValid; + + lookupValid = Ext.getCmp('basic-resource').getValue().toString().length > 0 && + Ext.getCmp('basic-localjobid').getValue().toString().length > 0; + + Ext.getCmp('job-viewer-search-lookup').setDisabled(!lookupValid); + + searchValid = this.searchStore.getCount() > 0 && + Ext.getCmp('search_start_date').isValid() && + Ext.getCmp('search_end_date').isValid(); + + Ext.getCmp('job-viewer-search-search').setDisabled(!searchValid); + }, + + /** + * Indicates that the UI should be validated. If it is currently not in + * a valid state the the user should be notified. + * + */ + validate: function (options) { + if (this.shown === true) { + this._validateResults(options); + } + }, // validate + + /** + * Indicates that the provided field was selected and as such if the + * field has a child field ( field whose values depend on this fields + * value ) then update the child fields parameters, remove all current + * values and let the user pick from the possibly new values. + * + * @param {Object} field that has been selected. + */ + field_selected: function (field) { + var self = this; + if (CCR.exists(self.valueField) && CCR.exists(self.valueField.store)) { + self.valueField.store.proxy.setUrl(self.valueField.store.proxy.url + ('/' + field)); + + self.valueField.store.removeAll(); + self.valueField.setValue(null); + self._selectInitial('search-value'); + } + }, // field_selected + + /** + * Indicates that this component should reset search and value field + * components of the provided panel. + * + * @param panel + */ + reset_criteria: function (panel, all) { + if (CCR.exists(panel)) { + var field = CCR.exists(panel.searchField) ? panel.searchField : null; + var value = CCR.exists(panel.valueField) ? panel.valueField : null; + var add = Ext.getCmp('search-add'); + + if (CCR.exists(value) && CCR.exists(value.setValue)) { + value.setValue(null); + if (CCR.exists(add)) { + add.disable(); + } + } + + if (all) { + if (CCR.exists(field) && CCR.exists(field.setValue)) { + field.setValue(null); + } + } + } + }, // reset_criteria + + /** + * Indicates that the user wishes to close this component. We need to + * reset everything so that it's ready for use the next time the window + * is opened. + * + * @param panel + * @param reload + */ + close_search: function (panel, reload) { + if (CCR.exists(panel)) { + reload = reload || false; + + // HIDE: this window. + panel.ownerCt.hide(); + + // CHECK: if we should be reloading + if (reload) { + this.jobViewer.fireEvent('reload_root'); + this.jobViewer.fireEvent('activate'); + } + + panel.searchStore.removeAll(); + panel.resetResults(); + + var basicResource = Ext.getCmp('basic-resource'); + var basicJobNumber = Ext.getCmp('basic-localjobid'); + + if(basicResource) { + basicResource.setValue(''); + } + if(basicJobNumber) { + basicJobNumber.setValue(''); + } + + var resultsGrid = Ext.getCmp('search_results'); + if(resultsGrid) { + resultsGrid.getEl().unmask(); + resultsGrid.setDisabled(true); + } + + var searchField = Ext.getCmp('search-field'); + searchField.setValue(null); + var searchValue = Ext.getCmp('search-value'); + searchValue.setValue(null); + var searchNameField = Ext.getCmp('job-viewer-search-name'); + searchNameField.setValue(''); + searchNameField.clearInvalid(); + + this.shown = false; + this.editing = false; + + delete this.dtype; + delete this.dtypeId; + delete this.children; + } + }, // close_search + + /** + * An event that can be called when a user wishes to 'edit' an already + * existing search. It accepts the {Ext.tree.AsyncTreeNode} value from + * the SearchHistoryTree and then sets up the Search Panel for the + * type of Search that was performed. Making sure that all fo the user + * modifiable values are set per the node passed in. + * + * @param {Ext.tree.AsyncTreeNode} node + **/ + edit_search: function(node) { + this.editing = true; + var params = node.attributes; + + if (!node.attributes.searchterms.params) { + CCR.error('Error Viewing Search', 'Unable to view search, no data provided.'); + return false; + } + + this.title = params.text; + this.dtype = params.dtype; + this.dtypeId = params[this.dtype]; + this._retrieveSelected(node); + + this.ownerCt.setTitle("Editing Search: " + this.title); + Ext.getCmp('job-viewer-search-name').setValue(this.title); + + if (!this.ownerCt.isVisible()) { + this.ownerCt.show(); + } + + if (params.searchterms.params.start_date) { + this._editAdvancedSearch.call(this, params.searchterms.params); + } else { + this._editQuickSearch.call(this, params.searchterms.params); + } + + return true; + } // edit_search + }, // listeners + + /** + * A private helper function that determines whether or not the state of the + * results store is considered 'valid' such that the 'Save Results' button + * should be enabled. + * + * @param {Object} options + **/ + _validateResults: function(options) { + options = options || {}; + var validate = options.validate !== undefined ? options.validate : false; + if (this.resultsStore) { + var field = Ext.getCmp('job-viewer-search-name'); + + var searchNameIsValid = options.name && validate + ? field.validateValue(options.name) + : validate + ? field.isValid() + : true; + + var valid = this.resultsStore.getCount() > 0 && + this._getSelectedRecords().length > 0 && + searchNameIsValid; + + var saveResults = Ext.getCmp('job-viewer-search-save-results'); + var saveAs = Ext.getCmp('job-viewer-search-save-as'); + + this._toggle(valid, saveResults); + this._toggle(valid && this.editing === true, saveAs); + } + }, + + /** + * Selects the initial item for a component to the record identified by the index provided by the user. + * This method requires that the identified component have a function called 'getStore' that supports a function + * 'getCount' to work as intended. + * + * @param {String} id of the component that has a 'store' ( ie. Ext.form.ComboBox ) whose value you want to set. + * @param {null|Number} index of the initial record to set. Defaults to 0. + * @param {null|Function} callback to be executed when this function is complete. + * @private + */ + _selectInitial: function (id, index, callback) { + callback = callback || function () { + }; + + if (CCR.exists(id)) { + var field = Ext.getCmp(id); + var store = CCR.exists(field) && CCR.exists(field.getStore) ? field.getStore() : null; + var hasRecords = CCR.exists(store) && CCR.exists(store.getCount) ? store.getCount() > 0 : false; + + /** + * + * @param {Ext.data.ArrayStore} store + * @param {Ext.form.ComboBox} field + * @param {Function} callback + */ + var selectRecord = function (store, field, callback) { + if(index === undefined) { + return; + } + var record = store.getAt(index); + if (field.isExpanded()) { + field.select(index, true); + } else { + var getName = function (record) { + if (CCR.exists(record)) { + var properties = ['name', 'short_name']; + for (var property in properties) { + if(properties.hasOwnProperty(property)) { + var value = record.get(properties[property]); + if (CCR.exists(value)) { + return value; + } + } + } + } + return ""; + }; + var text = getName(record); + field.setValue(text); + field.selectedIndex = index; + } + field.fireEvent('select', field, record, index); + callback(record); + }; + + if (hasRecords) { + selectRecord(store, field, callback); + } else { + store.load({ + callback: function (/*records, options*/) { + selectRecord(store, field, callback); + } + }); + } + } + }, // select_initial + + /** + * Format the provided date value in a such a way that the ExtJS Date + * components can deal with it. + * + * @param {Date} value + * @returns {string} + * @private + */ + _formatDate: function (value) { + if (!CCR.isType(value, '[object Date]')) { + return String(value); + } + + var pad = CCR.pad; + var left = CCR.STR_PAD_LEFT; + + var year = value.getFullYear(); + var month = value.getMonth(); + var date = value.getDate(); + + var dates = [ + year, + pad(String(month + 1), 2, '0', left), + pad(String(date), 2, '0', left) + ]; + + return dates.join('-'); + }, // _formatDate + + /** + * Helper function that returns a new JsonStore that is suitable for use + * as this components result store. + * + * @returns {*} + * @private + */ + _createResultsStore: function () { + var self = this; + + return new Ext.data.JsonStore({ + id: 'results_store', + url: XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + '/search/jobs', + proxy: new Ext.data.HttpProxy({ + api: { + read: { + method: 'GET', + url : XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + '/search/jobs' + } + } + }), + root: 'results', + totalProperty: 'totalCount', + fields: [ + {name: 'dtype', mapping: 'dtype', type: 'string'}, + {name: 'jobid', mapping: 'jobid', type: 'int'}, + { name: 'local_job_id', mapping: 'local_job_id', type: 'string' }, + {name: 'name', mapping: 'name', type: 'string'}, + {name: 'realm', mapping: 'realm', type: 'string'}, + {name: 'resource', mapping: 'resource', type: 'string'}, + {name: 'text', mapping: 'text', type: 'string'}, + {name: 'included', mapping: 'included', type: 'bool', defaultValue: false}, + {name: 'username', mapping: 'username', type: 'string'} ], + listeners: { + + /** + * + * @param {Ext.data.Store} store + * @param {[]Ext.data.Record} records + * @param {Object} params + **/ + load: function(store, records /*, params*/) { + for ( var i = 0; i < records.length; i++) { + var record = records[i]; + var id = self._getId(record); + var checked = self.children.indexOf(id) >= 0; + self._handleIncludeRecord(record, checked); + } + self.fireEvent('validate'); + }, + + exception : function (proxy, type, action, options, response /*, arg*/) { + var data = JSON.parse(response.responseText); + var status = response.status; + var message = data.message || 'An error occurred while attempting to execute the requested operation. Response Code: [' + status + ']'; + Ext.MessageBox.show({ + title : 'Error: Performing Search', + msg : message, + icon : Ext.MessageBox.ERROR, + buttons: Ext.MessageBox.OK + }); + + } + } + }); + }, // _getResultsStore + + /** + * Returns an ArrayStore that is suitable for use as this components + * search criteria store. + * + * @returns {*} + * @private + */ + _getSearchStore: function () { + var self = this; + if (this.searchStore === null || this.searchStore === undefined) { + this.searchStore = new Ext.data.ArrayStore({ + id: 'search-grid-store', + proxy: new Ext.data.MemoryProxy(), + fields: ['realm', 'field', 'fieldDisplay', 'operator', 'value', 'valueId'], + listeners: { + remove: function(/*store, record, index*/) { + self.fireEvent('validate_search_criteria'); + }, + add: function(/*store, records, index*/) { + self.fireEvent('validate_search_criteria'); + } + } + }); + } + return this.searchStore; + }, // _getSearchStore + + /** + * Retrieve the value of the date field identified by the provided prefix. + * + * @param {String} prefix + * @returns {*} + * @private + */ + _getDateValue: function (prefix) { + if (!CCR.exists(prefix) || typeof prefix !== 'string' || prefix.length < 1) { + return null; + } + + var dateField = Ext.getCmp(prefix + '_date'); + + var date = dateField.getValue(); + + return date; + }, // _getDateValue + + /** + * Retrieve all of the currently 'selected' records across all pages. + * + * @returns {*} + * @private + */ + _getSelectedRecords: function () { + var included = this.resultsStore.query('included', true); + + var existingselections = {}; + for(var i = 0; i < included.items.length; i++) { + existingselections[ included.items[i].data[included.items[i].data.dtype] ] = 1; + } + + for (var key in this.selected) { + if (this.selected.hasOwnProperty(key) && !existingselections.hasOwnProperty(key)) { + included.add(key, this.selected[key]); + } + } + + return included; + }, // _getSelectedRecords + + /** + * Returns an array of objects that only contain a particular set of properties. + * + * @param values + * @returns {Array} + * @private + */ + _resultsToJSON: function (values) { + if (CCR.exists(values) && CCR.exists(values.length) && values.length > 0) { + var results = []; + + var attributes = ['resource', 'name', 'jobid', 'text', 'realm', 'dtype', 'local_job_id']; + for (var i = 0; i < values.length; i++) { + var value = values.get(i); + var temp = {}; + for (var j = 0; j < attributes.length; j++) { + var attribute = attributes[j]; + if (CCR.exists(attribute)) { + temp[attribute] = value.get(attribute); + } + } + results.push(temp); + } + return results; + } + return []; + }, // resultsToJSON + + /** + * Return an array of ExtJS components to be used as this components + * items property. + * + * @returns {*[]} + * @private + */ + _getItems: function () { + var self = this; + + var checkColumn = new Ext.grid.CheckColumn({ + header: 'Include', + dataIndex: 'included', + id: 'included', + width: 55, + checkchange: function (record, dataIndex, checked) { + self._handleIncludeRecord(record, checked); + self.fireEvent('validate'); + } + }); - The top-down ordering of the components in the 'customOrder' array corresponds to the left-right ordering of - the components in the top toolbar. In this example, the 'Custom' button (specific to this module) is placed - to the right of the Export menu and to the left of the Print button. - - */ - + return [ + { + xtype: 'panel', + layout: 'border', + height: 48, + width: 502, + colspan: 1, + border: true, + style: 'margin-left: -2px; margin-top: -2px; background-color: #D0D0D0;', + items: [ + { + xtype: 'fieldset', + region: 'center', + border: false, + items: [ + new XDMoD.RealTimeValidatingTextField({ + id: 'job-viewer-search-name', + region: 'center', + emptyText: 'Enter Search Name...', + width: 354, + allowBlank: false, + style: 'margin-right: 20px', + fieldLabel: 'Search Name', + tooltip: { + text: 'the name value for the current search', + xtype: 'quicktip' + } + }) + ] + } + ] + }, + { + title: "Results", + id: 'search_results_container', + xtype: "panel", + height: 582, + rowspan: 3, + layout: 'border', + listeners: { + activate: function() { + if (self.rendered) { + self.fireEvent('validate'); + } + } + }, + items: [ + { + xtype: 'editorgrid', + id: 'search_results', + region: 'center', + autoExpandColumn: 'name', + plugins: [checkColumn], + loadMask: false, + store: this.resultsStore, + disabled: true, + sm: new Ext.grid.RowSelectionModel({singleSelect:true}), + listeners: { + rowclick: function(searchgrid, rowIndex) { + var record = searchgrid.store.getAt(rowIndex); + var checked = !record.get('included'); + record.set('included', checked); + checkColumn.checkchange(record, rowIndex, checked); + } + }, + columns: [ + checkColumn, + { + header: 'Job Id', + dataIndex: 'local_job_id' + }, + { + header: 'Resource', + dataIndex: 'resource', + width: 140, + id: 'resource' + }, + { + header: 'Name', + dataIndex: 'name', + id: 'name', + width: 210 + } + ], + bbar: new Ext.PagingToolbar({ + pageSize: self._DEFAULT_PAGE_SIZE, + displayInfo: true, + displayMsg: 'Displaying jobs {0} - {1} of {2}', + emptyMsg: 'No jobs to display', + store: self.resultsStore, + listeners: { + load: function (store, records, options) { + this.onLoad(store, records, options); + }, + beforechange: function(bbar, params) { + var searchParams = bbar.store.searchParams; + for(var p in searchParams) { + if(searchParams.hasOwnProperty(p) && !params.hasOwnProperty(p)) { + params[p] = searchParams[p]; + } + } + } + } + }) + } + ] + }, + { + xtype: 'panel', + title: 'Quick Job Lookup', + tools: [{ + id: 'help', + qtip: 'Use the quick lookup form to search for a job based on its ID and the resource on which it ran.' + }], + layout: 'border', + height: 160, + items: [{ + xtype: 'fieldset', + region: 'center', + items: [{ + xtype: 'realmcombo', + id: 'basic-search-realm', + panel: self, + realmSelectEvent: 'basic_search_realm_selected' + }, + { + xtype: 'combo', + fieldLabel: 'Resource', + id: 'basic-resource', + emptyText: 'Select a Resource', + triggerAction: 'all', + selectOnFocus: true, + displayField: 'long_name', + valueField: 'id', + typeAhead: true, + mode: 'local', + forceSelection: true, + enableKeyEvents: true, + store: new Ext.data.JsonStore({ + proxy: new Ext.data.HttpProxy({ + url: XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + '/dimensions/resource', + method: 'GET' + }), + baseParams: { + realm: CCR.xdmod.ui.rawDataAllowedRealms[0], + token: self.token + }, + storeId: 'jobviewer-basicsearch-resource', + autoLoad: true, + root: 'results', + fields: [ + {name: 'id', type: 'string'}, + {name: 'short_name', type: 'string'}, + {name: 'long_name', type: 'string'} + ], + listeners: { + exception: function (proxy, type, action, options, response) { + CCR.xdmod.ui.presentFailureResponse(response, { + title: 'Job Viewer', + wrapperMessage: 'The Quick Job Lookup resource list failed to load.' + }); + } + } + }), + listeners: { + select: function( /* combo, record, index */ ) { + self.fireEvent('validate_search_criteria'); + }, + blur: function( /*combo, event*/ ) { + self.fireEvent('validate_search_criteria'); + } + } + }, + { + xtype: 'textfield', + fieldLabel: 'Job Number', + emptyText: 'Enter Job #', + id: 'basic-localjobid', + stripCharsRe: /(^\s+|\s+$)/g, + width: 200, + enableKeyEvents: true, + listeners: { + keyup: function( /*field, event*/ ) { + self.fireEvent('validate_search_criteria'); + }, + specialkey: function(field, event) { + if (event.getKey() === event.ENTER) { + self.fireEvent('validate_search_criteria'); + return false; + } + } + } + }], + buttons: [{ + xtype: 'button', + disabled: true, + text: 'Search', + id: 'job-viewer-search-lookup', + handler: function( /*button, event*/ ) { + var resourceField = Ext.getCmp('basic-resource'); + var localjobidField = Ext.getCmp('basic-localjobid'); + var realmField = Ext.getCmp('basic-search-realm'); + var params = { + realm: realmField.getValue(), + params: JSON.stringify({ + resource_id: resourceField.getValue(), + local_job_id: localjobidField.getValue() + }) + }; + self.fireEvent('search_requested', self, 'Lookup', params); + } + }] + }] + }, + { + xtype: 'panel', + id: 'job-viewer-advanced-search', + title: 'Advanced Search', + tools: [{ + id: 'help', + qtip: 'Use the advanced search form to search for jobs based on one or more filters and a date range.' + }], + height: 375, + layout: 'border', + items: [{ + region: 'center', + id: 'criteria_advanced', + xtype: 'fieldset', + labelWidth: 55, + items: [{ + xtype: 'datefield', + id: 'search_start_date', + format: 'Y-m-d', + name: 'start_date', + fieldLabel: 'Start', + enableKeyEvents: true, + submitValue: false, + update: false, + validator: function(/*val*/) { + return self._dateFieldValidator('startDateField', 'start date'); + }, + listeners: { + beforerender: function (field) { + field.setValue(self._getDefaultStartDate()); + }, + keyup: function (/*field, record, index*/) { + self.fireEvent('validate_search_criteria'); + }, + select: function (/*field, record, index*/) { + self.fireEvent('validate_search_criteria'); + } + } + }, + { + xtype: 'datefield', + id: 'search_end_date', + name: 'end_date', + format: 'Y-m-d', + fieldLabel: 'End', + submitValue: false, + enableKeyEvents: true, + update: false, + validator: function(/*val*/) { + return self._dateFieldValidator('endDateField', 'end date'); + }, + listeners: { + beforerender: function (field) { + field.setValue(self._getDefaultEndDate()); + }, + keyup: function (/*field, record, index*/) { + self.fireEvent('validate_search_criteria'); + }, + select: function (/*field, record, index*/) { + self.fireEvent('validate_search_criteria'); + } + } + }, + { + id: 'realm-field', + xtype: 'realmcombo', + panel: self, + realmSelectEvent: 'realm_selected' + }, + { + xtype: 'compositefield', + fieldLabel: 'Filter', + defaults: { + margins: '0 8 26 0' + }, + items: [ + + { + id: 'search-field', + xtype: 'uxgroupcombo', + emptyText: 'Select a Field...', + triggerAction: 'all', + selectOnFocus: true, + displayField: 'name', + width: 175, + valueField: 'id', + typeAhead: true, + mode: 'local', + forceSelection: true, + groupTextTpl: '{text}', + tpl: '
{name}
', + store: new Ext.data.GroupingStore({ + proxy: new Ext.data.HttpProxy({ + method: 'GET', + url: XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + '/dimensions' + }), + baseParams: { + token: self.token, + realm: CCR.xdmod.ui.rawDataAllowedRealms[0], + querygroup: 'tg_usage' + + }, + sortInfo: { + field: 'name', + direction: 'ASC' + }, + groupField: 'Category', + autoLoad: true, + storeId: 'dimensionResults', + reader: new Ext.data.JsonReader({ + root: 'results', + idParameter: 'id', + fields: ['id', 'name', 'Category', 'description'] + }), + listeners: { + exception: function (proxy, type, action, exception, response) { + switch (response.status) { + case 403: + case 500: + var details = Ext.decode(response.responseText); + Ext.Msg.alert("Error " + response.status + " " + response.statusText, details.message); + break; + case 401: + // Do nothing + break; + default: + Ext.Msg.alert(response.status + ' ' + response.statusText, response.responseText); + } + } + } + }), + listeners: { + select: function (field, record /*, index*/) { + if (CCR.exists(record)) { + var value = record.get('id'); + self.fireEvent('field_selected', value); + } + + }, + afterrender: function (field) { + if (!CCR.exists(self.searchField)) { + self.searchField = field; + } + } + } + }, + { + id: 'search-operator', + xtype: 'label', + text: '=', + style: 'margin-top: 4px; font-weight: bold', + listeners: { + afterrender: function (field) { + if (!CCR.exists(self.operatorField)) { + self.operatorField = field; + } + } + } + }, + { + id: 'search-value', + xtype: 'combo', + emptyText: 'Select a Value...', + triggerAction: 'all', + selectOnFocus: true, + displayField: 'long_name', + width: 170, + valueField: 'id', + typeAhead: true, + mode: 'local', + forceSelection: true, + enableKeyEvents: true, + store: new Ext.data.JsonStore({ + proxy: new Ext.data.HttpProxy({ + method: 'GET', + url: XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + '/dimensions' + }), + baseParams: { + token: self.token, + querygroup: 'tg_usage', + realm: CCR.xdmod.ui.rawDataAllowedRealms[0], + filter: 'true' + }, + storeId: 'valueStore', + idProperty: 'id', + root: 'results', + fields: [ + {name: 'id', type: 'string'}, + {name: 'name', type: 'string'}, + {name: 'short_name', type: 'string'}, + {name: 'long_name', type: 'string'} + ], + listeners: { + exception: function (proxy, type, action, exception, response) { + switch (response.status) { + case 403: + case 500: + var details = Ext.decode(response.responseText); + Ext.Msg.alert("Error " + response.status + " " + response.statusText, details.message); + break; + case 401: + // Do nothing + break; + default: + Ext.Msg.alert(response.status + ' ' + response.statusText, response.responseText); + } + } + } + }), + listeners: { + afterrender: function (field) { + if (!CCR.exists(self.valueField)) { + self.valueField = field; + } + }, + select: function (field, record /*, index*/) { + if (CCR.exists(record)) { + var addCmp = Ext.getCmp('search-add'); + if (addCmp) { + addCmp.enable(); + } + } + }, + keyup: function(field /*, event*/) { + var value = field.el.dom.value; + var addCriteriaCmp = Ext.getCmp('search-add'); + if (!CCR.exists(value)) { + addCriteriaCmp.disable(); + } else { + addCriteriaCmp.enable(); + } + } + } + }, + { + id: 'search-add', + xtype: 'button', + disabled: true, + text: 'Add', + handler: function (button /*, event*/ ) { + var realmField = Ext.getCmp('realm-field'); + var searchField = Ext.getCmp('search-field'); + var searchValue = Ext.getCmp('search-value'); + + var realm = realmField ? realmField.getValue() : null; + var fieldRecord = searchField ? searchField.store.getAt(searchField.selectedIndex) : null; + var field = fieldRecord ? fieldRecord.get('id') : null; + var fieldDisplay = fieldRecord ? fieldRecord.get('name') : null; + var operator = '='; + if (searchValue.selectedIndex >= 0) { + var searchRecord = searchValue.store.getAt(searchValue.selectedIndex); + var valueName = searchRecord.get('short_name'); + var value = searchRecord.get('id'); + + self.fireEvent('add_condition', realm, field, fieldDisplay, operator, valueName, value); + self.fireEvent('reset_criteria', self); + } else { + searchValue.setValue(null); + button.disable(); + } + } + } + ] + }, + { + xtype: 'panel', + layout: 'fit', + height: 200, + items: [ + { + xtype: 'grid', + id: 'job-viewer-search-criteria-grid', + region: 'center', + autoExpandColumn: 'value', + store: this._getSearchStore(), + columns: [ + { + id: 'field', + width: 200, + header: 'Field', + dataIndex: 'fieldDisplay' + }, + { + id: 'operator', + width: 57, + header: 'Operator', + dataIndex: 'operator' + }, + { + id: 'value', + width: 200, + header: 'Value', + dataIndex: 'value' + }, + { + xtype: 'actioncolumn', + width: 25, + items: [ + { + icon: '../../../gui/images/delete.png', + handler: function (grid, rowIndex /*, colIndex*/) { + var record = grid.store.getAt(rowIndex); + self.fireEvent('remove_condition', record); + } + } + ] + } + + ] + } + ] + + }], + buttons: [{ + xtype: 'button', + text: 'Search', + id: 'job-viewer-search-search', + disabled: true, + handler: function( /*button, event*/ ) { + var startDate = Ext.getCmp('search_start_date').getValue(); + var endDate = Ext.getCmp('search_end_date').getValue(); + var realmField = Ext.getCmp('realm-field'); + + var searchParams = {}; + var total = self.searchStore.getCount(); + for (var i = 0; i < total; i++) { + var searchParam = self.searchStore.getAt(i); + + var field = searchParam.get('field'); + var value = searchParam.get('valueId'); + + if (!searchParams[field]) { + searchParams[field] = []; + } + searchParams[field].push(value); + } + + var params = { + 'start_date': self._formatDate(startDate), + 'end_date': self._formatDate(endDate), + 'realm': realmField.getValue(), + 'limit': self._DEFAULT_PAGE_SIZE, + 'start': 0, + 'params': JSON.stringify(searchParams) + }; + self.fireEvent('search_requested', self, 'Search', params); + } + }] + }] + + }, + { + xtype: 'toolbar', + colspan: 2, + width: 1000, + height: 27, + minHeight: 27, items: [ - mainArea, - downloadOptions, - downloadButton + '->', + { + xtype: 'button', + id: 'job-viewer-search-save-as', + text: 'Save As', + enabled: false, + tooltip: { + text: 'Save the selected results under a new name', + xtype: 'quicktip' + }, + handler: function(button, event, defaultText) { + var formattedDate = self._formatDate(new Date()); + var extension = Math.floor((Math.random() * 1024) + 1); + Ext.MessageBox.prompt('Pick Search Name', 'Please select a name with which to identify this collection of Search Criteria and it\'s associated results.', function(button, text) { + if (button === 'ok') { + var hasText = CCR.exists(text) && text.length >= 3; + + if (!hasText) { + Ext.MessageBox.show({ + title: 'Invalid Name', + msg: 'You must provide a name with which to identify these results if you wish to save them.', + icon: Ext.MessageBox.ERROR, + buttons: Ext.MessageBox.OK + }); + return; + } + + self._upsertSearchRecord(text); + } + }, this, false, defaultText || 'search-' + formattedDate + '-' + extension); + } + }, + { + xtype: 'button', + id: 'job-viewer-search-save-results', + text: 'Save Results', + tooltip: { + text: 'Select jobs from the results pane above and click here to save the jobs in the search history.', + xtype: "quicktip" + }, + handler: function(button, event, defaultText) { + var title = Ext.getCmp('job-viewer-search-name').getValue(); + if(self.editing === true) { + title = title !== "" ? title : self.title; + self._upsertSearchRecord(title, self.dtypeId); + } else { + if (title) { + self._upsertSearchRecord(title); + } else { + var formattedDate = self._formatDate(new Date()); + var extension = Math.floor((Math.random() * 1024) + 1); + var title = defaultText || 'search-' + formattedDate + '-' + extension; + self._upsertSearchRecord(title); + } + } + } + }, + { + xtype: 'button', + id: 'job-viewer-search-cancel', + text: 'Cancel', + handler: function( /*button, event*/ ) { + self.fireEvent('close_search', self, false); + } + } ] - - }); //Ext.apply - - XDMoD.Module.DataExport.superclass.initComponent.apply(this, arguments); - - }, //initComponent + }]; + }, // getItems + + /** + * A helper function that takes care of either updating or inserting the current search + * record w/ the provided title. + * + * @param {String} title + * @param {String|null} id + **/ + _upsertSearchRecord: function(title, id) { + var self = this; + var selected = self._resultsToJSON(self._getSelectedRecords()); + + var params = { + text: Ext.util.Format.htmlEncode(title), + searchterms: { + params: self.resultsStore.searchParams + }, + results: selected + }; + + var realmField = Ext.getCmp('realm-field'); + var realm = realmField ? realmField.getValue() : null; + var idFragment = id !== undefined ? '/' + id : ''; + var url = XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + '/search/history' + idFragment + '?realm=' + realm + '&token=' + XDMoD.REST.token; + + Ext.Ajax.request({ + url: url, + method: 'POST', + params: { + 'data': JSON.stringify(params) + }, + success: function (response) { + var exists = CCR.exists; + + var data = JSON.parse(response.responseText); + var success = exists(data) && data.success; + if (success) { + + var search = CCR.isType(data.results, CCR.Types.Array) ? data.results[0] : data.results; + var dtype = search['dtype']; + var value = search[dtype]; + + var newToken = ['realm=' + realm, dtype + '=' + value].join('&'); + var current = Ext.History.getToken(); + var token = CCR.tokenize(current); + var tab = token && token.tab ? token.tab : self.jobViewer.id; + + Ext.History.add("#" + tab + '?' + newToken); + } + self.fireEvent('close_search', self, true); + }, + error: function (/*response*/) { + Ext.MessageBox.show({ + title: 'Save Error', + msg: 'There was an error saving search [' + title + '].', + icon: Ext.MessageBox.ERROR, + buttons: Ext.MessageBox.OK + }); + } + }); + }, + + /** + * + * @param {Ext.data.Record} record + * @param {boolean|null} checked + **/ + _handleIncludeRecord: function(record, checked) { + var self = this; + var dtype = record.get('dtype'); + var id = record.get(dtype); + var exists = CCR.exists(self.selected[id]); + + if (checked && !exists) { + self.selected[id] = record; + } + if ( !checked && exists) { + delete self.selected[id]; + } + + exists = CCR.exists(self.selected[id]); + if (checked && exists) { + record.set('included', true); + } + }, + + _dateFieldValidator: function (field_id, label) { + var validDates = { + startDateField: Date.parseDate(Ext.getCmp('search_start_date').getRawValue(), 'Y-m-d'), + endDateField: Date.parseDate(Ext.getCmp('search_end_date').getRawValue(), 'Y-m-d') + }; + if (validDates[field_id] === undefined) { + return 'Invalid ' + label + ' (must be of the form YYYY-MM-DD)'; + } + if (validDates.startDateField !== undefined && validDates.endDateField !== undefined) { + if (validDates.startDateField > validDates.endDateField) { + return 'Start date must be less than or equal to the end date'; + } + } + return true; + }, + + /** + * Load the Quick Job Lookup form with the provided searchTerms. + * + * @param {Object} searchTerms + */ + _editQuickSearch: function(searchTerms) { + var self = this; + + var params = JSON.parse(searchTerms.params); + + Ext.getCmp('basic-search-realm').setValue(searchTerms.realm); + Ext.getCmp('basic-localjobid').setValue(params.local_job_id); + + var resourceField = Ext.getCmp('basic-resource'); + resourceField.store.load({ + params: { + realm: searchTerms.realm + }, + callback: function () { + resourceField.setValue(params.resource_id); + self.fireEvent('validate_search_criteria'); + } + }); + }, + + /** + * Performs the steps necessary to get the SearchPanel ready for + * editing an 'Advanced Search'. + * + * @param {Object} searchTerms + */ + _editAdvancedSearch: function(searchTerms) { + var self = this; + + var startDateField = Ext.getCmp('search_start_date'); + var endDateField = Ext.getCmp('search_end_date'); + + var startDate = searchTerms.start_date; + var endDate = searchTerms.end_date; + + startDateField.setValue(startDate); + endDateField.setValue(endDate); + + var realmField = Ext.getCmp('realm-field'); + + var realm = searchTerms.realm; + realmField.setValue(searchTerms.realm); + + var clearSearchCriteria = function() { + var store = CCR.exists(self.searchStore) ? self.searchStore : null; + if (CCR.exists(store) && CCR.exists(store.removeAll)) { + store.removeAll(); + } + }; + + var addSearchCriteria = function() { + var value; + var i; + + var params = JSON.parse(searchTerms.params); + var advSearch = Ext.getCmp("job-viewer-advanced-search", {removeMask: true}); + var searchTermMask = new Ext.LoadMask(advSearch.el); + var shown = 0; + for (var field in params) { + if (params.hasOwnProperty(field)) { + for (i = 0; i < params[field].length; i++) { + + if (shown <= 0) { + searchTermMask.show(); + } + shown++; + + value = params[field][i]; + var displayPromises = [ + self._findFieldDisplay.call(self, field), + self._findFieldValueDisplay.call(self, field, value) + ]; + + /** + * Defines a partial function that will be supplied with + * the current values of field and value so that they + * will be in scope ( and hold the correct value ) + * when called by the promise. + * + * @param {String} field the id value of the field + * @param {String} value the id value of the value + * @param {Array} displayValues the return values + * from the promises. + * @returns {Function} to be executed after all of + * the display promises have + * resolved. + **/ + var partialThen = function(field, value, displayValues) { + return function(displayValues) { + var fieldDisplay = displayValues[0]; + var valueDisplay = displayValues[1]; + self.fireEvent('add_condition', realm, field, fieldDisplay, '=', valueDisplay, value); + if (--shown <= 0) { + searchTermMask.hide(); + } + }; + }; + RSVP + .all(displayPromises) + .then(partialThen(field, value)) + .catch(function(reason) { + CCR.error('Error retrieving criteria information', reason); + searchTermMask.hide(); + }); + } + } + } + }; + + clearSearchCriteria(); + addSearchCriteria(); + self.resetResults(); + }, + + /** + * Retrieves the value that should be displayed to the user for the provided + * 'field' value. + * + * @param {String} field + * @param {Function} callback + * + * @returns {RSVP.Promise} + */ + _findFieldDisplay: function(field) { + var self = this; + var url = XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + + '/dimensions/' + field + '/name?token=' + XDMoD.REST.token; + + return this._getPromise(url, ['results', 'name']); + }, + + /** + * Retrieves the string that should be displayed to the user for the provided + * 'field' and 'value'. + * + * @param {String} field + * @param {String} value + * + * @return {RSVP.Promise} + */ + _findFieldValueDisplay: function(field, value) { + var self = this; + var url = XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + + '/dimensions/' + field + + '/values/' + value + + '/name?token=' + XDMoD.REST.token; + + return this._getPromise(url, ['results', 'name']); + }, + + _getPromise: function(url, resolveProperties) { + return new RSVP.Promise( + function(resolve, reject){ + Ext.Ajax.request({ + url: url, + success: function(response) { + var data = JSON.parse(response.responseText); + var success = data.success; + if (success === true) { + var resolveValue = data; + for (var i = 0; i < resolveProperties.length; i++) { + var property = resolveProperties[i]; + resolveValue = resolveValue[property]; + } + resolve(resolveValue); + } + }, + failure: function(response) { + var data = JSON.parse(response.responseText); + var message = data.message || 'An unknown error has occured.'; + reject(message); + } + }); + } + ); + }, + + /** + * Attempt to retrieve the already 'selected' values from the provided + * node. A 'selected' value is defined as an entry in this nodes childNodes + * property that contains a 'jobid' attribute. These are then used to + * mark the returned results such that it represents the current search + * state. + * + * @param {Ext.tree.AsyncTreeNode} node + **/ + _retrieveSelected: function(node) { + var self = this; + + /** + * + * @param {Array} selected + * @param {Array} idProperties + * @returns {Array} suitable to set 'this.children' to. + **/ + var processSelected = function(selected, idProperties) { + var results = []; + for ( var i = 0; i < selected.length; i++) { + var item = selected[i]; + var id = item[idProperties[0]]; + for ( var j = 1; j < idProperties.length; j++) { + id = id[idProperties[j]]; + } + results.push(id); + } + return results; + }; + + if (node.childNodes && node.childNodes.length && node.childNodes.length > 0) { + this.children = processSelected(node.childNodes, ['attributes', 'jobid']); + } else { + var realm = this._getNodeValue(this._getParentNode(node, 'realm'), 'realm'); + realm = realm !== null ? realm : ''; + + var url = XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + + '/search/history/' + this.dtypeId + + '?realm=' + realm + + '&token=' + XDMoD.REST.token; + + this._getPromise(url, ['results']) + .then(function(results){ + self.children = processSelected(results, ['jobid']); + }); + } + }, + + /** + * Attempt to retrieve an 'id' property from the provided 'item'. An 'id' + * property is defined as one that can be found via first querying for + * a 'dtype' property. Then using that value to retrieve the 'id'. + * + * @param {Object} item the item to be used in retrieving the id property + * + * @return {null|*} returns null if no 'dtype' property can be found or + * if there is no 'get' method for the provided item. + * else it attempts to provide the value of + * item[item.dtype] or item.get(item.dtype) + **/ + _getId: function(item) { + if (item.dtype !== undefined) { + return item[item.dtype]; + } else if (item.get !== undefined) { + var dtype = item.get('dtype'); + return item.get(dtype); + } + return null; + }, + + _getTitle: function() { + return this.title; + }, + + /** + * If the condition evaluates to true then 'fn' is called with all of the + * arguments following 'elseFn'. If it evaluates to false then 'elseFn' + * is called with all of the arguments following 'elseFn'; + * + * @param {Boolean} condition + * @param {Function} fn + * @param {Function} elseFn + * + * @return {*} the return value of 'fn' if 'condition' is true else the + * return value of 'elseFn' + **/ + _onOrCondition: function(condition, fn, elseFn) { + var args = Array.prototype.slice.call(arguments, 3); + if (condition) { + return fn.apply(null, args); + } else { + return elseFn.apply(null, args); + } + }, + + /** + * Toggle the 'enabled' state of an {Ext.Component} based on the provided + * boolean 'condition'. + * + * @param {Boolean} condition a Boolean expression that will control + * whether the provided component is + * enabled or disabled. + * @param {Ext.Component} cmp the component to be enabled / disabled. + **/ + _toggle: function(condition, cmp) { + var enable = function(cmp) { + if (cmp && cmp.enable) { + cmp.enable(); + } + }; + + var disable = function(cmp) { + if (cmp && cmp.disable) { + cmp.disable(); + } + }; + + this._onOrCondition(condition, enable, disable, cmp); + }, + + /** + * Attempts to follow the 'parentNode' links of the provided 'child' + * node until a parent is found that contains an attribute that matches + * the provided 'dtype'. + * + * @param {Ext.tree.AsyncTreeNode} child the node with which to start the + * search. + * @param {String} dtype the attribute to use in qualifying + * the parentNode. + * + * @return {null|Ext.tree.AsyncTreeNode} returns null if no parent is found + * or it returns the node identified + * by the provided 'dtype' by + * following the parentNode links of + * the provided 'child'. + **/ + _getParentNode: function(child, dtype) { + var parent = child.parentNode; + if (parent) { + var attributes = parent.attributes || {}; + if (attributes[dtype] !== undefined) { + return parent; + } + return this._getParentNode(parent, dtype); + } + return null; + }, + + /** + * Attempts to retrieve the value for the provided key 'attribute' from the + * specified 'node'. + * + * @param {Ext.tree.AsyncTreeNode} node + * @param {attribute} attribute + * + * @return {null|*} returns null if the attribute is not found. Else it + * returns the attribute value. + **/ + _getNodeValue: function(node, attribute) { + if (node && node.attributes && node.attributes[attribute] !== undefined) { + return node.attributes[attribute]; + } + return null; + }, + + _getDefaultStartDate: function() { + var start = new Date(); + start.setDate(start.getDate() - 7); + return start; + }, + + _getDefaultEndDate: function() { + var now = new Date(); + now.setDate(now.getDate() - 1); + return now; + }, + + _getFieldDisplayValue: function(field) { + return field && field.store + ? field.store.getById(field.getValue()).get(field.displayField) + : field && field.getValue + ? field.getValue() + : null; + }, + + /** + * Attempt to generate a default search name with the given parameters. + * + * @param {Object|Array} params the parameters to use in generating a + * default name + * @param {String} [prefix=''] an optional parameter that will be + * pre-pended to the output. + * @param {String} [delim='-'] an optional parameter that will be + * used to delimit the param values + * + * @return {String} + **/ + _generateDefaultName: function(params, prefix, delim) { + var args = []; + prefix = prefix !== undefined ? prefix : ''; + delim = delim !== undefined ? delim : '-'; + + args.push(prefix); + if (typeof params === 'object') { + for ( var key in params) { + if (params.hasOwnProperty(key)) { + args.push(params[key]); + } + } + } else if (Array.isArray(params)) { + args = params; + } + return args + .filter(function(element, index, array) { + return element !== undefined && + element !== null && + element !== ''; + }) + .join(delim); + } + +}); + + +XDMoD.Module.DataExport.RealmCombo = Ext.extend(Ext.form.ComboBox, { + fieldLabel: 'Realm', + mode: 'local', + typeAhead: true, + triggerAction: 'all', + selectOnFocus: true, + forceSelection: true, + store: CCR.xdmod.ui.rawDataAllowedRealms, + value: CCR.xdmod.ui.rawDataAllowedRealms[0], + listeners: { + select: function (field) { + if (field.startValue !== field.getValue()) { + this.panel.fireEvent(this.realmSelectEvent, field.getValue()); + } + }, + blur: function (field) { + if (field.startValue !== field.getValue()) { + this.panel.fireEvent(this.realmSelectEvent, field.getValue()); + } + } + } }); //XDMoD.Module.DataExport From 975be8a5d1c96760b6fbbfcea64fe645ded2ba3c Mon Sep 17 00:00:00 2001 From: Rudra Chakraborty Date: Wed, 10 Apr 2019 15:40:29 -0400 Subject: [PATCH 026/217] thing --- html/gui/js/modules/DataExport.js | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 7e404b9511..719ed90c13 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -394,8 +394,8 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { // CHECK: if we should be reloading if (reload) { - this.jobViewer.fireEvent('reload_root'); - this.jobViewer.fireEvent('activate'); + this.dataExport.fireEvent('reload_root'); + this.dataExport.fireEvent('activate'); } panel.searchStore.removeAll(); @@ -613,12 +613,12 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { return new Ext.data.JsonStore({ id: 'results_store', - url: XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + '/search/jobs', + url: XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/search/jobs', proxy: new Ext.data.HttpProxy({ api: { read: { method: 'GET', - url : XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + '/search/jobs' + url : XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/search/jobs' } } }), @@ -927,7 +927,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { enableKeyEvents: true, store: new Ext.data.JsonStore({ proxy: new Ext.data.HttpProxy({ - url: XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + '/dimensions/resource', + url: XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/dimensions/resource', method: 'GET' }), baseParams: { @@ -1095,7 +1095,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { store: new Ext.data.GroupingStore({ proxy: new Ext.data.HttpProxy({ method: 'GET', - url: XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + '/dimensions' + url: XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/dimensions' }), baseParams: { token: self.token, @@ -1176,7 +1176,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { store: new Ext.data.JsonStore({ proxy: new Ext.data.HttpProxy({ method: 'GET', - url: XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + '/dimensions' + url: XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/dimensions' }), baseParams: { token: self.token, @@ -1450,7 +1450,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { var realmField = Ext.getCmp('realm-field'); var realm = realmField ? realmField.getValue() : null; var idFragment = id !== undefined ? '/' + id : ''; - var url = XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + '/search/history' + idFragment + '?realm=' + realm + '&token=' + XDMoD.REST.token; + var url = XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/search/history' + idFragment + '?realm=' + realm + '&token=' + XDMoD.REST.token; Ext.Ajax.request({ url: url, @@ -1472,7 +1472,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { var newToken = ['realm=' + realm, dtype + '=' + value].join('&'); var current = Ext.History.getToken(); var token = CCR.tokenize(current); - var tab = token && token.tab ? token.tab : self.jobViewer.id; + var tab = token && token.tab ? token.tab : self.dataExport.id; Ext.History.add("#" + tab + '?' + newToken); } @@ -1659,7 +1659,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { */ _findFieldDisplay: function(field) { var self = this; - var url = XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + + var url = XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/dimensions/' + field + '/name?token=' + XDMoD.REST.token; return this._getPromise(url, ['results', 'name']); @@ -1676,7 +1676,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { */ _findFieldValueDisplay: function(field, value) { var self = this; - var url = XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + + var url = XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/dimensions/' + field + '/values/' + value + '/name?token=' + XDMoD.REST.token; @@ -1748,7 +1748,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { var realm = this._getNodeValue(this._getParentNode(node, 'realm'), 'realm'); realm = realm !== null ? realm : ''; - var url = XDMoD.REST.url + '/' + self.jobViewer.rest.warehouse + + var url = XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/search/history/' + this.dtypeId + '?realm=' + realm + '&token=' + XDMoD.REST.token; From 39c4f5f821bcf5559bead28b9887de74bc59d8cc Mon Sep 17 00:00:00 2001 From: Rudra Chakraborty Date: Thu, 11 Apr 2019 09:40:33 -0400 Subject: [PATCH 027/217] thing --- html/gui/js/modules/DataExport.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 719ed90c13..be3d9e7b09 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -613,7 +613,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { return new Ext.data.JsonStore({ id: 'results_store', - url: XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/search/jobs', + url: 'infill', proxy: new Ext.data.HttpProxy({ api: { read: { @@ -667,7 +667,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { } } }); - }, // _getResultsStore + }, // _createResultsStore /** * Returns an ArrayStore that is suitable for use as this components @@ -927,7 +927,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { enableKeyEvents: true, store: new Ext.data.JsonStore({ proxy: new Ext.data.HttpProxy({ - url: XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/dimensions/resource', + url: 'infill', method: 'GET' }), baseParams: { @@ -1095,7 +1095,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { store: new Ext.data.GroupingStore({ proxy: new Ext.data.HttpProxy({ method: 'GET', - url: XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/dimensions' + url: 'infill', }), baseParams: { token: self.token, @@ -1176,7 +1176,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { store: new Ext.data.JsonStore({ proxy: new Ext.data.HttpProxy({ method: 'GET', - url: XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/dimensions' + url: 'infill', }), baseParams: { token: self.token, @@ -1450,10 +1450,10 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { var realmField = Ext.getCmp('realm-field'); var realm = realmField ? realmField.getValue() : null; var idFragment = id !== undefined ? '/' + id : ''; - var url = XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/search/history' + idFragment + '?realm=' + realm + '&token=' + XDMoD.REST.token; + //var url = XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/search/history' + idFragment + '?realm=' + realm + '&token=' + XDMoD.REST.token; Ext.Ajax.request({ - url: url, + url: 'infill', method: 'POST', params: { 'data': JSON.stringify(params) @@ -1688,7 +1688,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { return new RSVP.Promise( function(resolve, reject){ Ext.Ajax.request({ - url: url, + url: 'infill', success: function(response) { var data = JSON.parse(response.responseText); var success = data.success; From 70d36e366e3ae1f982fd5913ca592733ff3f85e5 Mon Sep 17 00:00:00 2001 From: Rudra Chakraborty Date: Thu, 11 Apr 2019 12:40:17 -0400 Subject: [PATCH 028/217] thing --- html/gui/js/modules/DataExport.js | 1265 +++++++++++++++-------------- 1 file changed, 659 insertions(+), 606 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index be3d9e7b09..1109ffb384 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -1,5 +1,5 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { - /** + /** * The default number of results to retrieve during paging operations. * * @var {Number} @@ -26,14 +26,14 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { var self = this; this.addEvents( - 'add_condition', - 'remove_condition', - 'perform_search', - 'cancel_search', - 'close_search', - 'reset_criteria', - 'realm_selected', - 'field_selected' + 'add_condition', + 'remove_condition', + 'perform_search', + 'cancel_search', + 'close_search', + 'reset_criteria', + 'realm_selected', + 'field_selected' ); this.resultsStore = this._createResultsStore(); @@ -45,26 +45,29 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { XDMoD.Module.DataExport.SearchPanel.superclass.initComponent.apply( Ext.apply(this, { - layout: 'table', - width: 1000, + layout: 'table', + width: 1000, autoScroll: true, + border: false, + layoutConfig: { + columns: 2 + }, + style: 'background-color: #D0D0D0;', + defaults: { + frame: false, border: false, - layoutConfig: { - columns: 2 - }, - style: 'background-color: #D0D0D0;', - defaults: { - frame: false, - border: false, - width: 500, - height: 300 - }, - items: self._getItems() - }), arguments + width: 500, + height: 300 + }, + items: self._getItems() + }), arguments ); - this.resetResults = function() { - self.resultsStore.loadData({results:[], totalCount: 0}, false); + this.resetResults = function () { + self.resultsStore.loadData({ + results: [], + totalCount: 0 + }, false); self.selected = {}; }; @@ -133,7 +136,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * @param {Number} x new x coordinate of the window. * @param {Number} y new y coordinate of the window. **/ - move: function(window, x, y) { + move: function (window, x, y) { var adjX = x < 0 ? 0 : x; var adjY = y < 0 ? 0 : y; @@ -148,7 +151,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * we capture it here so that we can clean up the UI and have * it ready for its next use. **/ - hide: function() { + hide: function () { this.fireEvent('close_search', this, false); }, @@ -191,7 +194,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * * @param panel */ - cancel_search: function (/*panel*/) { + cancel_search: function ( /*panel*/ ) { this.ownerCt.hide(); }, // cancel_search @@ -202,10 +205,9 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { */ search_requested: function (panel, searchType, searchParams) { var self = this; - if(!this.loadMask) { + if (!this.loadMask) { this.loadMask = new Ext.LoadMask( - panel.getEl(), - { + panel.getEl(), { id: 'job-viewer-search-mask', store: this.resultsStore }); @@ -219,7 +221,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { var resultsGrid, saveButton, text, params, prefix; - if(!success) { + if (!success) { // Store load failure is handled by the exception listener return; } @@ -230,7 +232,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { resultsGrid = Ext.getCmp('search_results'); - if(self.resultsStore.getTotalCount() === 0) { + if (self.resultsStore.getTotalCount() === 0) { resultsGrid.setDisabled(true); resultsGrid.getEl().mask(searchType + ' returned zero jobs.', 'x-mask'); } else { @@ -305,7 +307,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { }); }, - validate_search_criteria: function() { + validate_search_criteria: function () { var lookupValid, searchValid; lookupValid = Ext.getCmp('basic-resource').getValue().toString().length > 0 && @@ -404,15 +406,15 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { var basicResource = Ext.getCmp('basic-resource'); var basicJobNumber = Ext.getCmp('basic-localjobid'); - if(basicResource) { + if (basicResource) { basicResource.setValue(''); } - if(basicJobNumber) { + if (basicJobNumber) { basicJobNumber.setValue(''); } var resultsGrid = Ext.getCmp('search_results'); - if(resultsGrid) { + if (resultsGrid) { resultsGrid.getEl().unmask(); resultsGrid.setDisabled(true); } @@ -443,7 +445,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * * @param {Ext.tree.AsyncTreeNode} node **/ - edit_search: function(node) { + edit_search: function (node) { this.editing = true; var params = node.attributes; @@ -481,21 +483,21 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * * @param {Object} options **/ - _validateResults: function(options) { + _validateResults: function (options) { options = options || {}; var validate = options.validate !== undefined ? options.validate : false; if (this.resultsStore) { var field = Ext.getCmp('job-viewer-search-name'); - var searchNameIsValid = options.name && validate - ? field.validateValue(options.name) - : validate - ? field.isValid() - : true; + var searchNameIsValid = options.name && validate ? + field.validateValue(options.name) : + validate ? + field.isValid() : + true; var valid = this.resultsStore.getCount() > 0 && - this._getSelectedRecords().length > 0 && - searchNameIsValid; + this._getSelectedRecords().length > 0 && + searchNameIsValid; var saveResults = Ext.getCmp('job-viewer-search-save-results'); var saveAs = Ext.getCmp('job-viewer-search-save-as'); @@ -516,8 +518,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * @private */ _selectInitial: function (id, index, callback) { - callback = callback || function () { - }; + callback = callback || function () {}; if (CCR.exists(id)) { var field = Ext.getCmp(id); @@ -531,7 +532,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * @param {Function} callback */ var selectRecord = function (store, field, callback) { - if(index === undefined) { + if (index === undefined) { return; } var record = store.getAt(index); @@ -542,7 +543,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { if (CCR.exists(record)) { var properties = ['name', 'short_name']; for (var property in properties) { - if(properties.hasOwnProperty(property)) { + if (properties.hasOwnProperty(property)) { var value = record.get(properties[property]); if (CCR.exists(value)) { return value; @@ -564,7 +565,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { selectRecord(store, field, callback); } else { store.load({ - callback: function (/*records, options*/) { + callback: function ( /*records, options*/ ) { selectRecord(store, field, callback); } }); @@ -618,22 +619,58 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { api: { read: { method: 'GET', - url : XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/search/jobs' + url: 'infill', } } }), root: 'results', totalProperty: 'totalCount', - fields: [ - {name: 'dtype', mapping: 'dtype', type: 'string'}, - {name: 'jobid', mapping: 'jobid', type: 'int'}, - { name: 'local_job_id', mapping: 'local_job_id', type: 'string' }, - {name: 'name', mapping: 'name', type: 'string'}, - {name: 'realm', mapping: 'realm', type: 'string'}, - {name: 'resource', mapping: 'resource', type: 'string'}, - {name: 'text', mapping: 'text', type: 'string'}, - {name: 'included', mapping: 'included', type: 'bool', defaultValue: false}, - {name: 'username', mapping: 'username', type: 'string'} + fields: [{ + name: 'dtype', + mapping: 'dtype', + type: 'string' + }, + { + name: 'jobid', + mapping: 'jobid', + type: 'int' + }, + { + name: 'local_job_id', + mapping: 'local_job_id', + type: 'string' + }, + { + name: 'name', + mapping: 'name', + type: 'string' + }, + { + name: 'realm', + mapping: 'realm', + type: 'string' + }, + { + name: 'resource', + mapping: 'resource', + type: 'string' + }, + { + name: 'text', + mapping: 'text', + type: 'string' + }, + { + name: 'included', + mapping: 'included', + type: 'bool', + defaultValue: false + }, + { + name: 'username', + mapping: 'username', + type: 'string' + } ], listeners: { @@ -643,8 +680,8 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * @param {[]Ext.data.Record} records * @param {Object} params **/ - load: function(store, records /*, params*/) { - for ( var i = 0; i < records.length; i++) { + load: function (store, records /*, params*/ ) { + for (var i = 0; i < records.length; i++) { var record = records[i]; var id = self._getId(record); var checked = self.children.indexOf(id) >= 0; @@ -653,14 +690,14 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { self.fireEvent('validate'); }, - exception : function (proxy, type, action, options, response /*, arg*/) { + exception: function (proxy, type, action, options, response /*, arg*/ ) { var data = JSON.parse(response.responseText); var status = response.status; var message = data.message || 'An error occurred while attempting to execute the requested operation. Response Code: [' + status + ']'; Ext.MessageBox.show({ - title : 'Error: Performing Search', - msg : message, - icon : Ext.MessageBox.ERROR, + title: 'Error: Performing Search', + msg: message, + icon: Ext.MessageBox.ERROR, buttons: Ext.MessageBox.OK }); @@ -684,10 +721,10 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { proxy: new Ext.data.MemoryProxy(), fields: ['realm', 'field', 'fieldDisplay', 'operator', 'value', 'valueId'], listeners: { - remove: function(/*store, record, index*/) { + remove: function ( /*store, record, index*/ ) { self.fireEvent('validate_search_criteria'); }, - add: function(/*store, records, index*/) { + add: function ( /*store, records, index*/ ) { self.fireEvent('validate_search_criteria'); } } @@ -725,8 +762,8 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { var included = this.resultsStore.query('included', true); var existingselections = {}; - for(var i = 0; i < included.items.length; i++) { - existingselections[ included.items[i].data[included.items[i].data.dtype] ] = 1; + for (var i = 0; i < included.items.length; i++) { + existingselections[included.items[i].data[included.items[i].data.dtype]] = 1; } for (var key in this.selected) { @@ -787,8 +824,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { } }); - return [ - { + return [{ xtype: 'panel', layout: 'border', height: 48, @@ -796,28 +832,26 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { colspan: 1, border: true, style: 'margin-left: -2px; margin-top: -2px; background-color: #D0D0D0;', - items: [ - { - xtype: 'fieldset', - region: 'center', - border: false, - items: [ - new XDMoD.RealTimeValidatingTextField({ - id: 'job-viewer-search-name', - region: 'center', - emptyText: 'Enter Search Name...', - width: 354, - allowBlank: false, - style: 'margin-right: 20px', - fieldLabel: 'Search Name', - tooltip: { - text: 'the name value for the current search', - xtype: 'quicktip' - } - }) - ] - } - ] + items: [{ + xtype: 'fieldset', + region: 'center', + border: false, + items: [ + new XDMoD.RealTimeValidatingTextField({ + id: 'job-viewer-search-name', + region: 'center', + emptyText: 'Enter Search Name...', + width: 354, + allowBlank: false, + style: 'margin-right: 20px', + fieldLabel: 'Search Name', + tooltip: { + text: 'the name value for the current search', + xtype: 'quicktip' + } + }) + ] + }] }, { title: "Results", @@ -827,72 +861,72 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { rowspan: 3, layout: 'border', listeners: { - activate: function() { + activate: function () { if (self.rendered) { self.fireEvent('validate'); } } }, - items: [ - { - xtype: 'editorgrid', - id: 'search_results', - region: 'center', - autoExpandColumn: 'name', - plugins: [checkColumn], - loadMask: false, - store: this.resultsStore, - disabled: true, - sm: new Ext.grid.RowSelectionModel({singleSelect:true}), - listeners: { - rowclick: function(searchgrid, rowIndex) { - var record = searchgrid.store.getAt(rowIndex); - var checked = !record.get('included'); - record.set('included', checked); - checkColumn.checkchange(record, rowIndex, checked); - } + items: [{ + xtype: 'editorgrid', + id: 'search_results', + region: 'center', + autoExpandColumn: 'name', + plugins: [checkColumn], + loadMask: false, + store: this.resultsStore, + disabled: true, + sm: new Ext.grid.RowSelectionModel({ + singleSelect: true + }), + listeners: { + rowclick: function (searchgrid, rowIndex) { + var record = searchgrid.store.getAt(rowIndex); + var checked = !record.get('included'); + record.set('included', checked); + checkColumn.checkchange(record, rowIndex, checked); + } + }, + columns: [ + checkColumn, + { + header: 'Job Id', + dataIndex: 'local_job_id' }, - columns: [ - checkColumn, - { - header: 'Job Id', - dataIndex: 'local_job_id' - }, - { - header: 'Resource', - dataIndex: 'resource', - width: 140, - id: 'resource' + { + header: 'Resource', + dataIndex: 'resource', + width: 140, + id: 'resource' + }, + { + header: 'Name', + dataIndex: 'name', + id: 'name', + width: 210 + } + ], + bbar: new Ext.PagingToolbar({ + pageSize: self._DEFAULT_PAGE_SIZE, + displayInfo: true, + displayMsg: 'Displaying jobs {0} - {1} of {2}', + emptyMsg: 'No jobs to display', + store: self.resultsStore, + listeners: { + load: function (store, records, options) { + this.onLoad(store, records, options); }, - { - header: 'Name', - dataIndex: 'name', - id: 'name', - width: 210 - } - ], - bbar: new Ext.PagingToolbar({ - pageSize: self._DEFAULT_PAGE_SIZE, - displayInfo: true, - displayMsg: 'Displaying jobs {0} - {1} of {2}', - emptyMsg: 'No jobs to display', - store: self.resultsStore, - listeners: { - load: function (store, records, options) { - this.onLoad(store, records, options); - }, - beforechange: function(bbar, params) { - var searchParams = bbar.store.searchParams; - for(var p in searchParams) { - if(searchParams.hasOwnProperty(p) && !params.hasOwnProperty(p)) { - params[p] = searchParams[p]; - } + beforechange: function (bbar, params) { + var searchParams = bbar.store.searchParams; + for (var p in searchParams) { + if (searchParams.hasOwnProperty(p) && !params.hasOwnProperty(p)) { + params[p] = searchParams[p]; } } } - }) - } - ] + } + }) + }] }, { xtype: 'panel', @@ -907,85 +941,94 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { xtype: 'fieldset', region: 'center', items: [{ - xtype: 'realmcombo', - id: 'basic-search-realm', - panel: self, - realmSelectEvent: 'basic_search_realm_selected' - }, - { - xtype: 'combo', - fieldLabel: 'Resource', - id: 'basic-resource', - emptyText: 'Select a Resource', - triggerAction: 'all', - selectOnFocus: true, - displayField: 'long_name', - valueField: 'id', - typeAhead: true, - mode: 'local', - forceSelection: true, - enableKeyEvents: true, - store: new Ext.data.JsonStore({ - proxy: new Ext.data.HttpProxy({ - url: 'infill', - method: 'GET' + xtype: 'realmcombo', + id: 'basic-search-realm', + panel: self, + realmSelectEvent: 'basic_search_realm_selected' + }, + { + xtype: 'combo', + fieldLabel: 'Resource', + id: 'basic-resource', + emptyText: 'Select a Resource', + triggerAction: 'all', + selectOnFocus: true, + displayField: 'long_name', + valueField: 'id', + typeAhead: true, + mode: 'local', + forceSelection: true, + enableKeyEvents: true, + store: new Ext.data.JsonStore({ + proxy: new Ext.data.HttpProxy({ + url: 'infill', + method: 'GET' + }), + baseParams: { + realm: CCR.xdmod.ui.rawDataAllowedRealms[0], + token: self.token + }, + storeId: 'jobviewer-basicsearch-resource', + autoLoad: true, + root: 'results', + fields: [{ + name: 'id', + type: 'string' + }, + { + name: 'short_name', + type: 'string' + }, + { + name: 'long_name', + type: 'string' + } + ], + listeners: { + exception: function (proxy, type, action, options, response) { + CCR.xdmod.ui.presentFailureResponse(response, { + title: 'Job Viewer', + wrapperMessage: 'The Quick Job Lookup resource list failed to load.' + }); + } + } }), - baseParams: { - realm: CCR.xdmod.ui.rawDataAllowedRealms[0], - token: self.token - }, - storeId: 'jobviewer-basicsearch-resource', - autoLoad: true, - root: 'results', - fields: [ - {name: 'id', type: 'string'}, - {name: 'short_name', type: 'string'}, - {name: 'long_name', type: 'string'} - ], listeners: { - exception: function (proxy, type, action, options, response) { - CCR.xdmod.ui.presentFailureResponse(response, { - title: 'Job Viewer', - wrapperMessage: 'The Quick Job Lookup resource list failed to load.' - }); + select: function ( /* combo, record, index */ ) { + self.fireEvent('validate_search_criteria'); + }, + blur: function ( /*combo, event*/ ) { + self.fireEvent('validate_search_criteria'); } } - }), - listeners: { - select: function( /* combo, record, index */ ) { - self.fireEvent('validate_search_criteria'); - }, - blur: function( /*combo, event*/ ) { - self.fireEvent('validate_search_criteria'); - } - } - }, - { - xtype: 'textfield', - fieldLabel: 'Job Number', - emptyText: 'Enter Job #', - id: 'basic-localjobid', - stripCharsRe: /(^\s+|\s+$)/g, - width: 200, - enableKeyEvents: true, - listeners: { - keyup: function( /*field, event*/ ) { - self.fireEvent('validate_search_criteria'); - }, - specialkey: function(field, event) { - if (event.getKey() === event.ENTER) { + }, + { + xtype: 'textfield', + fieldLabel: 'Job Number', + emptyText: 'Enter Job #', + id: 'basic-localjobid', + stripCharsRe: /(^\s+|\s+$)/g, + width: 200, + enableKeyEvents: true, + listeners: { + keyup: function ( /*field, event*/ ) { self.fireEvent('validate_search_criteria'); - return false; + }, + specialkey: function (field, event) { + if (event.getKey() === event.ENTER) { + self.fireEvent('validate_search_criteria'); + return false; + } } } } - }], + ], buttons: [{ xtype: 'button', disabled: true, text: 'Search', id: 'job-viewer-search-lookup', - handler: function( /*button, event*/ ) { + handler: function ( /*button, event*/ ) { var resourceField = Ext.getCmp('basic-resource'); var localjobidField = Ext.getCmp('basic-localjobid'); var realmField = Ext.getCmp('basic-search-realm'); @@ -1017,267 +1060,276 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { xtype: 'fieldset', labelWidth: 55, items: [{ - xtype: 'datefield', - id: 'search_start_date', - format: 'Y-m-d', - name: 'start_date', - fieldLabel: 'Start', - enableKeyEvents: true, - submitValue: false, - update: false, - validator: function(/*val*/) { - return self._dateFieldValidator('startDateField', 'start date'); - }, - listeners: { - beforerender: function (field) { - field.setValue(self._getDefaultStartDate()); - }, - keyup: function (/*field, record, index*/) { - self.fireEvent('validate_search_criteria'); + xtype: 'datefield', + id: 'search_start_date', + format: 'Y-m-d', + name: 'start_date', + fieldLabel: 'Start', + enableKeyEvents: true, + submitValue: false, + update: false, + validator: function ( /*val*/ ) { + return self._dateFieldValidator('startDateField', 'start date'); }, - select: function (/*field, record, index*/) { - self.fireEvent('validate_search_criteria'); + listeners: { + beforerender: function (field) { + field.setValue(self._getDefaultStartDate()); + }, + keyup: function ( /*field, record, index*/ ) { + self.fireEvent('validate_search_criteria'); + }, + select: function ( /*field, record, index*/ ) { + self.fireEvent('validate_search_criteria'); + } } - } - }, - { - xtype: 'datefield', - id: 'search_end_date', - name: 'end_date', - format: 'Y-m-d', - fieldLabel: 'End', - submitValue: false, - enableKeyEvents: true, - update: false, - validator: function(/*val*/) { - return self._dateFieldValidator('endDateField', 'end date'); }, - listeners: { - beforerender: function (field) { - field.setValue(self._getDefaultEndDate()); - }, - keyup: function (/*field, record, index*/) { - self.fireEvent('validate_search_criteria'); + { + xtype: 'datefield', + id: 'search_end_date', + name: 'end_date', + format: 'Y-m-d', + fieldLabel: 'End', + submitValue: false, + enableKeyEvents: true, + update: false, + validator: function ( /*val*/ ) { + return self._dateFieldValidator('endDateField', 'end date'); }, - select: function (/*field, record, index*/) { - self.fireEvent('validate_search_criteria'); + listeners: { + beforerender: function (field) { + field.setValue(self._getDefaultEndDate()); + }, + keyup: function ( /*field, record, index*/ ) { + self.fireEvent('validate_search_criteria'); + }, + select: function ( /*field, record, index*/ ) { + self.fireEvent('validate_search_criteria'); + } } - } - }, - { - id: 'realm-field', - xtype: 'realmcombo', - panel: self, - realmSelectEvent: 'realm_selected' - }, - { - xtype: 'compositefield', - fieldLabel: 'Filter', - defaults: { - margins: '0 8 26 0' }, - items: [ - - { - id: 'search-field', - xtype: 'uxgroupcombo', - emptyText: 'Select a Field...', - triggerAction: 'all', - selectOnFocus: true, - displayField: 'name', - width: 175, - valueField: 'id', - typeAhead: true, - mode: 'local', - forceSelection: true, - groupTextTpl: '{text}', - tpl: '
{name}
', - store: new Ext.data.GroupingStore({ - proxy: new Ext.data.HttpProxy({ - method: 'GET', - url: 'infill', - }), - baseParams: { - token: self.token, - realm: CCR.xdmod.ui.rawDataAllowedRealms[0], - querygroup: 'tg_usage' - - }, - sortInfo: { - field: 'name', - direction: 'ASC' - }, - groupField: 'Category', - autoLoad: true, - storeId: 'dimensionResults', - reader: new Ext.data.JsonReader({ - root: 'results', - idParameter: 'id', - fields: ['id', 'name', 'Category', 'description'] + { + id: 'realm-field', + xtype: 'realmcombo', + panel: self, + realmSelectEvent: 'realm_selected' + }, + { + xtype: 'compositefield', + fieldLabel: 'Filter', + defaults: { + margins: '0 8 26 0' + }, + items: [ + + { + id: 'search-field', + xtype: 'uxgroupcombo', + emptyText: 'Select a Field...', + triggerAction: 'all', + selectOnFocus: true, + displayField: 'name', + width: 175, + valueField: 'id', + typeAhead: true, + mode: 'local', + forceSelection: true, + groupTextTpl: '{text}', + tpl: '
{name}
', + store: new Ext.data.GroupingStore({ + proxy: new Ext.data.HttpProxy({ + method: 'GET', + url: 'infill', + }), + baseParams: { + token: self.token, + realm: CCR.xdmod.ui.rawDataAllowedRealms[0], + querygroup: 'tg_usage' + + }, + sortInfo: { + field: 'name', + direction: 'ASC' + }, + groupField: 'Category', + autoLoad: true, + storeId: 'dimensionResults', + reader: new Ext.data.JsonReader({ + root: 'results', + idParameter: 'id', + fields: ['id', 'name', 'Category', 'description'] + }), + listeners: { + exception: function (proxy, type, action, exception, response) { + switch (response.status) { + case 403: + case 500: + var details = Ext.decode(response.responseText); + Ext.Msg.alert("Error " + response.status + " " + response.statusText, details.message); + break; + case 401: + // Do nothing + break; + default: + Ext.Msg.alert(response.status + ' ' + response.statusText, response.responseText); + } + } + } }), listeners: { - exception: function (proxy, type, action, exception, response) { - switch (response.status) { - case 403: - case 500: - var details = Ext.decode(response.responseText); - Ext.Msg.alert("Error " + response.status + " " + response.statusText, details.message); - break; - case 401: - // Do nothing - break; - default: - Ext.Msg.alert(response.status + ' ' + response.statusText, response.responseText); + select: function (field, record /*, index*/ ) { + if (CCR.exists(record)) { + var value = record.get('id'); + self.fireEvent('field_selected', value); } - } - } - }), - listeners: { - select: function (field, record /*, index*/) { - if (CCR.exists(record)) { - var value = record.get('id'); - self.fireEvent('field_selected', value); - } - }, - afterrender: function (field) { - if (!CCR.exists(self.searchField)) { - self.searchField = field; - } - } - } - }, - { - id: 'search-operator', - xtype: 'label', - text: '=', - style: 'margin-top: 4px; font-weight: bold', - listeners: { - afterrender: function (field) { - if (!CCR.exists(self.operatorField)) { - self.operatorField = field; + }, + afterrender: function (field) { + if (!CCR.exists(self.searchField)) { + self.searchField = field; + } } } - } - }, - { - id: 'search-value', - xtype: 'combo', - emptyText: 'Select a Value...', - triggerAction: 'all', - selectOnFocus: true, - displayField: 'long_name', - width: 170, - valueField: 'id', - typeAhead: true, - mode: 'local', - forceSelection: true, - enableKeyEvents: true, - store: new Ext.data.JsonStore({ - proxy: new Ext.data.HttpProxy({ - method: 'GET', - url: 'infill', - }), - baseParams: { - token: self.token, - querygroup: 'tg_usage', - realm: CCR.xdmod.ui.rawDataAllowedRealms[0], - filter: 'true' - }, - storeId: 'valueStore', - idProperty: 'id', - root: 'results', - fields: [ - {name: 'id', type: 'string'}, - {name: 'name', type: 'string'}, - {name: 'short_name', type: 'string'}, - {name: 'long_name', type: 'string'} - ], + }, + { + id: 'search-operator', + xtype: 'label', + text: '=', + style: 'margin-top: 4px; font-weight: bold', listeners: { - exception: function (proxy, type, action, exception, response) { - switch (response.status) { - case 403: - case 500: - var details = Ext.decode(response.responseText); - Ext.Msg.alert("Error " + response.status + " " + response.statusText, details.message); - break; - case 401: - // Do nothing - break; - default: - Ext.Msg.alert(response.status + ' ' + response.statusText, response.responseText); + afterrender: function (field) { + if (!CCR.exists(self.operatorField)) { + self.operatorField = field; } } } - }), - listeners: { - afterrender: function (field) { - if (!CCR.exists(self.valueField)) { - self.valueField = field; + }, + { + id: 'search-value', + xtype: 'combo', + emptyText: 'Select a Value...', + triggerAction: 'all', + selectOnFocus: true, + displayField: 'long_name', + width: 170, + valueField: 'id', + typeAhead: true, + mode: 'local', + forceSelection: true, + enableKeyEvents: true, + store: new Ext.data.JsonStore({ + proxy: new Ext.data.HttpProxy({ + method: 'GET', + url: 'infill', + }), + baseParams: { + token: self.token, + querygroup: 'tg_usage', + realm: CCR.xdmod.ui.rawDataAllowedRealms[0], + filter: 'true' + }, + storeId: 'valueStore', + idProperty: 'id', + root: 'results', + fields: [{ + name: 'id', + type: 'string' + }, + { + name: 'name', + type: 'string' + }, + { + name: 'short_name', + type: 'string' + }, + { + name: 'long_name', + type: 'string' + } + ], + listeners: { + exception: function (proxy, type, action, exception, response) { + switch (response.status) { + case 403: + case 500: + var details = Ext.decode(response.responseText); + Ext.Msg.alert("Error " + response.status + " " + response.statusText, details.message); + break; + case 401: + // Do nothing + break; + default: + Ext.Msg.alert(response.status + ' ' + response.statusText, response.responseText); + } + } } - }, - select: function (field, record /*, index*/) { - if (CCR.exists(record)) { - var addCmp = Ext.getCmp('search-add'); - if (addCmp) { - addCmp.enable(); + }), + listeners: { + afterrender: function (field) { + if (!CCR.exists(self.valueField)) { + self.valueField = field; + } + }, + select: function (field, record /*, index*/ ) { + if (CCR.exists(record)) { + var addCmp = Ext.getCmp('search-add'); + if (addCmp) { + addCmp.enable(); + } + } + }, + keyup: function (field /*, event*/ ) { + var value = field.el.dom.value; + var addCriteriaCmp = Ext.getCmp('search-add'); + if (!CCR.exists(value)) { + addCriteriaCmp.disable(); + } else { + addCriteriaCmp.enable(); } } - }, - keyup: function(field /*, event*/) { - var value = field.el.dom.value; - var addCriteriaCmp = Ext.getCmp('search-add'); - if (!CCR.exists(value)) { - addCriteriaCmp.disable(); + } + }, + { + id: 'search-add', + xtype: 'button', + disabled: true, + text: 'Add', + handler: function (button /*, event*/ ) { + var realmField = Ext.getCmp('realm-field'); + var searchField = Ext.getCmp('search-field'); + var searchValue = Ext.getCmp('search-value'); + + var realm = realmField ? realmField.getValue() : null; + var fieldRecord = searchField ? searchField.store.getAt(searchField.selectedIndex) : null; + var field = fieldRecord ? fieldRecord.get('id') : null; + var fieldDisplay = fieldRecord ? fieldRecord.get('name') : null; + var operator = '='; + if (searchValue.selectedIndex >= 0) { + var searchRecord = searchValue.store.getAt(searchValue.selectedIndex); + var valueName = searchRecord.get('short_name'); + var value = searchRecord.get('id'); + + self.fireEvent('add_condition', realm, field, fieldDisplay, operator, valueName, value); + self.fireEvent('reset_criteria', self); } else { - addCriteriaCmp.enable(); + searchValue.setValue(null); + button.disable(); } } } - }, - { - id: 'search-add', - xtype: 'button', - disabled: true, - text: 'Add', - handler: function (button /*, event*/ ) { - var realmField = Ext.getCmp('realm-field'); - var searchField = Ext.getCmp('search-field'); - var searchValue = Ext.getCmp('search-value'); - - var realm = realmField ? realmField.getValue() : null; - var fieldRecord = searchField ? searchField.store.getAt(searchField.selectedIndex) : null; - var field = fieldRecord ? fieldRecord.get('id') : null; - var fieldDisplay = fieldRecord ? fieldRecord.get('name') : null; - var operator = '='; - if (searchValue.selectedIndex >= 0) { - var searchRecord = searchValue.store.getAt(searchValue.selectedIndex); - var valueName = searchRecord.get('short_name'); - var value = searchRecord.get('id'); - - self.fireEvent('add_condition', realm, field, fieldDisplay, operator, valueName, value); - self.fireEvent('reset_criteria', self); - } else { - searchValue.setValue(null); - button.disable(); - } - } - } - ] - }, - { - xtype: 'panel', - layout: 'fit', - height: 200, - items: [ - { + ] + }, + { + xtype: 'panel', + layout: 'fit', + height: 200, + items: [{ xtype: 'grid', id: 'job-viewer-search-criteria-grid', region: 'center', autoExpandColumn: 'value', store: this._getSearchStore(), - columns: [ - { + columns: [{ id: 'field', width: 200, header: 'Field', @@ -1298,28 +1350,26 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { { xtype: 'actioncolumn', width: 25, - items: [ - { - icon: '../../../gui/images/delete.png', - handler: function (grid, rowIndex /*, colIndex*/) { - var record = grid.store.getAt(rowIndex); - self.fireEvent('remove_condition', record); - } + items: [{ + icon: '../../../gui/images/delete.png', + handler: function (grid, rowIndex /*, colIndex*/ ) { + var record = grid.store.getAt(rowIndex); + self.fireEvent('remove_condition', record); } - ] + }] } ] - } - ] + }] - }], + } + ], buttons: [{ xtype: 'button', text: 'Search', id: 'job-viewer-search-search', disabled: true, - handler: function( /*button, event*/ ) { + handler: function ( /*button, event*/ ) { var startDate = Ext.getCmp('search_start_date').getValue(); var endDate = Ext.getCmp('search_end_date').getValue(); var realmField = Ext.getCmp('realm-field'); @@ -1351,81 +1401,82 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { }] }] - }, - { - xtype: 'toolbar', - colspan: 2, - width: 1000, - height: 27, - minHeight: 27, - items: [ - '->', - { - xtype: 'button', - id: 'job-viewer-search-save-as', - text: 'Save As', - enabled: false, - tooltip: { - text: 'Save the selected results under a new name', - xtype: 'quicktip' - }, - handler: function(button, event, defaultText) { - var formattedDate = self._formatDate(new Date()); - var extension = Math.floor((Math.random() * 1024) + 1); - Ext.MessageBox.prompt('Pick Search Name', 'Please select a name with which to identify this collection of Search Criteria and it\'s associated results.', function(button, text) { - if (button === 'ok') { - var hasText = CCR.exists(text) && text.length >= 3; - - if (!hasText) { - Ext.MessageBox.show({ - title: 'Invalid Name', - msg: 'You must provide a name with which to identify these results if you wish to save them.', - icon: Ext.MessageBox.ERROR, - buttons: Ext.MessageBox.OK - }); - return; - } + }, + { + xtype: 'toolbar', + colspan: 2, + width: 1000, + height: 27, + minHeight: 27, + items: [ + '->', + { + xtype: 'button', + id: 'job-viewer-search-save-as', + text: 'Save As', + enabled: false, + tooltip: { + text: 'Save the selected results under a new name', + xtype: 'quicktip' + }, + handler: function (button, event, defaultText) { + var formattedDate = self._formatDate(new Date()); + var extension = Math.floor((Math.random() * 1024) + 1); + Ext.MessageBox.prompt('Pick Search Name', 'Please select a name with which to identify this collection of Search Criteria and it\'s associated results.', function (button, text) { + if (button === 'ok') { + var hasText = CCR.exists(text) && text.length >= 3; + + if (!hasText) { + Ext.MessageBox.show({ + title: 'Invalid Name', + msg: 'You must provide a name with which to identify these results if you wish to save them.', + icon: Ext.MessageBox.ERROR, + buttons: Ext.MessageBox.OK + }); + return; + } - self._upsertSearchRecord(text); - } - }, this, false, defaultText || 'search-' + formattedDate + '-' + extension); - } - }, - { - xtype: 'button', - id: 'job-viewer-search-save-results', - text: 'Save Results', - tooltip: { - text: 'Select jobs from the results pane above and click here to save the jobs in the search history.', - xtype: "quicktip" + self._upsertSearchRecord(text); + } + }, this, false, defaultText || 'search-' + formattedDate + '-' + extension); + } }, - handler: function(button, event, defaultText) { - var title = Ext.getCmp('job-viewer-search-name').getValue(); - if(self.editing === true) { - title = title !== "" ? title : self.title; - self._upsertSearchRecord(title, self.dtypeId); - } else { - if (title) { - self._upsertSearchRecord(title); + { + xtype: 'button', + id: 'job-viewer-search-save-results', + text: 'Save Results', + tooltip: { + text: 'Select jobs from the results pane above and click here to save the jobs in the search history.', + xtype: "quicktip" + }, + handler: function (button, event, defaultText) { + var title = Ext.getCmp('job-viewer-search-name').getValue(); + if (self.editing === true) { + title = title !== "" ? title : self.title; + self._upsertSearchRecord(title, self.dtypeId); } else { - var formattedDate = self._formatDate(new Date()); - var extension = Math.floor((Math.random() * 1024) + 1); - var title = defaultText || 'search-' + formattedDate + '-' + extension; - self._upsertSearchRecord(title); + if (title) { + self._upsertSearchRecord(title); + } else { + var formattedDate = self._formatDate(new Date()); + var extension = Math.floor((Math.random() * 1024) + 1); + var title = defaultText || 'search-' + formattedDate + '-' + extension; + self._upsertSearchRecord(title); + } } } + }, + { + xtype: 'button', + id: 'job-viewer-search-cancel', + text: 'Cancel', + handler: function ( /*button, event*/ ) { + self.fireEvent('close_search', self, false); + } } - }, - { - xtype: 'button', - id: 'job-viewer-search-cancel', - text: 'Cancel', - handler: function( /*button, event*/ ) { - self.fireEvent('close_search', self, false); - } - } - ] - }]; + ] + } + ]; }, // getItems /** @@ -1435,7 +1486,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * @param {String} title * @param {String|null} id **/ - _upsertSearchRecord: function(title, id) { + _upsertSearchRecord: function (title, id) { var self = this; var selected = self._resultsToJSON(self._getSelectedRecords()); @@ -1450,9 +1501,9 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { var realmField = Ext.getCmp('realm-field'); var realm = realmField ? realmField.getValue() : null; var idFragment = id !== undefined ? '/' + id : ''; - //var url = XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + '/search/history' + idFragment + '?realm=' + realm + '&token=' + XDMoD.REST.token; + url: 'infill', - Ext.Ajax.request({ + Ext.Ajax.request({ url: 'infill', method: 'POST', params: { @@ -1478,15 +1529,15 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { } self.fireEvent('close_search', self, true); }, - error: function (/*response*/) { + error: function ( /*response*/ ) { Ext.MessageBox.show({ - title: 'Save Error', - msg: 'There was an error saving search [' + title + '].', - icon: Ext.MessageBox.ERROR, - buttons: Ext.MessageBox.OK + title: 'Save Error', + msg: 'There was an error saving search [' + title + '].', + icon: Ext.MessageBox.ERROR, + buttons: Ext.MessageBox.OK }); } - }); + }); }, /** @@ -1494,7 +1545,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * @param {Ext.data.Record} record * @param {boolean|null} checked **/ - _handleIncludeRecord: function(record, checked) { + _handleIncludeRecord: function (record, checked) { var self = this; var dtype = record.get('dtype'); var id = record.get(dtype); @@ -1503,7 +1554,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { if (checked && !exists) { self.selected[id] = record; } - if ( !checked && exists) { + if (!checked && exists) { delete self.selected[id]; } @@ -1512,21 +1563,21 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { record.set('included', true); } }, - + _dateFieldValidator: function (field_id, label) { - var validDates = { - startDateField: Date.parseDate(Ext.getCmp('search_start_date').getRawValue(), 'Y-m-d'), - endDateField: Date.parseDate(Ext.getCmp('search_end_date').getRawValue(), 'Y-m-d') - }; - if (validDates[field_id] === undefined) { - return 'Invalid ' + label + ' (must be of the form YYYY-MM-DD)'; - } - if (validDates.startDateField !== undefined && validDates.endDateField !== undefined) { - if (validDates.startDateField > validDates.endDateField) { - return 'Start date must be less than or equal to the end date'; - } + var validDates = { + startDateField: Date.parseDate(Ext.getCmp('search_start_date').getRawValue(), 'Y-m-d'), + endDateField: Date.parseDate(Ext.getCmp('search_end_date').getRawValue(), 'Y-m-d') + }; + if (validDates[field_id] === undefined) { + return 'Invalid ' + label + ' (must be of the form YYYY-MM-DD)'; + } + if (validDates.startDateField !== undefined && validDates.endDateField !== undefined) { + if (validDates.startDateField > validDates.endDateField) { + return 'Start date must be less than or equal to the end date'; } - return true; + } + return true; }, /** @@ -1534,7 +1585,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * * @param {Object} searchTerms */ - _editQuickSearch: function(searchTerms) { + _editQuickSearch: function (searchTerms) { var self = this; var params = JSON.parse(searchTerms.params); @@ -1560,7 +1611,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * * @param {Object} searchTerms */ - _editAdvancedSearch: function(searchTerms) { + _editAdvancedSearch: function (searchTerms) { var self = this; var startDateField = Ext.getCmp('search_start_date'); @@ -1577,19 +1628,21 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { var realm = searchTerms.realm; realmField.setValue(searchTerms.realm); - var clearSearchCriteria = function() { + var clearSearchCriteria = function () { var store = CCR.exists(self.searchStore) ? self.searchStore : null; if (CCR.exists(store) && CCR.exists(store.removeAll)) { store.removeAll(); } }; - var addSearchCriteria = function() { + var addSearchCriteria = function () { var value; var i; var params = JSON.parse(searchTerms.params); - var advSearch = Ext.getCmp("job-viewer-advanced-search", {removeMask: true}); + var advSearch = Ext.getCmp("job-viewer-advanced-search", { + removeMask: true + }); var searchTermMask = new Ext.LoadMask(advSearch.el); var shown = 0; for (var field in params) { @@ -1621,8 +1674,8 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * the display promises have * resolved. **/ - var partialThen = function(field, value, displayValues) { - return function(displayValues) { + var partialThen = function (field, value, displayValues) { + return function (displayValues) { var fieldDisplay = displayValues[0]; var valueDisplay = displayValues[1]; self.fireEvent('add_condition', realm, field, fieldDisplay, '=', valueDisplay, value); @@ -1632,12 +1685,12 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { }; }; RSVP - .all(displayPromises) - .then(partialThen(field, value)) - .catch(function(reason) { - CCR.error('Error retrieving criteria information', reason); - searchTermMask.hide(); - }); + .all(displayPromises) + .then(partialThen(field, value)) + .catch(function (reason) { + CCR.error('Error retrieving criteria information', reason); + searchTermMask.hide(); + }); } } } @@ -1657,9 +1710,9 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * * @returns {RSVP.Promise} */ - _findFieldDisplay: function(field) { + _findFieldDisplay: function (field) { var self = this; - var url = XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + + url: 'infill', '/dimensions/' + field + '/name?token=' + XDMoD.REST.token; return this._getPromise(url, ['results', 'name']); @@ -1674,9 +1727,9 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * * @return {RSVP.Promise} */ - _findFieldValueDisplay: function(field, value) { + _findFieldValueDisplay: function (field, value) { var self = this; - var url = XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + + url: 'infill', '/dimensions/' + field + '/values/' + value + '/name?token=' + XDMoD.REST.token; @@ -1684,12 +1737,12 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { return this._getPromise(url, ['results', 'name']); }, - _getPromise: function(url, resolveProperties) { + _getPromise: function (url, resolveProperties) { return new RSVP.Promise( - function(resolve, reject){ + function (resolve, reject) { Ext.Ajax.request({ url: 'infill', - success: function(response) { + success: function (response) { var data = JSON.parse(response.responseText); var success = data.success; if (success === true) { @@ -1701,7 +1754,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { resolve(resolveValue); } }, - failure: function(response) { + failure: function (response) { var data = JSON.parse(response.responseText); var message = data.message || 'An unknown error has occured.'; reject(message); @@ -1720,7 +1773,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * * @param {Ext.tree.AsyncTreeNode} node **/ - _retrieveSelected: function(node) { + _retrieveSelected: function (node) { var self = this; /** @@ -1729,12 +1782,12 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * @param {Array} idProperties * @returns {Array} suitable to set 'this.children' to. **/ - var processSelected = function(selected, idProperties) { + var processSelected = function (selected, idProperties) { var results = []; - for ( var i = 0; i < selected.length; i++) { + for (var i = 0; i < selected.length; i++) { var item = selected[i]; var id = item[idProperties[0]]; - for ( var j = 1; j < idProperties.length; j++) { + for (var j = 1; j < idProperties.length; j++) { id = id[idProperties[j]]; } results.push(id); @@ -1748,13 +1801,13 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { var realm = this._getNodeValue(this._getParentNode(node, 'realm'), 'realm'); realm = realm !== null ? realm : ''; - var url = XDMoD.REST.url + '/' + self.dataExport.rest.warehouse + - '/search/history/' + this.dtypeId + - '?realm=' + realm + - '&token=' + XDMoD.REST.token; + url: 'infill', + '/search/history/' + this.dtypeId + + '?realm=' + realm + + '&token=' + XDMoD.REST.token; this._getPromise(url, ['results']) - .then(function(results){ + .then(function (results) { self.children = processSelected(results, ['jobid']); }); } @@ -1772,7 +1825,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * else it attempts to provide the value of * item[item.dtype] or item.get(item.dtype) **/ - _getId: function(item) { + _getId: function (item) { if (item.dtype !== undefined) { return item[item.dtype]; } else if (item.get !== undefined) { @@ -1782,7 +1835,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { return null; }, - _getTitle: function() { + _getTitle: function () { return this.title; }, @@ -1798,7 +1851,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * @return {*} the return value of 'fn' if 'condition' is true else the * return value of 'elseFn' **/ - _onOrCondition: function(condition, fn, elseFn) { + _onOrCondition: function (condition, fn, elseFn) { var args = Array.prototype.slice.call(arguments, 3); if (condition) { return fn.apply(null, args); @@ -1816,14 +1869,14 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * enabled or disabled. * @param {Ext.Component} cmp the component to be enabled / disabled. **/ - _toggle: function(condition, cmp) { - var enable = function(cmp) { + _toggle: function (condition, cmp) { + var enable = function (cmp) { if (cmp && cmp.enable) { cmp.enable(); } }; - var disable = function(cmp) { + var disable = function (cmp) { if (cmp && cmp.disable) { cmp.disable(); } @@ -1848,7 +1901,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * following the parentNode links of * the provided 'child'. **/ - _getParentNode: function(child, dtype) { + _getParentNode: function (child, dtype) { var parent = child.parentNode; if (parent) { var attributes = parent.attributes || {}; @@ -1870,31 +1923,31 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * @return {null|*} returns null if the attribute is not found. Else it * returns the attribute value. **/ - _getNodeValue: function(node, attribute) { - if (node && node.attributes && node.attributes[attribute] !== undefined) { + _getNodeValue: function (node, attribute) { + if (node && node.attributes && node.attributes[attribute] !== undefined) { return node.attributes[attribute]; } return null; }, - _getDefaultStartDate: function() { + _getDefaultStartDate: function () { var start = new Date(); start.setDate(start.getDate() - 7); return start; }, - _getDefaultEndDate: function() { + _getDefaultEndDate: function () { var now = new Date(); now.setDate(now.getDate() - 1); return now; }, - _getFieldDisplayValue: function(field) { - return field && field.store - ? field.store.getById(field.getValue()).get(field.displayField) - : field && field.getValue - ? field.getValue() - : null; + _getFieldDisplayValue: function (field) { + return field && field.store ? + field.store.getById(field.getValue()).get(field.displayField) : + field && field.getValue ? + field.getValue() : + null; }, /** @@ -1909,14 +1962,14 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * * @return {String} **/ - _generateDefaultName: function(params, prefix, delim) { + _generateDefaultName: function (params, prefix, delim) { var args = []; prefix = prefix !== undefined ? prefix : ''; delim = delim !== undefined ? delim : '-'; args.push(prefix); if (typeof params === 'object') { - for ( var key in params) { + for (var key in params) { if (params.hasOwnProperty(key)) { args.push(params[key]); } @@ -1925,7 +1978,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { args = params; } return args - .filter(function(element, index, array) { + .filter(function (element, index, array) { return element !== undefined && element !== null && element !== ''; From 624f480966f3ddbee5f541efadb47bd09f939cc3 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 16 May 2019 08:29:48 -0400 Subject: [PATCH 029/217] Replace everything --- html/gui/js/modules/DataExport.js | 2065 ++--------------------------- 1 file changed, 93 insertions(+), 1972 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 1109ffb384..bbe37c531a 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -1,2014 +1,135 @@ +/** + * Data warehouse export module. + */ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { + module_id: 'data_export', + title: 'Data Export', + usesToolbar: false, + /** * The default number of results to retrieve during paging operations. * * @var {Number} */ _DEFAULT_PAGE_SIZE: 24, - title: 'Data Export', // <-- rename this - module_id: 'data_export', // <-- rename this (see the section on REPORT CHECKBOX for how to name this) - usesToolbar: false, - _DEFAULT_SEARCH_NAME_SIZE: 2, - - /** - * Default constructor, just used to setup the events to be listened for, - * some sane defaults, as well as the child components. - * properties of note: - * searchStore: local Ext.data.ArrayStore() used to store the search - * terms before submitting as a 'search'. - * realmField: Combobox containing the realms available for selection. - * searchField: Combobox dependent on 'realmField' selection. Provides - * a selection of dimensions available for selection. - * valueField: Combobox dependent on 'searchField' selection. Provides - * a selection of dimension values available for selection. - **/ - initComponent: function () { - var self = this; - - this.addEvents( - 'add_condition', - 'remove_condition', - 'perform_search', - 'cancel_search', - 'close_search', - 'reset_criteria', - 'realm_selected', - 'field_selected' - ); - - this.resultsStore = this._createResultsStore(); - - this.selected = {}; - this.children = []; - - this.editing = false; - - XDMoD.Module.DataExport.SearchPanel.superclass.initComponent.apply( - Ext.apply(this, { - layout: 'table', - width: 1000, - autoScroll: true, - border: false, - layoutConfig: { - columns: 2 - }, - style: 'background-color: #D0D0D0;', - defaults: { - frame: false, - border: false, - width: 500, - height: 300 - }, - items: self._getItems() - }), arguments - ); - - this.resetResults = function () { - self.resultsStore.loadData({ - results: [], - totalCount: 0 - }, false); - self.selected = {}; - }; - - }, // initComponent - - listeners: { - - /** - * Fired when this component is first rendered to the page. We use it - * to setup event forwarding of the 'load' event from the search_results - * component to this components bottom toolbar. We do that here because - * it's only at this point that both components are available. - */ - render: function () { - var searchResults = Ext.getCmp('search_results'); - var bbar = searchResults ? searchResults.getBottomToolbar() : null; - if (bbar) { - bbar.relayEvents(this.resultsStore, ['load']); - } - }, // render - - /** - * This is the 'show' event that has been forwarded from the containing - * window. - * @param {Ext.Window} window - **/ - show: function (window) { - var body = Ext.getBody(); - var bodyBox = body.getBox(); - var bodyHeight = bodyBox.height; - var bodyWidth = bodyBox.width; - - var box = window.getBox(); - var x = box.x; - var y = box.y; - var height = box.height; - var width = box.width; - - var adjHeight = Math.min(bodyHeight, height); - var adjWidth = Math.min(bodyWidth, width); - var adjX = x < 0 ? 0 : x; - var adjY = y < 0 ? 0 : y; - - window.setHeight(adjHeight); - window.setWidth(adjWidth); - window.setPosition(adjX, adjY); - - this.shown = true; - if (!this.children) { - this.children = []; - } - - this.fireEvent('validate'); - this.fireEvent('validate_search_criteria'); - if (!this.editing) { - this.ownerCt.setTitle("Search"); - } - }, // show - - /** - * Forwarded 'move' event from this panel's Ext.Window parent. - * This will keep the window from being moved outside of the users - * visible space. - * - * @param {Ext.Window} window the window object that was moved. - * @param {Number} x new x coordinate of the window. - * @param {Number} y new y coordinate of the window. - **/ - move: function (window, x, y) { - var adjX = x < 0 ? 0 : x; - var adjY = y < 0 ? 0 : y; - - if ((adjX === 0 && adjX !== x) || - (adjY === 0 && adjY !== y)) { - window.setPosition(adjX, adjY); - } - }, - - /** - * Forwarded 'hide' event from this panel's Ext.Window parent. - * we capture it here so that we can clean up the UI and have - * it ready for its next use. - **/ - hide: function () { - this.fireEvent('close_search', this, false); - }, - - /** - * - * @param {String} realm - * @param {String} field - * @param {String} fieldDisplay - * @param {String} operator - * @param {String} value - * @param {String} valueId - */ - add_condition: function (realm, field, fieldDisplay, operator, value, valueId) { - var record = new this.searchStore.recordType({ - realm: realm, - field: field, - fieldDisplay: fieldDisplay, - operator: operator, - value: value, - valueId: valueId - }); - this.searchStore.add(record); - }, // add_condition - - /** - * Indicates that this component should remove the provided record from - * it's search store. - * - * @param {Ext.data.Record} record - */ - remove_condition: function (record) { - if (CCR.isType(record, CCR.Types.Object)) { - this.searchStore.remove(record); - } - }, // remove_condition - - /** - * Indicates that this component should cancel the currently active - * search. - * - * @param panel - */ - cancel_search: function ( /*panel*/ ) { - this.ownerCt.hide(); - }, // cancel_search - - /** - * Indicates that the user has requested a search be performed. - * - * @param panel - */ - search_requested: function (panel, searchType, searchParams) { - var self = this; - if (!this.loadMask) { - this.loadMask = new Ext.LoadMask( - panel.getEl(), { - id: 'job-viewer-search-mask', - store: this.resultsStore - }); - } - - panel.resetResults(); - - this.resultsStore.load({ - params: searchParams, - callback: function (records, options, success) { - - var resultsGrid, saveButton, text, params, prefix; - - if (!success) { - // Store load failure is handled by the exception listener - return; - } - - // Cache the search parameters in the store object for use if the - // search is saved (ExtJs does not provide a native call to do this). - self.resultsStore.searchParams = searchParams; - - resultsGrid = Ext.getCmp('search_results'); - - if (self.resultsStore.getTotalCount() === 0) { - resultsGrid.setDisabled(true); - resultsGrid.getEl().mask(searchType + ' returned zero jobs.', 'x-mask'); - } else { - resultsGrid.getEl().unmask(); - resultsGrid.setDisabled(false); - - if (searchType == 'Lookup') { - // Since the Quick Lookup search is designed to find an exact job, - // automatically select the jobs returned - self.resultsStore.each(function (record) { - record.set('included', true); - text = record.get('text'); - }); - var resource = self._getFieldDisplayValue(Ext.getCmp('basic-resource')); - var localJobId = self._getFieldDisplayValue(Ext.getCmp('basic-localjobid')); - params = [resource, localJobId]; - } else if (searchType == 'Search') { - params = [ - 'search', - self._formatDate(new Date()), - Math.floor((Math.random() * 1024) + 1) - ]; - } - - var searchNameField = Ext.getCmp('job-viewer-search-name'); - var searchName = searchNameField.getValue(); - - if (!searchName) { - if (!params) { - params = JSON.parse(searchParams.params); - } - searchNameField.setValue(self._generateDefaultName(params, prefix)); - searchNameField.focus(true); - } - - self.fireEvent('validate'); - } - } - }); - }, // search_requested - - basic_search_realm_selected: function (realm) { - var basicSearch = Ext.getCmp('job-viewer-search-lookup'); - basicSearch.disable(); - var jobidField = Ext.getCmp('basic-localjobid'); - jobidField.reset(); - var resource = Ext.getCmp('basic-resource'); - resource.store.load({ - params: { - realm: realm - } - }); - }, - - /** - * Event fired when a realm has been selected. - * - * @param realm - */ - realm_selected: function (realm) { - Ext.getCmp('search-add').disable(); - Ext.getCmp('job-viewer-search-search').disable(); - this.searchStore.removeAll(); - this.valueField.store.setBaseParam({ - realm: realm - }); - this.searchField.reset(); - this.searchField.store.load({ - params: { - realm: realm - } - }); - }, - - validate_search_criteria: function () { - var lookupValid, searchValid; - - lookupValid = Ext.getCmp('basic-resource').getValue().toString().length > 0 && - Ext.getCmp('basic-localjobid').getValue().toString().length > 0; - - Ext.getCmp('job-viewer-search-lookup').setDisabled(!lookupValid); - - searchValid = this.searchStore.getCount() > 0 && - Ext.getCmp('search_start_date').isValid() && - Ext.getCmp('search_end_date').isValid(); - - Ext.getCmp('job-viewer-search-search').setDisabled(!searchValid); - }, - - /** - * Indicates that the UI should be validated. If it is currently not in - * a valid state the the user should be notified. - * - */ - validate: function (options) { - if (this.shown === true) { - this._validateResults(options); - } - }, // validate - - /** - * Indicates that the provided field was selected and as such if the - * field has a child field ( field whose values depend on this fields - * value ) then update the child fields parameters, remove all current - * values and let the user pick from the possibly new values. - * - * @param {Object} field that has been selected. - */ - field_selected: function (field) { - var self = this; - if (CCR.exists(self.valueField) && CCR.exists(self.valueField.store)) { - self.valueField.store.proxy.setUrl(self.valueField.store.proxy.url + ('/' + field)); - - self.valueField.store.removeAll(); - self.valueField.setValue(null); - self._selectInitial('search-value'); - } - }, // field_selected - - /** - * Indicates that this component should reset search and value field - * components of the provided panel. - * - * @param panel - */ - reset_criteria: function (panel, all) { - if (CCR.exists(panel)) { - var field = CCR.exists(panel.searchField) ? panel.searchField : null; - var value = CCR.exists(panel.valueField) ? panel.valueField : null; - var add = Ext.getCmp('search-add'); - - if (CCR.exists(value) && CCR.exists(value.setValue)) { - value.setValue(null); - if (CCR.exists(add)) { - add.disable(); - } - } - - if (all) { - if (CCR.exists(field) && CCR.exists(field.setValue)) { - field.setValue(null); - } - } - } - }, // reset_criteria - - /** - * Indicates that the user wishes to close this component. We need to - * reset everything so that it's ready for use the next time the window - * is opened. - * - * @param panel - * @param reload - */ - close_search: function (panel, reload) { - if (CCR.exists(panel)) { - reload = reload || false; - - // HIDE: this window. - panel.ownerCt.hide(); - - // CHECK: if we should be reloading - if (reload) { - this.dataExport.fireEvent('reload_root'); - this.dataExport.fireEvent('activate'); - } - - panel.searchStore.removeAll(); - panel.resetResults(); - - var basicResource = Ext.getCmp('basic-resource'); - var basicJobNumber = Ext.getCmp('basic-localjobid'); - - if (basicResource) { - basicResource.setValue(''); - } - if (basicJobNumber) { - basicJobNumber.setValue(''); - } - - var resultsGrid = Ext.getCmp('search_results'); - if (resultsGrid) { - resultsGrid.getEl().unmask(); - resultsGrid.setDisabled(true); - } - - var searchField = Ext.getCmp('search-field'); - searchField.setValue(null); - var searchValue = Ext.getCmp('search-value'); - searchValue.setValue(null); - var searchNameField = Ext.getCmp('job-viewer-search-name'); - searchNameField.setValue(''); - searchNameField.clearInvalid(); - - this.shown = false; - this.editing = false; - - delete this.dtype; - delete this.dtypeId; - delete this.children; - } - }, // close_search - - /** - * An event that can be called when a user wishes to 'edit' an already - * existing search. It accepts the {Ext.tree.AsyncTreeNode} value from - * the SearchHistoryTree and then sets up the Search Panel for the - * type of Search that was performed. Making sure that all fo the user - * modifiable values are set per the node passed in. - * - * @param {Ext.tree.AsyncTreeNode} node - **/ - edit_search: function (node) { - this.editing = true; - var params = node.attributes; - - if (!node.attributes.searchterms.params) { - CCR.error('Error Viewing Search', 'Unable to view search, no data provided.'); - return false; - } - - this.title = params.text; - this.dtype = params.dtype; - this.dtypeId = params[this.dtype]; - this._retrieveSelected(node); - - this.ownerCt.setTitle("Editing Search: " + this.title); - Ext.getCmp('job-viewer-search-name').setValue(this.title); - if (!this.ownerCt.isVisible()) { - this.ownerCt.show(); - } - if (params.searchterms.params.start_date) { - this._editAdvancedSearch.call(this, params.searchterms.params); - } else { - this._editQuickSearch.call(this, params.searchterms.params); - } - - return true; - } // edit_search - }, // listeners - - /** - * A private helper function that determines whether or not the state of the - * results store is considered 'valid' such that the 'Save Results' button - * should be enabled. - * - * @param {Object} options - **/ - _validateResults: function (options) { - options = options || {}; - var validate = options.validate !== undefined ? options.validate : false; - if (this.resultsStore) { - var field = Ext.getCmp('job-viewer-search-name'); - - var searchNameIsValid = options.name && validate ? - field.validateValue(options.name) : - validate ? - field.isValid() : - true; - - var valid = this.resultsStore.getCount() > 0 && - this._getSelectedRecords().length > 0 && - searchNameIsValid; - - var saveResults = Ext.getCmp('job-viewer-search-save-results'); - var saveAs = Ext.getCmp('job-viewer-search-save-as'); - - this._toggle(valid, saveResults); - this._toggle(valid && this.editing === true, saveAs); - } - }, - - /** - * Selects the initial item for a component to the record identified by the index provided by the user. - * This method requires that the identified component have a function called 'getStore' that supports a function - * 'getCount' to work as intended. - * - * @param {String} id of the component that has a 'store' ( ie. Ext.form.ComboBox ) whose value you want to set. - * @param {null|Number} index of the initial record to set. Defaults to 0. - * @param {null|Function} callback to be executed when this function is complete. - * @private - */ - _selectInitial: function (id, index, callback) { - callback = callback || function () {}; - - if (CCR.exists(id)) { - var field = Ext.getCmp(id); - var store = CCR.exists(field) && CCR.exists(field.getStore) ? field.getStore() : null; - var hasRecords = CCR.exists(store) && CCR.exists(store.getCount) ? store.getCount() > 0 : false; - - /** - * - * @param {Ext.data.ArrayStore} store - * @param {Ext.form.ComboBox} field - * @param {Function} callback - */ - var selectRecord = function (store, field, callback) { - if (index === undefined) { - return; - } - var record = store.getAt(index); - if (field.isExpanded()) { - field.select(index, true); - } else { - var getName = function (record) { - if (CCR.exists(record)) { - var properties = ['name', 'short_name']; - for (var property in properties) { - if (properties.hasOwnProperty(property)) { - var value = record.get(properties[property]); - if (CCR.exists(value)) { - return value; - } - } - } - } - return ""; - }; - var text = getName(record); - field.setValue(text); - field.selectedIndex = index; - } - field.fireEvent('select', field, record, index); - callback(record); - }; - - if (hasRecords) { - selectRecord(store, field, callback); - } else { - store.load({ - callback: function ( /*records, options*/ ) { - selectRecord(store, field, callback); - } - }); - } - } - }, // select_initial - - /** - * Format the provided date value in a such a way that the ExtJS Date - * components can deal with it. - * - * @param {Date} value - * @returns {string} - * @private - */ - _formatDate: function (value) { - if (!CCR.isType(value, '[object Date]')) { - return String(value); - } - - var pad = CCR.pad; - var left = CCR.STR_PAD_LEFT; - - var year = value.getFullYear(); - var month = value.getMonth(); - var date = value.getDate(); - - var dates = [ - year, - pad(String(month + 1), 2, '0', left), - pad(String(date), 2, '0', left) + initComponent: function () { + var requestData = [ + ['Jobs', '2018-01-01', '2018-12-31', 'CSV', 'Pending'] ]; - return dates.join('-'); - }, // _formatDate + var requestStore = new Ext.data.ArrayStore({ + fields: [ + {name: 'realm'}, + {name: 'start_date'}, + {name: 'end_date'}, + {name: 'format'}, + {name: 'state'} + ] + }); - /** - * Helper function that returns a new JsonStore that is suitable for use - * as this components result store. - * - * @returns {*} - * @private - */ - _createResultsStore: function () { - var self = this; + requestStore.loadData(requestData); - return new Ext.data.JsonStore({ - id: 'results_store', - url: 'infill', - proxy: new Ext.data.HttpProxy({ - api: { - read: { - method: 'GET', - url: 'infill', - } - } - }), - root: 'results', - totalProperty: 'totalCount', - fields: [{ - name: 'dtype', - mapping: 'dtype', - type: 'string' - }, - { - name: 'jobid', - mapping: 'jobid', - type: 'int' - }, - { - name: 'local_job_id', - mapping: 'local_job_id', - type: 'string' - }, + Ext.apply(this, { + items: [ { - name: 'name', - mapping: 'name', - type: 'string' - }, - { - name: 'realm', - mapping: 'realm', - type: 'string' - }, - { - name: 'resource', - mapping: 'resource', - type: 'string' - }, - { - name: 'text', - mapping: 'text', - type: 'string' - }, - { - name: 'included', - mapping: 'included', - type: 'bool', - defaultValue: false - }, - { - name: 'username', - mapping: 'username', - type: 'string' - } - ], - listeners: { - - /** - * - * @param {Ext.data.Store} store - * @param {[]Ext.data.Record} records - * @param {Object} params - **/ - load: function (store, records /*, params*/ ) { - for (var i = 0; i < records.length; i++) { - var record = records[i]; - var id = self._getId(record); - var checked = self.children.indexOf(id) >= 0; - self._handleIncludeRecord(record, checked); - } - self.fireEvent('validate'); - }, - - exception: function (proxy, type, action, options, response /*, arg*/ ) { - var data = JSON.parse(response.responseText); - var status = response.status; - var message = data.message || 'An error occurred while attempting to execute the requested operation. Response Code: [' + status + ']'; - Ext.MessageBox.show({ - title: 'Error: Performing Search', - msg: message, - icon: Ext.MessageBox.ERROR, - buttons: Ext.MessageBox.OK - }); - - } - } - }); - }, // _createResultsStore - - /** - * Returns an ArrayStore that is suitable for use as this components - * search criteria store. - * - * @returns {*} - * @private - */ - _getSearchStore: function () { - var self = this; - if (this.searchStore === null || this.searchStore === undefined) { - this.searchStore = new Ext.data.ArrayStore({ - id: 'search-grid-store', - proxy: new Ext.data.MemoryProxy(), - fields: ['realm', 'field', 'fieldDisplay', 'operator', 'value', 'valueId'], - listeners: { - remove: function ( /*store, record, index*/ ) { - self.fireEvent('validate_search_criteria'); - }, - add: function ( /*store, records, index*/ ) { - self.fireEvent('validate_search_criteria'); - } - } - }); - } - return this.searchStore; - }, // _getSearchStore - - /** - * Retrieve the value of the date field identified by the provided prefix. - * - * @param {String} prefix - * @returns {*} - * @private - */ - _getDateValue: function (prefix) { - if (!CCR.exists(prefix) || typeof prefix !== 'string' || prefix.length < 1) { - return null; - } - - var dateField = Ext.getCmp(prefix + '_date'); - - var date = dateField.getValue(); - - return date; - }, // _getDateValue - - /** - * Retrieve all of the currently 'selected' records across all pages. - * - * @returns {*} - * @private - */ - _getSelectedRecords: function () { - var included = this.resultsStore.query('included', true); - - var existingselections = {}; - for (var i = 0; i < included.items.length; i++) { - existingselections[included.items[i].data[included.items[i].data.dtype]] = 1; - } - - for (var key in this.selected) { - if (this.selected.hasOwnProperty(key) && !existingselections.hasOwnProperty(key)) { - included.add(key, this.selected[key]); - } - } - - return included; - }, // _getSelectedRecords - - /** - * Returns an array of objects that only contain a particular set of properties. - * - * @param values - * @returns {Array} - * @private - */ - _resultsToJSON: function (values) { - if (CCR.exists(values) && CCR.exists(values.length) && values.length > 0) { - var results = []; - - var attributes = ['resource', 'name', 'jobid', 'text', 'realm', 'dtype', 'local_job_id']; - for (var i = 0; i < values.length; i++) { - var value = values.get(i); - var temp = {}; - for (var j = 0; j < attributes.length; j++) { - var attribute = attributes[j]; - if (CCR.exists(attribute)) { - temp[attribute] = value.get(attribute); - } - } - results.push(temp); - } - return results; - } - return []; - }, // resultsToJSON - - /** - * Return an array of ExtJS components to be used as this components - * items property. - * - * @returns {*[]} - * @private - */ - _getItems: function () { - var self = this; - - var checkColumn = new Ext.grid.CheckColumn({ - header: 'Include', - dataIndex: 'included', - id: 'included', - width: 55, - checkchange: function (record, dataIndex, checked) { - self._handleIncludeRecord(record, checked); - self.fireEvent('validate'); - } - }); - - return [{ - xtype: 'panel', - layout: 'border', - height: 48, - width: 502, - colspan: 1, - border: true, - style: 'margin-left: -2px; margin-top: -2px; background-color: #D0D0D0;', - items: [{ - xtype: 'fieldset', - region: 'center', - border: false, + xtype: 'form', + title: 'Request Data Export', + region: 'west', + width: 375, + layout: 'vbox', items: [ - new XDMoD.RealTimeValidatingTextField({ - id: 'job-viewer-search-name', - region: 'center', - emptyText: 'Enter Search Name...', - width: 354, - allowBlank: false, - style: 'margin-right: 20px', - fieldLabel: 'Search Name', - tooltip: { - text: 'the name value for the current search', - xtype: 'quicktip' - } - }) - ] - }] - }, - { - title: "Results", - id: 'search_results_container', - xtype: "panel", - height: 582, - rowspan: 3, - layout: 'border', - listeners: { - activate: function () { - if (self.rendered) { - self.fireEvent('validate'); - } - } - }, - items: [{ - xtype: 'editorgrid', - id: 'search_results', - region: 'center', - autoExpandColumn: 'name', - plugins: [checkColumn], - loadMask: false, - store: this.resultsStore, - disabled: true, - sm: new Ext.grid.RowSelectionModel({ - singleSelect: true - }), - listeners: { - rowclick: function (searchgrid, rowIndex) { - var record = searchgrid.store.getAt(rowIndex); - var checked = !record.get('included'); - record.set('included', checked); - checkColumn.checkchange(record, rowIndex, checked); - } - }, - columns: [ - checkColumn, { - header: 'Job Id', - dataIndex: 'local_job_id' + xtype: 'combo', + fieldLabel: 'Realm', + name: 'realm', + forceSelection: true, + emptyText: 'Select a realm', + // TODO: Switch to remote. + mode: 'local', + valueField: 'id', + displayField: 'name', + store: new Ext.data.ArrayStore({ + fields: ['id', 'name'], + data: [ + ['jobs', 'Jobs'], + ['supremm', 'SUPReMM'], + ['accounts', 'Accounts'], + ['allocations', 'Allocations'], + ['requests', 'Requests'], + ['resource_allocations', 'Resource Allocations'] + ] + }) }, { - header: 'Resource', - dataIndex: 'resource', - width: 140, - id: 'resource' + xtype: 'datefield', + fieldLabel: 'Start Date', + name: 'start_date' }, { - header: 'Name', - dataIndex: 'name', - id: 'name', - width: 210 - } - ], - bbar: new Ext.PagingToolbar({ - pageSize: self._DEFAULT_PAGE_SIZE, - displayInfo: true, - displayMsg: 'Displaying jobs {0} - {1} of {2}', - emptyMsg: 'No jobs to display', - store: self.resultsStore, - listeners: { - load: function (store, records, options) { - this.onLoad(store, records, options); - }, - beforechange: function (bbar, params) { - var searchParams = bbar.store.searchParams; - for (var p in searchParams) { - if (searchParams.hasOwnProperty(p) && !params.hasOwnProperty(p)) { - params[p] = searchParams[p]; - } - } - } - } - }) - }] - }, - { - xtype: 'panel', - title: 'Quick Job Lookup', - tools: [{ - id: 'help', - qtip: 'Use the quick lookup form to search for a job based on its ID and the resource on which it ran.' - }], - layout: 'border', - height: 160, - items: [{ - xtype: 'fieldset', - region: 'center', - items: [{ - xtype: 'realmcombo', - id: 'basic-search-realm', - panel: self, - realmSelectEvent: 'basic_search_realm_selected' + xtype: 'datefield', + fieldLabel: 'End Date', + name: 'end_date' }, { xtype: 'combo', - fieldLabel: 'Resource', - id: 'basic-resource', - emptyText: 'Select a Resource', - triggerAction: 'all', - selectOnFocus: true, - displayField: 'long_name', - valueField: 'id', - typeAhead: true, + fieldLabel: 'Format', + name: 'format', mode: 'local', - forceSelection: true, - enableKeyEvents: true, - store: new Ext.data.JsonStore({ - proxy: new Ext.data.HttpProxy({ - url: 'infill', - method: 'GET' - }), - baseParams: { - realm: CCR.xdmod.ui.rawDataAllowedRealms[0], - token: self.token - }, - storeId: 'jobviewer-basicsearch-resource', - autoLoad: true, - root: 'results', - fields: [{ - name: 'id', - type: 'string' - }, - { - name: 'short_name', - type: 'string' - }, - { - name: 'long_name', - type: 'string' - } - ], - listeners: { - exception: function (proxy, type, action, options, response) { - CCR.xdmod.ui.presentFailureResponse(response, { - title: 'Job Viewer', - wrapperMessage: 'The Quick Job Lookup resource list failed to load.' - }); - } - } - }), - listeners: { - select: function ( /* combo, record, index */ ) { - self.fireEvent('validate_search_criteria'); - }, - blur: function ( /*combo, event*/ ) { - self.fireEvent('validate_search_criteria'); - } - } + emptyText: 'Select an export format', + valueField: 'id', + displayField: 'name', + store: new Ext.data.ArrayStore({ + fields: ['id', 'name'], + data: [ + ['csv', 'CSV'], + ['json', 'JSON'] + ] + }) }, { - xtype: 'textfield', - fieldLabel: 'Job Number', - emptyText: 'Enter Job #', - id: 'basic-localjobid', - stripCharsRe: /(^\s+|\s+$)/g, - width: 200, - enableKeyEvents: true, - listeners: { - keyup: function ( /*field, event*/ ) { - self.fireEvent('validate_search_criteria'); - }, - specialkey: function (field, event) { - if (event.getKey() === event.ENTER) { - self.fireEvent('validate_search_criteria'); - return false; - } - } - } - } - ], - buttons: [{ - xtype: 'button', - disabled: true, - text: 'Search', - id: 'job-viewer-search-lookup', - handler: function ( /*button, event*/ ) { - var resourceField = Ext.getCmp('basic-resource'); - var localjobidField = Ext.getCmp('basic-localjobid'); - var realmField = Ext.getCmp('basic-search-realm'); - var params = { - realm: realmField.getValue(), - params: JSON.stringify({ - resource_id: resourceField.getValue(), - local_job_id: localjobidField.getValue() - }) - }; - self.fireEvent('search_requested', self, 'Lookup', params); + xtype: 'button', + text: 'Submit Request' } - }] - }] - }, - { - xtype: 'panel', - id: 'job-viewer-advanced-search', - title: 'Advanced Search', - tools: [{ - id: 'help', - qtip: 'Use the advanced search form to search for jobs based on one or more filters and a date range.' - }], - height: 375, - layout: 'border', - items: [{ + ] + }, + { + xtype: 'grid', region: 'center', - id: 'criteria_advanced', - xtype: 'fieldset', - labelWidth: 55, - items: [{ - xtype: 'datefield', - id: 'search_start_date', - format: 'Y-m-d', - name: 'start_date', - fieldLabel: 'Start', - enableKeyEvents: true, - submitValue: false, - update: false, - validator: function ( /*val*/ ) { - return self._dateFieldValidator('startDateField', 'start date'); - }, - listeners: { - beforerender: function (field) { - field.setValue(self._getDefaultStartDate()); - }, - keyup: function ( /*field, record, index*/ ) { - self.fireEvent('validate_search_criteria'); - }, - select: function ( /*field, record, index*/ ) { - self.fireEvent('validate_search_criteria'); - } - } - }, + // TODO: Replace with remote store. + store: requestStore, + columns: [ { - xtype: 'datefield', - id: 'search_end_date', - name: 'end_date', - format: 'Y-m-d', - fieldLabel: 'End', - submitValue: false, - enableKeyEvents: true, - update: false, - validator: function ( /*val*/ ) { - return self._dateFieldValidator('endDateField', 'end date'); - }, - listeners: { - beforerender: function (field) { - field.setValue(self._getDefaultEndDate()); - }, - keyup: function ( /*field, record, index*/ ) { - self.fireEvent('validate_search_criteria'); - }, - select: function ( /*field, record, index*/ ) { - self.fireEvent('validate_search_criteria'); - } - } + id: 'realm', + header: 'Realm', + dataIndex: 'realm' }, { - id: 'realm-field', - xtype: 'realmcombo', - panel: self, - realmSelectEvent: 'realm_selected' + id: 'start_date', + header: 'Start Date', + dataIndex: 'start_date' }, { - xtype: 'compositefield', - fieldLabel: 'Filter', - defaults: { - margins: '0 8 26 0' - }, - items: [ - - { - id: 'search-field', - xtype: 'uxgroupcombo', - emptyText: 'Select a Field...', - triggerAction: 'all', - selectOnFocus: true, - displayField: 'name', - width: 175, - valueField: 'id', - typeAhead: true, - mode: 'local', - forceSelection: true, - groupTextTpl: '{text}', - tpl: '
{name}
', - store: new Ext.data.GroupingStore({ - proxy: new Ext.data.HttpProxy({ - method: 'GET', - url: 'infill', - }), - baseParams: { - token: self.token, - realm: CCR.xdmod.ui.rawDataAllowedRealms[0], - querygroup: 'tg_usage' - - }, - sortInfo: { - field: 'name', - direction: 'ASC' - }, - groupField: 'Category', - autoLoad: true, - storeId: 'dimensionResults', - reader: new Ext.data.JsonReader({ - root: 'results', - idParameter: 'id', - fields: ['id', 'name', 'Category', 'description'] - }), - listeners: { - exception: function (proxy, type, action, exception, response) { - switch (response.status) { - case 403: - case 500: - var details = Ext.decode(response.responseText); - Ext.Msg.alert("Error " + response.status + " " + response.statusText, details.message); - break; - case 401: - // Do nothing - break; - default: - Ext.Msg.alert(response.status + ' ' + response.statusText, response.responseText); - } - } - } - }), - listeners: { - select: function (field, record /*, index*/ ) { - if (CCR.exists(record)) { - var value = record.get('id'); - self.fireEvent('field_selected', value); - } - - }, - afterrender: function (field) { - if (!CCR.exists(self.searchField)) { - self.searchField = field; - } - } - } - }, - { - id: 'search-operator', - xtype: 'label', - text: '=', - style: 'margin-top: 4px; font-weight: bold', - listeners: { - afterrender: function (field) { - if (!CCR.exists(self.operatorField)) { - self.operatorField = field; - } - } - } - }, - { - id: 'search-value', - xtype: 'combo', - emptyText: 'Select a Value...', - triggerAction: 'all', - selectOnFocus: true, - displayField: 'long_name', - width: 170, - valueField: 'id', - typeAhead: true, - mode: 'local', - forceSelection: true, - enableKeyEvents: true, - store: new Ext.data.JsonStore({ - proxy: new Ext.data.HttpProxy({ - method: 'GET', - url: 'infill', - }), - baseParams: { - token: self.token, - querygroup: 'tg_usage', - realm: CCR.xdmod.ui.rawDataAllowedRealms[0], - filter: 'true' - }, - storeId: 'valueStore', - idProperty: 'id', - root: 'results', - fields: [{ - name: 'id', - type: 'string' - }, - { - name: 'name', - type: 'string' - }, - { - name: 'short_name', - type: 'string' - }, - { - name: 'long_name', - type: 'string' - } - ], - listeners: { - exception: function (proxy, type, action, exception, response) { - switch (response.status) { - case 403: - case 500: - var details = Ext.decode(response.responseText); - Ext.Msg.alert("Error " + response.status + " " + response.statusText, details.message); - break; - case 401: - // Do nothing - break; - default: - Ext.Msg.alert(response.status + ' ' + response.statusText, response.responseText); - } - } - } - }), - listeners: { - afterrender: function (field) { - if (!CCR.exists(self.valueField)) { - self.valueField = field; - } - }, - select: function (field, record /*, index*/ ) { - if (CCR.exists(record)) { - var addCmp = Ext.getCmp('search-add'); - if (addCmp) { - addCmp.enable(); - } - } - }, - keyup: function (field /*, event*/ ) { - var value = field.el.dom.value; - var addCriteriaCmp = Ext.getCmp('search-add'); - if (!CCR.exists(value)) { - addCriteriaCmp.disable(); - } else { - addCriteriaCmp.enable(); - } - } - } - }, - { - id: 'search-add', - xtype: 'button', - disabled: true, - text: 'Add', - handler: function (button /*, event*/ ) { - var realmField = Ext.getCmp('realm-field'); - var searchField = Ext.getCmp('search-field'); - var searchValue = Ext.getCmp('search-value'); - - var realm = realmField ? realmField.getValue() : null; - var fieldRecord = searchField ? searchField.store.getAt(searchField.selectedIndex) : null; - var field = fieldRecord ? fieldRecord.get('id') : null; - var fieldDisplay = fieldRecord ? fieldRecord.get('name') : null; - var operator = '='; - if (searchValue.selectedIndex >= 0) { - var searchRecord = searchValue.store.getAt(searchValue.selectedIndex); - var valueName = searchRecord.get('short_name'); - var value = searchRecord.get('id'); - - self.fireEvent('add_condition', realm, field, fieldDisplay, operator, valueName, value); - self.fireEvent('reset_criteria', self); - } else { - searchValue.setValue(null); - button.disable(); - } - } - } - ] + id: 'end_date', + header: 'End Date', + dataIndex: 'end_date' }, { - xtype: 'panel', - layout: 'fit', - height: 200, - items: [{ - xtype: 'grid', - id: 'job-viewer-search-criteria-grid', - region: 'center', - autoExpandColumn: 'value', - store: this._getSearchStore(), - columns: [{ - id: 'field', - width: 200, - header: 'Field', - dataIndex: 'fieldDisplay' - }, - { - id: 'operator', - width: 57, - header: 'Operator', - dataIndex: 'operator' - }, - { - id: 'value', - width: 200, - header: 'Value', - dataIndex: 'value' - }, - { - xtype: 'actioncolumn', - width: 25, - items: [{ - icon: '../../../gui/images/delete.png', - handler: function (grid, rowIndex /*, colIndex*/ ) { - var record = grid.store.getAt(rowIndex); - self.fireEvent('remove_condition', record); - } - }] - } - - ] - }] - - } - ], - buttons: [{ - xtype: 'button', - text: 'Search', - id: 'job-viewer-search-search', - disabled: true, - handler: function ( /*button, event*/ ) { - var startDate = Ext.getCmp('search_start_date').getValue(); - var endDate = Ext.getCmp('search_end_date').getValue(); - var realmField = Ext.getCmp('realm-field'); - - var searchParams = {}; - var total = self.searchStore.getCount(); - for (var i = 0; i < total; i++) { - var searchParam = self.searchStore.getAt(i); - - var field = searchParam.get('field'); - var value = searchParam.get('valueId'); - - if (!searchParams[field]) { - searchParams[field] = []; - } - searchParams[field].push(value); - } - - var params = { - 'start_date': self._formatDate(startDate), - 'end_date': self._formatDate(endDate), - 'realm': realmField.getValue(), - 'limit': self._DEFAULT_PAGE_SIZE, - 'start': 0, - 'params': JSON.stringify(searchParams) - }; - self.fireEvent('search_requested', self, 'Search', params); - } - }] - }] - - }, - { - xtype: 'toolbar', - colspan: 2, - width: 1000, - height: 27, - minHeight: 27, - items: [ - '->', - { - xtype: 'button', - id: 'job-viewer-search-save-as', - text: 'Save As', - enabled: false, - tooltip: { - text: 'Save the selected results under a new name', - xtype: 'quicktip' - }, - handler: function (button, event, defaultText) { - var formattedDate = self._formatDate(new Date()); - var extension = Math.floor((Math.random() * 1024) + 1); - Ext.MessageBox.prompt('Pick Search Name', 'Please select a name with which to identify this collection of Search Criteria and it\'s associated results.', function (button, text) { - if (button === 'ok') { - var hasText = CCR.exists(text) && text.length >= 3; - - if (!hasText) { - Ext.MessageBox.show({ - title: 'Invalid Name', - msg: 'You must provide a name with which to identify these results if you wish to save them.', - icon: Ext.MessageBox.ERROR, - buttons: Ext.MessageBox.OK - }); - return; - } - - self._upsertSearchRecord(text); - } - }, this, false, defaultText || 'search-' + formattedDate + '-' + extension); - } - }, - { - xtype: 'button', - id: 'job-viewer-search-save-results', - text: 'Save Results', - tooltip: { - text: 'Select jobs from the results pane above and click here to save the jobs in the search history.', - xtype: "quicktip" + id: 'format', + header: 'Format', + dataIndex: 'format' }, - handler: function (button, event, defaultText) { - var title = Ext.getCmp('job-viewer-search-name').getValue(); - if (self.editing === true) { - title = title !== "" ? title : self.title; - self._upsertSearchRecord(title, self.dtypeId); - } else { - if (title) { - self._upsertSearchRecord(title); - } else { - var formattedDate = self._formatDate(new Date()); - var extension = Math.floor((Math.random() * 1024) + 1); - var title = defaultText || 'search-' + formattedDate + '-' + extension; - self._upsertSearchRecord(title); - } - } - } - }, - { - xtype: 'button', - id: 'job-viewer-search-cancel', - text: 'Cancel', - handler: function ( /*button, event*/ ) { - self.fireEvent('close_search', self, false); + { + id: 'state', + header: 'State', + dataIndex: 'state' } - } - ] - } - ]; - }, // getItems - - /** - * A helper function that takes care of either updating or inserting the current search - * record w/ the provided title. - * - * @param {String} title - * @param {String|null} id - **/ - _upsertSearchRecord: function (title, id) { - var self = this; - var selected = self._resultsToJSON(self._getSelectedRecords()); - - var params = { - text: Ext.util.Format.htmlEncode(title), - searchterms: { - params: self.resultsStore.searchParams - }, - results: selected - }; - - var realmField = Ext.getCmp('realm-field'); - var realm = realmField ? realmField.getValue() : null; - var idFragment = id !== undefined ? '/' + id : ''; - url: 'infill', - - Ext.Ajax.request({ - url: 'infill', - method: 'POST', - params: { - 'data': JSON.stringify(params) - }, - success: function (response) { - var exists = CCR.exists; - - var data = JSON.parse(response.responseText); - var success = exists(data) && data.success; - if (success) { - - var search = CCR.isType(data.results, CCR.Types.Array) ? data.results[0] : data.results; - var dtype = search['dtype']; - var value = search[dtype]; - - var newToken = ['realm=' + realm, dtype + '=' + value].join('&'); - var current = Ext.History.getToken(); - var token = CCR.tokenize(current); - var tab = token && token.tab ? token.tab : self.dataExport.id; - - Ext.History.add("#" + tab + '?' + newToken); - } - self.fireEvent('close_search', self, true); - }, - error: function ( /*response*/ ) { - Ext.MessageBox.show({ - title: 'Save Error', - msg: 'There was an error saving search [' + title + '].', - icon: Ext.MessageBox.ERROR, - buttons: Ext.MessageBox.OK - }); + ] } - }); - }, - - /** - * - * @param {Ext.data.Record} record - * @param {boolean|null} checked - **/ - _handleIncludeRecord: function (record, checked) { - var self = this; - var dtype = record.get('dtype'); - var id = record.get(dtype); - var exists = CCR.exists(self.selected[id]); - - if (checked && !exists) { - self.selected[id] = record; - } - if (!checked && exists) { - delete self.selected[id]; - } - - exists = CCR.exists(self.selected[id]); - if (checked && exists) { - record.set('included', true); - } - }, - - _dateFieldValidator: function (field_id, label) { - var validDates = { - startDateField: Date.parseDate(Ext.getCmp('search_start_date').getRawValue(), 'Y-m-d'), - endDateField: Date.parseDate(Ext.getCmp('search_end_date').getRawValue(), 'Y-m-d') - }; - if (validDates[field_id] === undefined) { - return 'Invalid ' + label + ' (must be of the form YYYY-MM-DD)'; - } - if (validDates.startDateField !== undefined && validDates.endDateField !== undefined) { - if (validDates.startDateField > validDates.endDateField) { - return 'Start date must be less than or equal to the end date'; - } - } - return true; - }, - - /** - * Load the Quick Job Lookup form with the provided searchTerms. - * - * @param {Object} searchTerms - */ - _editQuickSearch: function (searchTerms) { - var self = this; - - var params = JSON.parse(searchTerms.params); - - Ext.getCmp('basic-search-realm').setValue(searchTerms.realm); - Ext.getCmp('basic-localjobid').setValue(params.local_job_id); - - var resourceField = Ext.getCmp('basic-resource'); - resourceField.store.load({ - params: { - realm: searchTerms.realm - }, - callback: function () { - resourceField.setValue(params.resource_id); - self.fireEvent('validate_search_criteria'); - } + ] }); - }, - - /** - * Performs the steps necessary to get the SearchPanel ready for - * editing an 'Advanced Search'. - * - * @param {Object} searchTerms - */ - _editAdvancedSearch: function (searchTerms) { - var self = this; - - var startDateField = Ext.getCmp('search_start_date'); - var endDateField = Ext.getCmp('search_end_date'); - - var startDate = searchTerms.start_date; - var endDate = searchTerms.end_date; - - startDateField.setValue(startDate); - endDateField.setValue(endDate); - - var realmField = Ext.getCmp('realm-field'); - - var realm = searchTerms.realm; - realmField.setValue(searchTerms.realm); - - var clearSearchCriteria = function () { - var store = CCR.exists(self.searchStore) ? self.searchStore : null; - if (CCR.exists(store) && CCR.exists(store.removeAll)) { - store.removeAll(); - } - }; - - var addSearchCriteria = function () { - var value; - var i; - - var params = JSON.parse(searchTerms.params); - var advSearch = Ext.getCmp("job-viewer-advanced-search", { - removeMask: true - }); - var searchTermMask = new Ext.LoadMask(advSearch.el); - var shown = 0; - for (var field in params) { - if (params.hasOwnProperty(field)) { - for (i = 0; i < params[field].length; i++) { - - if (shown <= 0) { - searchTermMask.show(); - } - shown++; - - value = params[field][i]; - var displayPromises = [ - self._findFieldDisplay.call(self, field), - self._findFieldValueDisplay.call(self, field, value) - ]; - - /** - * Defines a partial function that will be supplied with - * the current values of field and value so that they - * will be in scope ( and hold the correct value ) - * when called by the promise. - * - * @param {String} field the id value of the field - * @param {String} value the id value of the value - * @param {Array} displayValues the return values - * from the promises. - * @returns {Function} to be executed after all of - * the display promises have - * resolved. - **/ - var partialThen = function (field, value, displayValues) { - return function (displayValues) { - var fieldDisplay = displayValues[0]; - var valueDisplay = displayValues[1]; - self.fireEvent('add_condition', realm, field, fieldDisplay, '=', valueDisplay, value); - if (--shown <= 0) { - searchTermMask.hide(); - } - }; - }; - RSVP - .all(displayPromises) - .then(partialThen(field, value)) - .catch(function (reason) { - CCR.error('Error retrieving criteria information', reason); - searchTermMask.hide(); - }); - } - } - } - }; - - clearSearchCriteria(); - addSearchCriteria(); - self.resetResults(); - }, - - /** - * Retrieves the value that should be displayed to the user for the provided - * 'field' value. - * - * @param {String} field - * @param {Function} callback - * - * @returns {RSVP.Promise} - */ - _findFieldDisplay: function (field) { - var self = this; - url: 'infill', - '/dimensions/' + field + '/name?token=' + XDMoD.REST.token; - - return this._getPromise(url, ['results', 'name']); - }, - - /** - * Retrieves the string that should be displayed to the user for the provided - * 'field' and 'value'. - * - * @param {String} field - * @param {String} value - * - * @return {RSVP.Promise} - */ - _findFieldValueDisplay: function (field, value) { - var self = this; - url: 'infill', - '/dimensions/' + field + - '/values/' + value + - '/name?token=' + XDMoD.REST.token; - - return this._getPromise(url, ['results', 'name']); - }, - - _getPromise: function (url, resolveProperties) { - return new RSVP.Promise( - function (resolve, reject) { - Ext.Ajax.request({ - url: 'infill', - success: function (response) { - var data = JSON.parse(response.responseText); - var success = data.success; - if (success === true) { - var resolveValue = data; - for (var i = 0; i < resolveProperties.length; i++) { - var property = resolveProperties[i]; - resolveValue = resolveValue[property]; - } - resolve(resolveValue); - } - }, - failure: function (response) { - var data = JSON.parse(response.responseText); - var message = data.message || 'An unknown error has occured.'; - reject(message); - } - }); - } - ); - }, - - /** - * Attempt to retrieve the already 'selected' values from the provided - * node. A 'selected' value is defined as an entry in this nodes childNodes - * property that contains a 'jobid' attribute. These are then used to - * mark the returned results such that it represents the current search - * state. - * - * @param {Ext.tree.AsyncTreeNode} node - **/ - _retrieveSelected: function (node) { - var self = this; - - /** - * - * @param {Array} selected - * @param {Array} idProperties - * @returns {Array} suitable to set 'this.children' to. - **/ - var processSelected = function (selected, idProperties) { - var results = []; - for (var i = 0; i < selected.length; i++) { - var item = selected[i]; - var id = item[idProperties[0]]; - for (var j = 1; j < idProperties.length; j++) { - id = id[idProperties[j]]; - } - results.push(id); - } - return results; - }; - - if (node.childNodes && node.childNodes.length && node.childNodes.length > 0) { - this.children = processSelected(node.childNodes, ['attributes', 'jobid']); - } else { - var realm = this._getNodeValue(this._getParentNode(node, 'realm'), 'realm'); - realm = realm !== null ? realm : ''; - - url: 'infill', - '/search/history/' + this.dtypeId + - '?realm=' + realm + - '&token=' + XDMoD.REST.token; - - this._getPromise(url, ['results']) - .then(function (results) { - self.children = processSelected(results, ['jobid']); - }); - } - }, - - /** - * Attempt to retrieve an 'id' property from the provided 'item'. An 'id' - * property is defined as one that can be found via first querying for - * a 'dtype' property. Then using that value to retrieve the 'id'. - * - * @param {Object} item the item to be used in retrieving the id property - * - * @return {null|*} returns null if no 'dtype' property can be found or - * if there is no 'get' method for the provided item. - * else it attempts to provide the value of - * item[item.dtype] or item.get(item.dtype) - **/ - _getId: function (item) { - if (item.dtype !== undefined) { - return item[item.dtype]; - } else if (item.get !== undefined) { - var dtype = item.get('dtype'); - return item.get(dtype); - } - return null; - }, - - _getTitle: function () { - return this.title; - }, - - /** - * If the condition evaluates to true then 'fn' is called with all of the - * arguments following 'elseFn'. If it evaluates to false then 'elseFn' - * is called with all of the arguments following 'elseFn'; - * - * @param {Boolean} condition - * @param {Function} fn - * @param {Function} elseFn - * - * @return {*} the return value of 'fn' if 'condition' is true else the - * return value of 'elseFn' - **/ - _onOrCondition: function (condition, fn, elseFn) { - var args = Array.prototype.slice.call(arguments, 3); - if (condition) { - return fn.apply(null, args); - } else { - return elseFn.apply(null, args); - } - }, - - /** - * Toggle the 'enabled' state of an {Ext.Component} based on the provided - * boolean 'condition'. - * - * @param {Boolean} condition a Boolean expression that will control - * whether the provided component is - * enabled or disabled. - * @param {Ext.Component} cmp the component to be enabled / disabled. - **/ - _toggle: function (condition, cmp) { - var enable = function (cmp) { - if (cmp && cmp.enable) { - cmp.enable(); - } - }; - - var disable = function (cmp) { - if (cmp && cmp.disable) { - cmp.disable(); - } - }; - - this._onOrCondition(condition, enable, disable, cmp); - }, - /** - * Attempts to follow the 'parentNode' links of the provided 'child' - * node until a parent is found that contains an attribute that matches - * the provided 'dtype'. - * - * @param {Ext.tree.AsyncTreeNode} child the node with which to start the - * search. - * @param {String} dtype the attribute to use in qualifying - * the parentNode. - * - * @return {null|Ext.tree.AsyncTreeNode} returns null if no parent is found - * or it returns the node identified - * by the provided 'dtype' by - * following the parentNode links of - * the provided 'child'. - **/ - _getParentNode: function (child, dtype) { - var parent = child.parentNode; - if (parent) { - var attributes = parent.attributes || {}; - if (attributes[dtype] !== undefined) { - return parent; - } - return this._getParentNode(parent, dtype); - } - return null; - }, - - /** - * Attempts to retrieve the value for the provided key 'attribute' from the - * specified 'node'. - * - * @param {Ext.tree.AsyncTreeNode} node - * @param {attribute} attribute - * - * @return {null|*} returns null if the attribute is not found. Else it - * returns the attribute value. - **/ - _getNodeValue: function (node, attribute) { - if (node && node.attributes && node.attributes[attribute] !== undefined) { - return node.attributes[attribute]; - } - return null; - }, - - _getDefaultStartDate: function () { - var start = new Date(); - start.setDate(start.getDate() - 7); - return start; - }, - - _getDefaultEndDate: function () { - var now = new Date(); - now.setDate(now.getDate() - 1); - return now; - }, - - _getFieldDisplayValue: function (field) { - return field && field.store ? - field.store.getById(field.getValue()).get(field.displayField) : - field && field.getValue ? - field.getValue() : - null; - }, - - /** - * Attempt to generate a default search name with the given parameters. - * - * @param {Object|Array} params the parameters to use in generating a - * default name - * @param {String} [prefix=''] an optional parameter that will be - * pre-pended to the output. - * @param {String} [delim='-'] an optional parameter that will be - * used to delimit the param values - * - * @return {String} - **/ - _generateDefaultName: function (params, prefix, delim) { - var args = []; - prefix = prefix !== undefined ? prefix : ''; - delim = delim !== undefined ? delim : '-'; - - args.push(prefix); - if (typeof params === 'object') { - for (var key in params) { - if (params.hasOwnProperty(key)) { - args.push(params[key]); - } - } - } else if (Array.isArray(params)) { - args = params; - } - return args - .filter(function (element, index, array) { - return element !== undefined && - element !== null && - element !== ''; - }) - .join(delim); + XDMoD.Module.DataExport.superclass.initComponent.call(this); } - }); - - -XDMoD.Module.DataExport.RealmCombo = Ext.extend(Ext.form.ComboBox, { - fieldLabel: 'Realm', - mode: 'local', - typeAhead: true, - triggerAction: 'all', - selectOnFocus: true, - forceSelection: true, - store: CCR.xdmod.ui.rawDataAllowedRealms, - value: CCR.xdmod.ui.rawDataAllowedRealms[0], - listeners: { - select: function (field) { - if (field.startValue !== field.getValue()) { - this.panel.fireEvent(this.realmSelectEvent, field.getValue()); - } - }, - blur: function (field) { - if (field.startValue !== field.getValue()) { - this.panel.fireEvent(this.realmSelectEvent, field.getValue()); - } - } - } - -}); //XDMoD.Module.DataExport From f01602c70343aa4e675269fd3de8cf029eb33e1a Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 16 May 2019 08:31:31 -0400 Subject: [PATCH 030/217] Replace tabs with spaces --- html/gui/js/modules/DataExport.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index bbe37c531a..6e976a255a 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -20,14 +20,14 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { ]; var requestStore = new Ext.data.ArrayStore({ - fields: [ - {name: 'realm'}, - {name: 'start_date'}, - {name: 'end_date'}, - {name: 'format'}, - {name: 'state'} - ] - }); + fields: [ + {name: 'realm'}, + {name: 'start_date'}, + {name: 'end_date'}, + {name: 'format'}, + {name: 'state'} + ] + }); requestStore.loadData(requestData); From 84dfd4e440c654bcab9a99515b3ca8d0c7448177 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 16 May 2019 09:32:11 -0400 Subject: [PATCH 031/217] Change style and add fieldset --- html/gui/js/modules/DataExport.js | 110 ++++++++++++++++-------------- 1 file changed, 59 insertions(+), 51 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 6e976a255a..40439058f2 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -38,59 +38,67 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { title: 'Request Data Export', region: 'west', width: 375, - layout: 'vbox', + bodyStyle:'padding:5px', items: [ { - xtype: 'combo', - fieldLabel: 'Realm', - name: 'realm', - forceSelection: true, - emptyText: 'Select a realm', - // TODO: Switch to remote. - mode: 'local', - valueField: 'id', - displayField: 'name', - store: new Ext.data.ArrayStore({ - fields: ['id', 'name'], - data: [ - ['jobs', 'Jobs'], - ['supremm', 'SUPReMM'], - ['accounts', 'Accounts'], - ['allocations', 'Allocations'], - ['requests', 'Requests'], - ['resource_allocations', 'Resource Allocations'] - ] - }) - }, - { - xtype: 'datefield', - fieldLabel: 'Start Date', - name: 'start_date' - }, - { - xtype: 'datefield', - fieldLabel: 'End Date', - name: 'end_date' - }, - { - xtype: 'combo', - fieldLabel: 'Format', - name: 'format', - mode: 'local', - emptyText: 'Select an export format', - valueField: 'id', - displayField: 'name', - store: new Ext.data.ArrayStore({ - fields: ['id', 'name'], - data: [ - ['csv', 'CSV'], - ['json', 'JSON'] - ] - }) - }, - { - xtype: 'button', - text: 'Submit Request' + xtype: 'fieldset', + columnWidth: 1, + items: [ + { + xtype: 'combo', + fieldLabel: 'Realm', + name: 'realm', + forceSelection: true, + emptyText: 'Select a realm', + // TODO: Switch to remote. + mode: 'local', + valueField: 'id', + displayField: 'name', + store: new Ext.data.ArrayStore({ + fields: ['id', 'name'], + data: [ + ['jobs', 'Jobs'], + ['supremm', 'SUPReMM'], + ['accounts', 'Accounts'], + ['allocations', 'Allocations'], + ['requests', 'Requests'], + ['resource_allocations', 'Resource Allocations'] + ] + }) + }, + { + xtype: 'datefield', + fieldLabel: 'Start Date', + name: 'start_date' + }, + { + xtype: 'datefield', + fieldLabel: 'End Date', + name: 'end_date' + }, + { + xtype: 'combo', + fieldLabel: 'Format', + name: 'format', + mode: 'local', + emptyText: 'Select an export format', + valueField: 'id', + displayField: 'name', + store: new Ext.data.ArrayStore({ + fields: ['id', 'name'], + data: [ + ['csv', 'CSV'], + ['json', 'JSON'] + ] + }) + } + ], + buttons: [ + { + xtype: 'button', + text: 'Submit Request' + } + ] } ] }, From 41424b1d0acbc82b31756ee96746119f6d6d9953 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 16 May 2019 12:58:53 -0400 Subject: [PATCH 032/217] Change data store, add empty text and more columns --- html/gui/js/modules/DataExport.js | 54 +++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 40439058f2..fb857da48a 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -15,22 +15,38 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { initComponent: function () { - var requestData = [ - ['Jobs', '2018-01-01', '2018-12-31', 'CSV', 'Pending'] - ]; - var requestStore = new Ext.data.ArrayStore({ fields: [ - {name: 'realm'}, - {name: 'start_date'}, - {name: 'end_date'}, - {name: 'format'}, - {name: 'state'} + 'realm', + 'start_date', + 'end_date', + 'format', + 'state', + 'requested_date', + 'expires_date' + ], + data: [ + [ + 'Jobs', + '2018-01-01', + '2018-12-31', + 'CSV', + 'Pending', + '2018-05-16', + null + ], + [ + 'Jobs', + '2017-01-01', + '2017-12-31', + 'CSV', + 'Available', + '2018-05-16', + '2018-07-01' + ] ] }); - requestStore.loadData(requestData); - Ext.apply(this, { items: [ { @@ -69,11 +85,13 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { { xtype: 'datefield', fieldLabel: 'Start Date', + emptyText: 'Start Date', name: 'start_date' }, { xtype: 'datefield', fieldLabel: 'End Date', + emptyText: 'End Date', name: 'end_date' }, { @@ -115,12 +133,12 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { }, { id: 'start_date', - header: 'Start Date', + header: 'Export Start Date', dataIndex: 'start_date' }, { id: 'end_date', - header: 'End Date', + header: 'Export End Date', dataIndex: 'end_date' }, { @@ -132,6 +150,16 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { id: 'state', header: 'State', dataIndex: 'state' + }, + { + id: 'requested_date', + header: 'Request Date', + dataIndex: 'requested_date' + }, + { + id: 'expiration_date', + header: 'Expiration Date', + dataIndex: 'expires_date' } ] } From c3810329c5cc571ea303e763b80a31a11af8f3d2 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 16 May 2019 14:13:31 -0400 Subject: [PATCH 033/217] Add controller provider --- .../WarehouseExportControllerProvider.php | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 classes/Rest/Controllers/WarehouseExportControllerProvider.php diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php new file mode 100644 index 0000000000..47296d0b4a --- /dev/null +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -0,0 +1,77 @@ +prefix; + + $controller + ->get( + "$root/warehouse/export/requests", + __CLASS__ . '::getRequests' + ); + + $controller + ->post( + "$root/warehouse/export/request", + __CLASS__ . '::createRequest' + ); + } + + /** + * Get all the existing export requests for the current user. + * + * @param Request $request + * @param Application $app + * @return array + * @throws AccessDeniedException + */ + public function getRequests(Request $request, Application $app) + { + $user = $this->authorize($request); + $handler = new QueryHandler(); + $results = $handler->listRequestsForUser($user->getId()); + return ['success' => true, 'results' => $results]; + } + + /** + * Create a new export request for the current user. + */ + public function createRequest(Request $request, Application $app) + { + $user = $this->authorize($request); + + // TODO: Validate input. + $realm = $this->getStringParam($request, 'realm'); + $startDate = $this->getStringParam($request, 'start_date'); + $endDate = $this->getStringParam($request, 'end_date'); + $format = $this->getStringParam($request, 'format'); + + $handler = new QueryHandler(); + + $handler->createRequestRecord( + $user->getId(), + $realm, + $startDate, + $endDate + ); + + return ['success' => true]; + } +} From 52f96d4eec72a55d0bfa0cde30cdc154f8629544 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 16 May 2019 14:55:05 -0400 Subject: [PATCH 034/217] Add grid title --- html/gui/js/modules/DataExport.js | 1 + 1 file changed, 1 insertion(+) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index fb857da48a..50f48cc1c7 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -122,6 +122,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { }, { xtype: 'grid', + title: 'Status of Export Requests', region: 'center', // TODO: Replace with remote store. store: requestStore, From ddcdc93ecf54f5c79435d40a288a9ecf8d8406d6 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 17 May 2019 07:33:46 -0400 Subject: [PATCH 035/217] Change text and add help tips --- html/gui/js/modules/DataExport.js | 35 +++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 50f48cc1c7..9f423ef836 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -31,7 +31,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { '2018-01-01', '2018-12-31', 'CSV', - 'Pending', + 'Submitted', '2018-05-16', null ], @@ -51,10 +51,16 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { items: [ { xtype: 'form', - title: 'Request Data Export', + title: 'Create Bulk Data Export Request', region: 'west', width: 375, bodyStyle:'padding:5px', + tools: [ + { + id: 'help', + qtip: XDMoD.Module.DataExport.createRequestHelpText + } + ], items: [ { xtype: 'fieldset', @@ -126,6 +132,12 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { region: 'center', // TODO: Replace with remote store. store: requestStore, + tools: [ + { + id: 'help', + qtip: XDMoD.Module.DataExport.exportStatusHelpText + } + ], columns: [ { id: 'realm', @@ -134,12 +146,12 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { }, { id: 'start_date', - header: 'Export Start Date', + header: 'Data Start Date', dataIndex: 'start_date' }, { id: 'end_date', - header: 'Export End Date', + header: 'Data End Date', dataIndex: 'end_date' }, { @@ -170,3 +182,18 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { XDMoD.Module.DataExport.superclass.initComponent.call(this); } }); + +XDMoD.Module.DataExport.createRequestHelpText = +'Create a new request to bulk export data from the data warehouse. Date ' + +'ranges are inclusive and are limited to one year. When the exported data is ' + +'ready you will receive an email notification.'; + +XDMoD.Module.DataExport.exportStatusHelpText = +'Bulk data export requests and their current statuses.

' + +'Status descriptions:
' + +'Submitted: Request has been submitted, but exported data is not yet ' + +'available.
' + +'Available: Requested data is available for download.
' + +'Expired: Requested data has expired and is no longer available.
' + +'Failed: Data export failed. Submit a support request for more ' + +'information.'; From aa75f83a92525d643d32f77251846ea08d92b465 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 17 May 2019 08:33:43 -0400 Subject: [PATCH 036/217] Add REST endpoint for batch export realms --- .../WarehouseExportControllerProvider.php | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 47296d0b4a..5322439742 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -21,6 +21,12 @@ public function setupRoutes( ) { $root = $this->prefix; + $controller + ->get( + "$root/warehouse/export/realms", + __CLASS__ . '::getRealms' + ); + $controller ->get( "$root/warehouse/export/requests", @@ -34,6 +40,31 @@ public function setupRoutes( ); } + /** + * Get all the realms available for exporting for the current user. + * + * @param Request $request + * @param Application $app + * @return array + * @throws AccessDeniedException + */ + public function getRealms(Request $request, Application $app) + { + $user = $this->authorize($request); + // TODO + return [ + 'success' => true, + 'results' => [ + ['id' => 'jobs', 'name' => 'Jobs'], + ['id' => 'supremm', 'name' => 'SUPReMM'], + ['id' => 'accounts', 'name' => 'Accounts'], + ['id' => 'allocations', 'name' => 'Allocations'], + ['id' => 'requests', 'name' => 'Requests'], + ['id' => 'resource_allocations', 'name' => 'Resource Allocations'] + ] + ]; + } + /** * Get all the existing export requests for the current user. * From 4c67ff2440498ac4652c609c3878fdd9cb762cb3 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 17 May 2019 09:03:13 -0400 Subject: [PATCH 037/217] Fix indentation and spacing --- html/gui/js/modules/DataExport.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 9f423ef836..db0fb374e6 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -17,13 +17,13 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { initComponent: function () { var requestStore = new Ext.data.ArrayStore({ fields: [ - 'realm', - 'start_date', - 'end_date', - 'format', - 'state', - 'requested_date', - 'expires_date' + 'realm', + 'start_date', + 'end_date', + 'format', + 'state', + 'requested_date', + 'expires_date' ], data: [ [ @@ -54,7 +54,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { title: 'Create Bulk Data Export Request', region: 'west', width: 375, - bodyStyle:'padding:5px', + bodyStyle: 'padding:5px', tools: [ { id: 'help', From 08e817a30e2c0a6a2bd2f43f0a756136bd39be95 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 17 May 2019 10:40:03 -0400 Subject: [PATCH 038/217] Add paging toolbar --- html/gui/js/modules/DataExport.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index db0fb374e6..895ba8ad9c 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -13,7 +13,6 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { */ _DEFAULT_PAGE_SIZE: 24, - initComponent: function () { var requestStore = new Ext.data.ArrayStore({ fields: [ @@ -174,7 +173,15 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { header: 'Expiration Date', dataIndex: 'expires_date' } - ] + ], + bbar: { + xtype: 'paging', + pageSize: this._DEFAULT_PAGE_SIZE, + displayInfo: true, + displayMsg: 'Displaying export requests {0} - {1} of {2}', + emptyMsg: 'No export requests to display', + store: requestStore + } } ] }); From 2bf991f14f08b0265ff949034d785cf49844c9dc Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 17 May 2019 12:43:55 -0400 Subject: [PATCH 039/217] Move data store out of component --- html/gui/js/modules/DataExport.js | 70 ++++++++++++++++--------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 895ba8ad9c..2e1977be8d 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -14,37 +14,8 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { _DEFAULT_PAGE_SIZE: 24, initComponent: function () { - var requestStore = new Ext.data.ArrayStore({ - fields: [ - 'realm', - 'start_date', - 'end_date', - 'format', - 'state', - 'requested_date', - 'expires_date' - ], - data: [ - [ - 'Jobs', - '2018-01-01', - '2018-12-31', - 'CSV', - 'Submitted', - '2018-05-16', - null - ], - [ - 'Jobs', - '2017-01-01', - '2017-12-31', - 'CSV', - 'Available', - '2018-05-16', - '2018-07-01' - ] - ] - }); + // TODO: Replace with JsonStore. + this.requestsStore = requestsStore; Ext.apply(this, { items: [ @@ -129,8 +100,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { xtype: 'grid', title: 'Status of Export Requests', region: 'center', - // TODO: Replace with remote store. - store: requestStore, + store: this.requestsStore, tools: [ { id: 'help', @@ -180,7 +150,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { displayInfo: true, displayMsg: 'Displaying export requests {0} - {1} of {2}', emptyMsg: 'No export requests to display', - store: requestStore + store: this.requestsStore } } ] @@ -204,3 +174,35 @@ XDMoD.Module.DataExport.exportStatusHelpText = 'Expired: Requested data has expired and is no longer available.
' + 'Failed: Data export failed. Submit a support request for more ' + 'information.'; + +var requestsStore = new Ext.data.ArrayStore({ + fields: [ + 'realm', + 'start_date', + 'end_date', + 'format', + 'state', + 'requested_date', + 'expires_date' + ], + data: [ + [ + 'Jobs', + '2018-01-01', + '2018-12-31', + 'CSV', + 'Submitted', + '2018-05-16', + null + ], + [ + 'Jobs', + '2017-01-01', + '2017-12-31', + 'CSV', + 'Available', + '2018-05-16', + '2018-07-01' + ] + ] +}); From 4d15ed5570ac84ca64370eb6528f10875459846e Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 17 May 2019 13:54:16 -0400 Subject: [PATCH 040/217] Refactor to separate components --- html/gui/js/modules/DataExport.js | 270 ++++++++++++++++-------------- 1 file changed, 147 insertions(+), 123 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 2e1977be8d..aced55539b 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -17,146 +17,170 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { // TODO: Replace with JsonStore. this.requestsStore = requestsStore; + this.requestForm = new XDMoD.Module.DataExport.RequestForm({ + region: 'west', + }); + + this.requestsGrid = new XDMoD.Module.DataExport.RequestsGrid({ + region: 'center', + pageSize: this._DEFAULT_PAGE_SIZE, + store: this.requestsStore + }); + + this.items = [ this.requestsGrid, this.requestForm ]; + + XDMoD.Module.DataExport.superclass.initComponent.call(this); + } +}); + +/** + * Data export request form. + */ +XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { + title: 'Create Bulk Data Export Request', + width: 375, + bodyStyle: 'padding:5px', + + initComponent: function () { Ext.apply(this, { + tools: [ + { + id: 'help', + qtip: XDMoD.Module.DataExport.createRequestHelpText + } + ], items: [ { - xtype: 'form', - title: 'Create Bulk Data Export Request', - region: 'west', - width: 375, - bodyStyle: 'padding:5px', - tools: [ - { - id: 'help', - qtip: XDMoD.Module.DataExport.createRequestHelpText - } - ], + xtype: 'fieldset', + columnWidth: 1, items: [ { - xtype: 'fieldset', - columnWidth: 1, - items: [ - { - xtype: 'combo', - fieldLabel: 'Realm', - name: 'realm', - forceSelection: true, - emptyText: 'Select a realm', - // TODO: Switch to remote. - mode: 'local', - valueField: 'id', - displayField: 'name', - store: new Ext.data.ArrayStore({ - fields: ['id', 'name'], - data: [ - ['jobs', 'Jobs'], - ['supremm', 'SUPReMM'], - ['accounts', 'Accounts'], - ['allocations', 'Allocations'], - ['requests', 'Requests'], - ['resource_allocations', 'Resource Allocations'] - ] - }) - }, - { - xtype: 'datefield', - fieldLabel: 'Start Date', - emptyText: 'Start Date', - name: 'start_date' - }, - { - xtype: 'datefield', - fieldLabel: 'End Date', - emptyText: 'End Date', - name: 'end_date' - }, - { - xtype: 'combo', - fieldLabel: 'Format', - name: 'format', - mode: 'local', - emptyText: 'Select an export format', - valueField: 'id', - displayField: 'name', - store: new Ext.data.ArrayStore({ - fields: ['id', 'name'], - data: [ - ['csv', 'CSV'], - ['json', 'JSON'] - ] - }) - } - ], - buttons: [ - { - xtype: 'button', - text: 'Submit Request' - } - ] - } - ] - }, - { - xtype: 'grid', - title: 'Status of Export Requests', - region: 'center', - store: this.requestsStore, - tools: [ - { - id: 'help', - qtip: XDMoD.Module.DataExport.exportStatusHelpText - } - ], - columns: [ - { - id: 'realm', - header: 'Realm', - dataIndex: 'realm' - }, - { - id: 'start_date', - header: 'Data Start Date', - dataIndex: 'start_date' - }, - { - id: 'end_date', - header: 'Data End Date', - dataIndex: 'end_date' - }, - { - id: 'format', - header: 'Format', - dataIndex: 'format' + xtype: 'combo', + fieldLabel: 'Realm', + name: 'realm', + forceSelection: true, + emptyText: 'Select a realm', + // TODO: Switch to remote. + mode: 'local', + valueField: 'id', + displayField: 'name', + store: new Ext.data.ArrayStore({ + fields: ['id', 'name'], + data: [ + ['jobs', 'Jobs'], + ['supremm', 'SUPReMM'], + ['accounts', 'Accounts'], + ['allocations', 'Allocations'], + ['requests', 'Requests'], + ['resource_allocations', 'Resource Allocations'] + ] + }) }, { - id: 'state', - header: 'State', - dataIndex: 'state' + xtype: 'datefield', + fieldLabel: 'Start Date', + emptyText: 'Start Date', + name: 'start_date' }, { - id: 'requested_date', - header: 'Request Date', - dataIndex: 'requested_date' + xtype: 'datefield', + fieldLabel: 'End Date', + emptyText: 'End Date', + name: 'end_date' }, { - id: 'expiration_date', - header: 'Expiration Date', - dataIndex: 'expires_date' + xtype: 'combo', + fieldLabel: 'Format', + name: 'format', + mode: 'local', + emptyText: 'Select an export format', + valueField: 'id', + displayField: 'name', + store: new Ext.data.ArrayStore({ + fields: ['id', 'name'], + data: [ + ['csv', 'CSV'], + ['json', 'JSON'] + ] + }) } ], - bbar: { - xtype: 'paging', - pageSize: this._DEFAULT_PAGE_SIZE, - displayInfo: true, - displayMsg: 'Displaying export requests {0} - {1} of {2}', - emptyMsg: 'No export requests to display', - store: this.requestsStore - } + buttons: [ + { + xtype: 'button', + text: 'Submit Request' + } + ] } ] }); - XDMoD.Module.DataExport.superclass.initComponent.call(this); + XDMoD.Module.DataExport.RequestForm.superclass.initComponent.call(this); + } +}); + +/** + * Data export request grid. + */ +XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { + title: 'Status of Export Requests', + + initComponent: function () { + Ext.apply(this, { + tools: [ + { + id: 'help', + qtip: XDMoD.Module.DataExport.exportStatusHelpText + } + ], + columns: [ + { + id: 'realm', + header: 'Realm', + dataIndex: 'realm' + }, + { + id: 'start_date', + header: 'Data Start Date', + dataIndex: 'start_date' + }, + { + id: 'end_date', + header: 'Data End Date', + dataIndex: 'end_date' + }, + { + id: 'format', + header: 'Format', + dataIndex: 'format' + }, + { + id: 'state', + header: 'State', + dataIndex: 'state' + }, + { + id: 'requested_date', + header: 'Request Date', + dataIndex: 'requested_date' + }, + { + id: 'expiration_date', + header: 'Expiration Date', + dataIndex: 'expires_date' + } + ], + bbar: { + xtype: 'paging', + store: this.store, + pageSize: this.pageSize, + displayInfo: true, + displayMsg: 'Displaying export requests {0} - {1} of {2}', + emptyMsg: 'No export requests to display', + } + }); + + XDMoD.Module.DataExport.RequestsGrid.superclass.initComponent.call(this); } }); From bba9322a360e125d515b568b6e9460d78e858a0e Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 17 May 2019 14:14:48 -0400 Subject: [PATCH 041/217] Change margins, etc. --- html/gui/js/modules/DataExport.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index aced55539b..c67d593ef2 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -19,10 +19,14 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { this.requestForm = new XDMoD.Module.DataExport.RequestForm({ region: 'west', + width: 375, + split: true, + margins: '2 0 2 2' }); this.requestsGrid = new XDMoD.Module.DataExport.RequestsGrid({ region: 'center', + margins: '2 2 2 0', pageSize: this._DEFAULT_PAGE_SIZE, store: this.requestsStore }); @@ -38,7 +42,6 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { */ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { title: 'Create Bulk Data Export Request', - width: 375, bodyStyle: 'padding:5px', initComponent: function () { From 54f024caee8e4123063cb49532ca0872edf666a5 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 17 May 2019 14:31:47 -0400 Subject: [PATCH 042/217] Fix style issues and add more example data --- html/gui/js/modules/DataExport.js | 91 +++++++++++++++++++------------ 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index c67d593ef2..b96a60a0f0 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -1,3 +1,54 @@ +// TODO: Replace this with a HTTP proxy JSON store. +var requestsStore = new Ext.data.ArrayStore({ + fields: [ + 'realm', + 'start_date', + 'end_date', + 'format', + 'state', + 'requested_date', + 'expires_date' + ], + data: [ + [ + 'Jobs', + '2018-01-01', + '2018-12-31', + 'CSV', + 'Submitted', + '2018-05-16', + null + ], + [ + 'SUPReMM', + '2017-01-01', + '2017-12-31', + 'CSV', + 'Available', + '2018-05-16', + '2018-07-01' + ], + [ + 'Jobs', + '2016-01-01', + '2016-12-31', + 'CSV', + 'Expired', + '2018-01-01', + '2018-05-01', + ], + [ + 'Jobs', + '2018-01-01', + '2018-12-31', + 'JSON', + 'Failed', + '2018-05-16', + null + ] + ] +}); + /** * Data warehouse export module. */ @@ -11,7 +62,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * * @var {Number} */ - _DEFAULT_PAGE_SIZE: 24, + defaultPageSize: 24, initComponent: function () { // TODO: Replace with JsonStore. @@ -27,11 +78,11 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { this.requestsGrid = new XDMoD.Module.DataExport.RequestsGrid({ region: 'center', margins: '2 2 2 0', - pageSize: this._DEFAULT_PAGE_SIZE, + pageSize: this.defaultPageSize, store: this.requestsStore }); - this.items = [ this.requestsGrid, this.requestForm ]; + this.items = [this.requestsGrid, this.requestForm]; XDMoD.Module.DataExport.superclass.initComponent.call(this); } @@ -179,7 +230,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { pageSize: this.pageSize, displayInfo: true, displayMsg: 'Displaying export requests {0} - {1} of {2}', - emptyMsg: 'No export requests to display', + emptyMsg: 'No export requests to display' } }); @@ -201,35 +252,3 @@ XDMoD.Module.DataExport.exportStatusHelpText = 'Expired: Requested data has expired and is no longer available.
' + 'Failed: Data export failed. Submit a support request for more ' + 'information.'; - -var requestsStore = new Ext.data.ArrayStore({ - fields: [ - 'realm', - 'start_date', - 'end_date', - 'format', - 'state', - 'requested_date', - 'expires_date' - ], - data: [ - [ - 'Jobs', - '2018-01-01', - '2018-12-31', - 'CSV', - 'Submitted', - '2018-05-16', - null - ], - [ - 'Jobs', - '2017-01-01', - '2017-12-31', - 'CSV', - 'Available', - '2018-05-16', - '2018-07-01' - ] - ] -}); From ade14cfb531416ac6a55cd8893354c8712300a95 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 17 May 2019 14:37:56 -0400 Subject: [PATCH 043/217] Remove trailing comma --- html/gui/js/modules/DataExport.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index b96a60a0f0..147e710eb0 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -35,7 +35,7 @@ var requestsStore = new Ext.data.ArrayStore({ 'CSV', 'Expired', '2018-01-01', - '2018-05-01', + '2018-05-01' ], [ 'Jobs', From d0f24b7e6eeae8b4e14e17d07a4762e3de9de2ad Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 20 May 2019 08:12:42 -0400 Subject: [PATCH 044/217] Add REST configuration and fix controller --- .../WarehouseExportControllerProvider.php | 50 ++++++++----------- configuration/rest.d/warehouse_export.json | 6 +++ 2 files changed, 27 insertions(+), 29 deletions(-) create mode 100644 configuration/rest.d/warehouse_export.json diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 5322439742..d8d9a1c397 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -20,24 +20,10 @@ public function setupRoutes( ControllerCollection $controller ) { $root = $this->prefix; - - $controller - ->get( - "$root/warehouse/export/realms", - __CLASS__ . '::getRealms' - ); - - $controller - ->get( - "$root/warehouse/export/requests", - __CLASS__ . '::getRequests' - ); - - $controller - ->post( - "$root/warehouse/export/request", - __CLASS__ . '::createRequest' - ); + $current = get_class($this); + $controller->get("$root/realms", "$current::getRealms"); + $controller->get("$root/requests", "$current::getRequests"); + $controller->post("$root/request", "$current::createRequest"); } /** @@ -51,18 +37,24 @@ public function setupRoutes( public function getRealms(Request $request, Application $app) { $user = $this->authorize($request); - // TODO - return [ - 'success' => true, - 'results' => [ - ['id' => 'jobs', 'name' => 'Jobs'], - ['id' => 'supremm', 'name' => 'SUPReMM'], - ['id' => 'accounts', 'name' => 'Accounts'], - ['id' => 'allocations', 'name' => 'Allocations'], - ['id' => 'requests', 'name' => 'Requests'], - ['id' => 'resource_allocations', 'name' => 'Resource Allocations'] - ] + + // TODO: Determine realms using user ACLs. + $realms = [ + ['id' => 'jobs', 'name' => 'Jobs'], + ['id' => 'supremm', 'name' => 'SUPReMM'], + ['id' => 'accounts', 'name' => 'Accounts'], + ['id' => 'allocations', 'name' => 'Allocations'], + ['id' => 'requests', 'name' => 'Requests'], + ['id' => 'resource_allocations', 'name' => 'Resource Allocations'] ]; + + return $app->json( + [ + 'success' => true, + 'data' => $realms, + 'total' => count($realms) + ] + ); } /** diff --git a/configuration/rest.d/warehouse_export.json b/configuration/rest.d/warehouse_export.json new file mode 100644 index 0000000000..f080b22b00 --- /dev/null +++ b/configuration/rest.d/warehouse_export.json @@ -0,0 +1,6 @@ +{ + "warehouse_export": { + "prefix": "warehouse/export", + "controller": "Rest\\Controllers\\WarehouseExportControllerProvider" + } +} From 9ea097e4fc02237a096a799b177efa6312f63385 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 20 May 2019 08:41:38 -0400 Subject: [PATCH 045/217] Use remote realm list, update various options --- html/gui/js/modules/DataExport.js | 42 ++++++++++++++++++------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 147e710eb0..36b3f93225 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -110,46 +110,54 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { items: [ { xtype: 'combo', - fieldLabel: 'Realm', name: 'realm', - forceSelection: true, + fieldLabel: 'Format', emptyText: 'Select a realm', - // TODO: Switch to remote. - mode: 'local', valueField: 'id', displayField: 'name', - store: new Ext.data.ArrayStore({ + allowBlank: false, + editable: false, + triggerAction: 'all', + mode: 'local', + store: new Ext.data.JsonStore({ + autoLoad: true, + autoDestroy: true, + root: 'data', fields: ['id', 'name'], - data: [ - ['jobs', 'Jobs'], - ['supremm', 'SUPReMM'], - ['accounts', 'Accounts'], - ['allocations', 'Allocations'], - ['requests', 'Requests'], - ['resource_allocations', 'Resource Allocations'] - ] + proxy: new Ext.data.HttpProxy({ + method: 'GET', + url: 'rest/v1/warehouse/export/realms' + }) }) }, { xtype: 'datefield', + name: 'start_date', fieldLabel: 'Start Date', emptyText: 'Start Date', - name: 'start_date' + format: 'Y-m-d', + allowBlank: false }, { xtype: 'datefield', + name: 'end_date', fieldLabel: 'End Date', emptyText: 'End Date', - name: 'end_date' + format: 'Y-m-d', + allowBlank: false }, { xtype: 'combo', - fieldLabel: 'Format', name: 'format', - mode: 'local', + fieldLabel: 'Format', emptyText: 'Select an export format', valueField: 'id', displayField: 'name', + mode: 'local', + editable: false, + lazyInit: false, + typeAhead: true, + triggerAction: 'all', store: new Ext.data.ArrayStore({ fields: ['id', 'name'], data: [ From cc15f782812fa347eaca18530106354826e24adb Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 20 May 2019 10:23:17 -0400 Subject: [PATCH 046/217] Fix typo --- html/gui/js/modules/DataExport.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 36b3f93225..3a0c751b85 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -111,7 +111,7 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { { xtype: 'combo', name: 'realm', - fieldLabel: 'Format', + fieldLabel: 'Realm', emptyText: 'Select a realm', valueField: 'id', displayField: 'name', From e4555f0ce4ce958a035b0fe80310f7a3b8c86eda Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 21 May 2019 12:02:47 -0400 Subject: [PATCH 047/217] Add ID field and types to data store --- html/gui/js/modules/DataExport.js | 47 ++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 3a0c751b85..22f8ce5054 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -1,16 +1,46 @@ // TODO: Replace this with a HTTP proxy JSON store. var requestsStore = new Ext.data.ArrayStore({ fields: [ - 'realm', - 'start_date', - 'end_date', - 'format', - 'state', - 'requested_date', - 'expires_date' + { + name: 'id', + type: 'int' + }, + { + name: 'realm', + type: 'string' + }, + { + name: 'start_date', + type: 'date', + dateFormat: 'Y-m-d' + }, + { + name: 'end_date', + type: 'date', + dateFormat: 'Y-m-d' + }, + { + name: 'format', + type: 'string' + }, + { + name: 'state', + type: 'string' + }, + { + name: 'requested_date', + type: 'date', + dateFormat: 'Y-m-d' + }, + { + name: 'expires_date', + type: 'date', + dateFormat: 'Y-m-d' + } ], data: [ [ + 1, 'Jobs', '2018-01-01', '2018-12-31', @@ -20,6 +50,7 @@ var requestsStore = new Ext.data.ArrayStore({ null ], [ + 2, 'SUPReMM', '2017-01-01', '2017-12-31', @@ -29,6 +60,7 @@ var requestsStore = new Ext.data.ArrayStore({ '2018-07-01' ], [ + 3, 'Jobs', '2016-01-01', '2016-12-31', @@ -38,6 +70,7 @@ var requestsStore = new Ext.data.ArrayStore({ '2018-05-01' ], [ + 4, 'Jobs', '2018-01-01', '2018-12-31', From 9da63e85c9249df19a45fca70973720977d070b1 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 24 May 2019 07:58:09 -0400 Subject: [PATCH 048/217] Rearrange columns and add button to bottom bar --- html/gui/js/modules/DataExport.js | 78 +++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 24 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 22f8ce5054..2a365f8aca 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -230,49 +230,79 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { ], columns: [ { - id: 'realm', + header: 'Request Date', + dataIndex: 'requested_date', + xtype: 'datecolumn', + format: 'Y-m-d' + }, + { + header: 'State', + dataIndex: 'state' + }, + { header: 'Realm', dataIndex: 'realm' }, { - id: 'start_date', header: 'Data Start Date', - dataIndex: 'start_date' + dataIndex: 'start_date', + xtype: 'datecolumn', + format: 'Y-m-d' }, { - id: 'end_date', header: 'Data End Date', - dataIndex: 'end_date' + dataIndex: 'end_date', + xtype: 'datecolumn', + format: 'Y-m-d' }, { - id: 'format', header: 'Format', dataIndex: 'format' }, { - id: 'state', - header: 'State', - dataIndex: 'state' + header: 'Expiration Date', + dataIndex: 'expires_date', + xtype: 'datecolumn', + format: 'Y-m-d' }, { - id: 'requested_date', - header: 'Request Date', - dataIndex: 'requested_date' + header: 'Actions', + xtype: 'templatecolumn', + tpl: new Ext.XTemplate( + '', + ' ', + ' ', + '' + ) + } + ], + bbar: [ + { + xtype: 'button', + text: 'Delete all expired requests', + handler: function () { + Ext.Msg.confirm( + 'Delete All Expired Requests', + 'Are you sure that you want to delete all expired requests? You cannot undo this operation.', + function (selection) { + if (selection === 'yes') { + Ext.Msg.alert('TODO', 'TODO: Delete all the expired requests'); + } + }, + this + ); + } }, + '->', { - id: 'expiration_date', - header: 'Expiration Date', - dataIndex: 'expires_date' + xtype: 'paging', + store: this.store, + pageSize: this.pageSize, + displayInfo: true, + displayMsg: 'Displaying export requests {0} - {1} of {2}', + emptyMsg: 'No export requests to display' } - ], - bbar: { - xtype: 'paging', - store: this.store, - pageSize: this.pageSize, - displayInfo: true, - displayMsg: 'Displaying export requests {0} - {1} of {2}', - emptyMsg: 'No export requests to display' - } + ] }); XDMoD.Module.DataExport.RequestsGrid.superclass.initComponent.call(this); From 1262706819e8010d8065fb7336b396571c4f7097 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 24 May 2019 09:45:22 -0400 Subject: [PATCH 049/217] Adjust combo options --- html/gui/js/modules/DataExport.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 2a365f8aca..1fcfb72d87 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -186,11 +186,10 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { emptyText: 'Select an export format', valueField: 'id', displayField: 'name', - mode: 'local', + allowBlank: false, editable: false, - lazyInit: false, - typeAhead: true, triggerAction: 'all', + mode: 'local', store: new Ext.data.ArrayStore({ fields: ['id', 'name'], data: [ From 5e2426fcaecd1a676bb5e3582c2af224c117ae1d Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 24 May 2019 11:00:48 -0400 Subject: [PATCH 050/217] Update get_tabs test output --- .../xdmod/user_admin/output/get_tabs-admin.json | 11 +++++++++++ .../user_admin/output/get_tabs-centerdirector.json | 11 +++++++++++ .../xdmod/user_admin/output/get_tabs-centerstaff.json | 11 +++++++++++ .../xdmod/user_admin/output/get_tabs-normaluser.json | 11 +++++++++++ .../xdmod/user_admin/output/get_tabs-principal.json | 11 +++++++++++ .../xdmod/user_admin/output/get_tabs-public_user.json | 11 +++++++++++ .../output/get_tabs-test.cd.one-center.json | 11 +++++++++++ .../output/get_tabs-test.cs.one-center.json | 11 +++++++++++ .../user_admin/output/get_tabs-test.normal-user.json | 11 +++++++++++ .../xdmod/user_admin/output/get_tabs-test.pi.json | 11 +++++++++++ .../user_admin/output/get_tabs-test.usr_dev.json | 11 +++++++++++ .../user_admin/output/get_tabs-test.usr_mgr.json | 11 +++++++++++ .../user_admin/output/get_tabs-test.usr_mgr_dev.json | 11 +++++++++++ 13 files changed, 143 insertions(+) diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json index ddc6d36311..c7939cd409 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 400, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json index ddc6d36311..c7939cd409 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 400, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json index ddc6d36311..c7939cd409 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 400, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json index ddc6d36311..c7939cd409 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 400, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json index ddc6d36311..c7939cd409 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 400, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-public_user.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-public_user.json index ff7b3dc38f..30b89fd3a0 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-public_user.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-public_user.json @@ -21,6 +21,17 @@ "tooltip": "Displays usage", "userManualSectionName": "Usage Tab" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 400, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "", + "userManualSectionName": "Data Export" + }, { "tab": "about_xdmod", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json index ddc6d36311..c7939cd409 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 400, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json index ddc6d36311..c7939cd409 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 400, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json index ddc6d36311..c7939cd409 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 400, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json index ddc6d36311..c7939cd409 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 400, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json index ddc6d36311..c7939cd409 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 400, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json index ddc6d36311..c7939cd409 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 400, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json index ddc6d36311..c7939cd409 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 400, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, From cbd1a06919237b8f3600c7b9abcc0105a149603c Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 24 May 2019 12:11:41 -0400 Subject: [PATCH 051/217] Fix actions --- html/gui/js/modules/DataExport.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 1fcfb72d87..aefa60970f 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -269,9 +269,8 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { xtype: 'templatecolumn', tpl: new Ext.XTemplate( '', - ' ', - ' ', - '' + ' ', + ' ' ) } ], From 29401cd969bed0d5d5e45389f41b92128457ace7 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 24 May 2019 12:13:22 -0400 Subject: [PATCH 052/217] Fix test output --- .../xdmod/user_admin/output/get_tabs-public_user.json | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-public_user.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-public_user.json index 30b89fd3a0..ff7b3dc38f 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-public_user.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-public_user.json @@ -21,17 +21,6 @@ "tooltip": "Displays usage", "userManualSectionName": "Usage Tab" }, - { - "tab": "data_export", - "isDefault": false, - "title": "Data Export", - "pos": 400, - "permitted_modules": null, - "javascriptClass": "XDMoD.Module.DataExport", - "javascriptReference": "CCR.xdmod.ui.dataExport", - "tooltip": "", - "userManualSectionName": "Data Export" - }, { "tab": "about_xdmod", "isDefault": false, From 1cbe6c80384471534bcf9d3beddc52b7f9801389 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 24 May 2019 13:57:00 -0400 Subject: [PATCH 053/217] Fix typos --- .../Rest/Controllers/WarehouseExportControllerProvider.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index d8d9a1c397..625822fc88 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -69,7 +69,7 @@ public function getRequests(Request $request, Application $app) { $user = $this->authorize($request); $handler = new QueryHandler(); - $results = $handler->listRequestsForUser($user->getId()); + $results = $handler->listRequestsForUser($user->getUserId()); return ['success' => true, 'results' => $results]; } @@ -89,7 +89,7 @@ public function createRequest(Request $request, Application $app) $handler = new QueryHandler(); $handler->createRequestRecord( - $user->getId(), + $user->getUserId(), $realm, $startDate, $endDate From 7d3a1d4dbf6c7d7fa2ff090e78bd4f70a94965cb Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 24 May 2019 14:00:24 -0400 Subject: [PATCH 054/217] Fix return values --- .../Rest/Controllers/WarehouseExportControllerProvider.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 625822fc88..cb624bcdf4 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -70,7 +70,7 @@ public function getRequests(Request $request, Application $app) $user = $this->authorize($request); $handler = new QueryHandler(); $results = $handler->listRequestsForUser($user->getUserId()); - return ['success' => true, 'results' => $results]; + return $app->json(['success' => true, 'results' => $results]); } /** @@ -95,6 +95,6 @@ public function createRequest(Request $request, Application $app) $endDate ); - return ['success' => true]; + return $app->json(['success' => true]); } } From 9d07d19e01d3b407467b8364d91475aa2ed6cbfe Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 24 May 2019 14:36:10 -0400 Subject: [PATCH 055/217] Add JSON store and update fields --- .../WarehouseExportControllerProvider.php | 2 +- html/gui/js/modules/DataExport.js | 104 +++++++++--------- 2 files changed, 50 insertions(+), 56 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index cb624bcdf4..ced264aa46 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -70,7 +70,7 @@ public function getRequests(Request $request, Application $app) $user = $this->authorize($request); $handler = new QueryHandler(); $results = $handler->listRequestsForUser($user->getUserId()); - return $app->json(['success' => true, 'results' => $results]); + return $app->json(['success' => true, 'data' => $results]); } /** diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index aefa60970f..794f04d289 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -1,5 +1,5 @@ -// TODO: Replace this with a HTTP proxy JSON store. -var requestsStore = new Ext.data.ArrayStore({ +var requestsStore = new Ext.data.JsonStore({ + root: 'data', fields: [ { name: 'id', @@ -20,66 +20,44 @@ var requestsStore = new Ext.data.ArrayStore({ dateFormat: 'Y-m-d' }, { - name: 'format', + name: 'export_file_format', type: 'string' }, { - name: 'state', - type: 'string' + name: 'requested_datetime', + type: 'date', + dateFormat: 'Y-m-d H:i:s' }, { - name: 'requested_date', + name: 'export_created_datetime', type: 'date', - dateFormat: 'Y-m-d' + dateFormat: 'Y-m-d H:i:s' }, { - name: 'expires_date', + name: 'export_expires_datetime', type: 'date', - dateFormat: 'Y-m-d' + dateFormat: 'Y-m-d H:i:s' + }, + { + name: 'export_expired', + type: 'boolean' + }, + { + name: 'state', + convert: function (v, record) { + // TODO + if (record.export_expired == '1') { + return 'Expired'; + } + + return 'Submitted'; + } } ], - data: [ - [ - 1, - 'Jobs', - '2018-01-01', - '2018-12-31', - 'CSV', - 'Submitted', - '2018-05-16', - null - ], - [ - 2, - 'SUPReMM', - '2017-01-01', - '2017-12-31', - 'CSV', - 'Available', - '2018-05-16', - '2018-07-01' - ], - [ - 3, - 'Jobs', - '2016-01-01', - '2016-12-31', - 'CSV', - 'Expired', - '2018-01-01', - '2018-05-01' - ], - [ - 4, - 'Jobs', - '2018-01-01', - '2018-12-31', - 'JSON', - 'Failed', - '2018-05-16', - null - ] - ] + proxy: new Ext.data.HttpProxy({ + method: 'GET', + url: 'rest/v1/warehouse/export/requests' + }) }); /** @@ -98,9 +76,10 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { defaultPageSize: 24, initComponent: function () { - // TODO: Replace with JsonStore. this.requestsStore = requestsStore; + this.on('afterrender', this.requestsStore.load, this.requestsStore); + this.requestForm = new XDMoD.Module.DataExport.RequestForm({ region: 'west', width: 375, @@ -202,7 +181,22 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { buttons: [ { xtype: 'button', - text: 'Submit Request' + text: 'Submit Request', + scope: this, + handler: function () { + Ext.Ajax.request({ + url: 'rest/v1/warehouse/export/request', + method: 'POST', + params: this.getForm().getValues(), + scope: this, + success: function (response) { + // TODO + }, + failure: function (response) { + // TODO + } + }); + } } ] } @@ -230,7 +224,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { columns: [ { header: 'Request Date', - dataIndex: 'requested_date', + dataIndex: 'requested_datetime', xtype: 'datecolumn', format: 'Y-m-d' }, @@ -256,7 +250,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { }, { header: 'Format', - dataIndex: 'format' + dataIndex: 'export_file_format' }, { header: 'Expiration Date', From 53a516508e0316ca5026f7cab6785b7b185045b0 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 24 May 2019 14:52:14 -0400 Subject: [PATCH 056/217] Fix comparison --- html/gui/js/modules/DataExport.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 794f04d289..4456ce3ced 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -46,7 +46,7 @@ var requestsStore = new Ext.data.JsonStore({ name: 'state', convert: function (v, record) { // TODO - if (record.export_expired == '1') { + if (record.export_expired === '1') { return 'Expired'; } From 97f9b8081d4063b0527f81836191670b06f598e8 Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Fri, 24 May 2019 16:04:16 -0400 Subject: [PATCH 057/217] Removed call to pdo quote function on insert of new record. --- classes/DataWarehouse/Export/QueryHandler.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index d1e5992783..d682ba1a91 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -48,7 +48,7 @@ public function createRequestRecord($userId, $realm, $startDate, $endDate) (NOW(), :user_id, :realm, :start_date, :end_date)"; $params = array('user_id' => $userId, - 'realm' => $this->pdo->quote($realm), + 'realm' => $this->pdo->$realm, 'start_date' => $startDate, 'end_date' => $endDate); From 5318215c71e2730400f27bae0bcb609d5ebed02e Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 28 May 2019 10:26:29 -0400 Subject: [PATCH 058/217] Refactor data store --- html/gui/js/modules/DataExport.js | 137 ++++++++++++++++-------------- 1 file changed, 75 insertions(+), 62 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 4456ce3ced..abbe3cf7dd 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -1,64 +1,4 @@ -var requestsStore = new Ext.data.JsonStore({ - root: 'data', - fields: [ - { - name: 'id', - type: 'int' - }, - { - name: 'realm', - type: 'string' - }, - { - name: 'start_date', - type: 'date', - dateFormat: 'Y-m-d' - }, - { - name: 'end_date', - type: 'date', - dateFormat: 'Y-m-d' - }, - { - name: 'export_file_format', - type: 'string' - }, - { - name: 'requested_datetime', - type: 'date', - dateFormat: 'Y-m-d H:i:s' - }, - { - name: 'export_created_datetime', - type: 'date', - dateFormat: 'Y-m-d H:i:s' - }, - { - name: 'export_expires_datetime', - type: 'date', - dateFormat: 'Y-m-d H:i:s' - }, - { - name: 'export_expired', - type: 'boolean' - }, - { - name: 'state', - convert: function (v, record) { - // TODO - if (record.export_expired === '1') { - return 'Expired'; - } - - return 'Submitted'; - } - } - ], - proxy: new Ext.data.HttpProxy({ - method: 'GET', - url: 'rest/v1/warehouse/export/requests' - }) -}); +Ext.ns('XDMoD.Module.DataExport'); /** * Data warehouse export module. @@ -76,7 +16,7 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { defaultPageSize: 24, initComponent: function () { - this.requestsStore = requestsStore; + this.requestsStore = new XDMoD.Module.DataExport.RequestsStore(); this.on('afterrender', this.requestsStore.load, this.requestsStore); @@ -315,3 +255,76 @@ XDMoD.Module.DataExport.exportStatusHelpText = 'Expired: Requested data has expired and is no longer available.
' + 'Failed: Data export failed. Submit a support request for more ' + 'information.'; + +/** + * Data warehouse export batch requests data store. + */ +XDMoD.Module.DataExport.RequestsStore = Ext.extend(Ext.data.JsonStore, { + constructor: function (config) { + config = config || {}; + Ext.apply(config, { + root: 'data', + fields: [ + { + name: 'id', + type: 'int' + }, + { + name: 'realm', + type: 'string' + }, + { + name: 'start_date', + type: 'date', + dateFormat: 'Y-m-d' + }, + { + name: 'end_date', + type: 'date', + dateFormat: 'Y-m-d' + }, + { + name: 'export_file_format', + type: 'string' + }, + { + name: 'requested_datetime', + type: 'date', + dateFormat: 'Y-m-d H:i:s' + }, + { + name: 'export_created_datetime', + type: 'date', + dateFormat: 'Y-m-d H:i:s' + }, + { + name: 'export_expires_datetime', + type: 'date', + dateFormat: 'Y-m-d H:i:s' + }, + { + name: 'export_expired', + type: 'boolean' + }, + { + name: 'state', + convert: function (v, record) { + // TODO + return 'Available'; + if (record.export_expired === '1') { + return 'Expired'; + } + + return 'Submitted'; + } + } + ], + proxy: new Ext.data.HttpProxy({ + method: 'GET', + url: 'rest/v1/warehouse/export/requests' + }) + }); + + XDMoD.Module.DataExport.RequestsStore.superclass.constructor.call(this, config); + } +}); From c5e3f25fe40667159ea57c5cb104f6ebe6de091a Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Tue, 28 May 2019 10:43:27 -0400 Subject: [PATCH 059/217] Getting tests running cleanly for arbitrary user (must exist in Users!) --- classes/DataWarehouse/Export/QueryHandler.php | 2 +- tests/component/lib/Export/ExportDBTest.php | 27 ++++++++++++++----- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index d682ba1a91..d1e5992783 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -48,7 +48,7 @@ public function createRequestRecord($userId, $realm, $startDate, $endDate) (NOW(), :user_id, :realm, :start_date, :end_date)"; $params = array('user_id' => $userId, - 'realm' => $this->pdo->$realm, + 'realm' => $this->pdo->quote($realm), 'start_date' => $startDate, 'end_date' => $endDate); diff --git a/tests/component/lib/Export/ExportDBTest.php b/tests/component/lib/Export/ExportDBTest.php index 6d8248a033..8aaf5e588d 100644 --- a/tests/component/lib/Export/ExportDBTest.php +++ b/tests/component/lib/Export/ExportDBTest.php @@ -5,6 +5,7 @@ use CCR\DB; use \Exception; use \DataWarehouse\Export\QueryHandler; +use \XDUser; class ExportDBTest extends BaseTest @@ -14,8 +15,8 @@ class ExportDBTest extends BaseTest { static $dbh = null; static $maxId = null; - private static $userId = 34; // Tom Furlani - private static $debug = FALSE; + private static $userName = self::NORMAL_USER_USER_NAME; + private static $debug = TRUE; public function testCountSubmitted() { @@ -28,14 +29,24 @@ public function testCountSubmitted() if (self::$debug) print("\n".__FUNCTION__.": submittedRecords=$submittedCount\n"); } + private function acquireUserId() + { + $userId = static::$dbh->query('SELECT MIN(id) AS id FROM Users')[0]['id']; + if ($userId == NULL) { + $userId = XDUser::getUserByUserName(self::$userName); + } + return $userId; + } + private function findSubmittedRecord() { $query = new QueryHandler(); + $userId = $this->acquireUserId(); //XDUser::getUserByUserName(self::$userName); // Find or create a record in submitted status to transition $maxSubmitted = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where export_succeeded IS NULL')[0]['id']; if ($maxSubmitted == NULL) { - $maxSubmitted = $query->createRequestRecord(self::$userId, 'Jobs', '2017-01-01','2017-08-01'); + $maxSubmitted = $query->createRequestRecord($userId, 'Jobs', '2017-01-01','2017-08-01'); } if (self::$debug) print("\n".__FUNCTION__.": maxSubmitted ID=$maxSubmitted\n"); @@ -47,16 +58,18 @@ private function findSubmittedRecord() public function testNewRecordCreation() { $query = new QueryHandler(); + $userId = $this->acquireUserId(); //XDUser::getUserByUserName(self::$userName); + //if (self::$debug) print("\n".__FUNCTION__.": userName=".self::$userName." userId=$userId\n"); // find the count $initialCount = $query->countSubmittedRecords(); // add new record and verify - $requestId = $query->createRequestRecord(self::$userId, 'Jobs', '2019-01-01','2019-03-01'); + $requestId = $query->createRequestRecord($userId, 'Jobs', '2019-01-01','2019-03-01'); $this->assertNotNull($requestId); // add another new record and verify - $requestId2 = $query->createRequestRecord(self::$userId, 'Accounts', '2016-12-01','2017-01-01'); + $requestId2 = $query->createRequestRecord($userId, 'Accounts', '2016-12-01','2017-01-01'); $this->assertNotNull($requestId2 ); $this->assertTrue($requestId2-$requestId==1); @@ -169,6 +182,7 @@ public function testSubmittedRecordFieldList() public function testUserRecordFieldList() { $query = new QueryHandler(); + $userId = $this->acquireUserId(); //XDUser::getUserByUserName(self::$userName); // Expect these keys from the associative array $expectedKeys = array( @@ -183,7 +197,7 @@ public function testUserRecordFieldList() ); // Requests via this user have been created as part of these tests - $actual = $query->listRequestsForUser(self::$userId); + $actual = $query->listRequestsForUser($userId); if (count($actual) > 0) { @@ -195,6 +209,7 @@ public function testUserRecordFieldList() // determine initial max id to enable cleanup after testing public static function setUpBeforeClass() { + parent::setUpBeforeClass(); static::$dbh = DB::factory('database'); static::$maxId = static::$dbh->query('SELECT COALESCE(MAX(id), 0) AS id FROM batch_export_requests')[0]['id']; } From 534becb321413456b10fb08088e6c5ccce0c6edb Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Tue, 28 May 2019 11:10:27 -0400 Subject: [PATCH 060/217] Add request date and format to query output from export requests table. --- classes/DataWarehouse/Export/QueryHandler.php | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index d1e5992783..8398eb46be 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -48,7 +48,7 @@ public function createRequestRecord($userId, $realm, $startDate, $endDate) (NOW(), :user_id, :realm, :start_date, :end_date)"; $params = array('user_id' => $userId, - 'realm' => $this->pdo->quote($realm), + 'realm' => $realm, 'start_date' => $startDate, 'end_date' => $endDate); @@ -135,7 +135,8 @@ public function countSubmittedRecords() // Return details of all export requests presently in Submitted state. public function listSubmittedRecords() { - $sql = "SELECT id, realm, start_date, end_date + $sql = "SELECT id, realm, start_date, end_date, + export_file_format, requested_datetime FROM batch_export_requests WHERE export_succeeded IS NULL"; @@ -150,11 +151,16 @@ public function listSubmittedRecords() // Return details of all export requests made by specified user. public function listRequestsForUser($user_id) { - $sql = "SELECT id, realm, start_date, end_date, + $sql = "SELECT id, + realm, + start_date, + end_date, export_succeeded, export_expired, export_expires_datetime, - export_created_datetime + export_created_datetime, + export_file_format, + requested_datetime FROM batch_export_requests WHERE user_id=:user_id ORDER BY id"; From 981e349e9a1501f9a0db55bdf080e41e29e911c2 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 28 May 2019 15:00:16 -0400 Subject: [PATCH 061/217] Restructure panels and add validation --- html/gui/js/modules/DataExport.js | 221 +++++++++++++++++++++++------- 1 file changed, 171 insertions(+), 50 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index abbe3cf7dd..f64634c88a 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -18,23 +18,48 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { initComponent: function () { this.requestsStore = new XDMoD.Module.DataExport.RequestsStore(); - this.on('afterrender', this.requestsStore.load, this.requestsStore); - this.requestForm = new XDMoD.Module.DataExport.RequestForm({ - region: 'west', - width: 375, - split: true, - margins: '2 0 2 2' + title: 'Create Bulk Data Export Request', + bodyStyle: 'padding: 5px 5px 0 5px', + border: false, + region: 'north' }); this.requestsGrid = new XDMoD.Module.DataExport.RequestsGrid({ + title: 'Status of Export Requests', region: 'center', margins: '2 2 2 0', pageSize: this.defaultPageSize, store: this.requestsStore }); - this.items = [this.requestsGrid, this.requestForm]; + this.requestsGrid.on('afterrender', this.requestsStore.load, this.requestsStore, { single: true }); + this.requestForm.on('actioncomplete', this.requestsGrid.reload, this.requestsGrid); + + this.items = [ + { + xtype: 'panel', + border: true, + width: 375, + split: true, + region: 'west', + margins: '2 0 0 2', + layout: 'vbox', + layoutConfig: { + align: 'stretch' + }, + items: [ + this.requestForm, + { + // Spacer panel + xtype: 'panel', + border: false, + flex: 1 + } + ] + }, + this.requestsGrid + ]; XDMoD.Module.DataExport.superclass.initComponent.call(this); } @@ -44,11 +69,17 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { * Data export request form. */ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { - title: 'Create Bulk Data Export Request', - bodyStyle: 'padding:5px', - initComponent: function () { + this.maxDateRangeText = '1 year'; + this.maxDateRangeInMilliseconds = 1000 * 60 * 60 * 24 * 365; + + Ext.apply(this.initialConfig, { + method: 'POST', + url: 'rest/v1/warehouse/export/request' + }); + Ext.apply(this, { + monitorValid: true, tools: [ { id: 'help', @@ -58,6 +89,9 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { items: [ { xtype: 'fieldset', + style: { + margin: '0' + }, columnWidth: 1, items: [ { @@ -71,16 +105,14 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { editable: false, triggerAction: 'all', mode: 'local', - store: new Ext.data.JsonStore({ + store: { + xtype: 'jsonstore', autoLoad: true, autoDestroy: true, root: 'data', fields: ['id', 'name'], - proxy: new Ext.data.HttpProxy({ - method: 'GET', - url: 'rest/v1/warehouse/export/realms' - }) - }) + url: 'rest/v1/warehouse/export/realms' + } }, { xtype: 'datefield', @@ -88,7 +120,8 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { fieldLabel: 'Start Date', emptyText: 'Start Date', format: 'Y-m-d', - allowBlank: false + allowBlank: false, + validator: this.validateStartDate.bind(this) }, { xtype: 'datefield', @@ -96,7 +129,8 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { fieldLabel: 'End Date', emptyText: 'End Date', format: 'Y-m-d', - allowBlank: false + allowBlank: false, + validator: this.validateEndDate.bind(this) }, { xtype: 'combo', @@ -109,41 +143,112 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { editable: false, triggerAction: 'all', mode: 'local', - store: new Ext.data.ArrayStore({ + store: { + xtype: 'arraystore', fields: ['id', 'name'], data: [ ['csv', 'CSV'], ['json', 'JSON'] ] - }) - } - ], - buttons: [ - { - xtype: 'button', - text: 'Submit Request', - scope: this, - handler: function () { - Ext.Ajax.request({ - url: 'rest/v1/warehouse/export/request', - method: 'POST', - params: this.getForm().getValues(), - scope: this, - success: function (response) { - // TODO - }, - failure: function (response) { - // TODO - } - }); } } ] } + ], + buttons: [ + { + xtype: 'button', + text: 'Submit Request', + formBind: true, + disabled: true, + scope: this, + handler: function () { + this.getForm().submit() + } + } ] }); XDMoD.Module.DataExport.RequestForm.superclass.initComponent.call(this); + + this.getForm().on('actionfailed', function (form, action) { + if (action.failureType === Ext.form.Action.CLIENT_INVALID) { + Ext.Msg.alert('Error', 'Validation failed, please check input values.'); + } else if (action.failureType === Ext.form.Action.CONNECT_FAILURE || action.failureType === Ext.form.Action.SERVER_INVALID) { + var response = action.response; + Ext.Msg.alert( + response.statusText || 'Error', + JSON.parse(response.responseText).message || 'Unknown Error' + ); + } else if (action.failureType === '') { + } else { + } + }); + + //this.on('clientvalidation', this.validateForm, this); + }, + + validateStartDate: function (startDate) { + try { + startDate = this.parseDate(startDate); + } catch (e) { + return e.message; + } + + var endDate = this.getForm().getFieldValues()['end_date']; + + if (endDate === '') { + return true; + } + + if (startDate > endDate) { + return 'Start date must be before the end date'; + } + + if (endDate - startDate > this.maxDateRangeInMilliseconds) { + return 'Date range must be less than ' + this.maxDateRangeText; + } + + return true; + }, + + validateEndDate: function (endDate) { + try { + endDate = this.parseDate(endDate); + } catch (e) { + return e.message; + } + + var startDate = this.getForm().getFieldValues()['start_date']; + + if (startDate === '') { + return true; + } + + if (startDate > endDate) { + return 'End date must be after the start date'; + } + + if (endDate - startDate > this.maxDateRangeInMilliseconds) { + return 'Date range must be less than ' + this.maxDateRangeText; + } + + return true; + }, + + parseDate: function (date) { + if (Ext.isDate(date)) { + return date; + } + + var format = 'Y-m-d'; + var parsedDate = Date.parseDate(date, format); + + if (parsedDate === undefined) { + throw new Error(date + ' is not a valid date - it must be in the format ' + format); + } + + return parsedDate; } }); @@ -151,10 +256,12 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { * Data export request grid. */ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { - title: 'Status of Export Requests', - initComponent: function () { + this.store.on('beforeload', this.showLoadingMask, this); + this.store.on('load', this.hideLoadingMask, this); + Ext.apply(this, { + loadMask: true, tools: [ { id: 'help', @@ -202,9 +309,13 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { header: 'Actions', xtype: 'templatecolumn', tpl: new Ext.XTemplate( - '', - ' ', - ' ' + '', + '', + ' ', + '', + '', + ' ', + '' ) } ], @@ -212,11 +323,13 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { { xtype: 'button', text: 'Delete all expired requests', + scope: this, handler: function () { Ext.Msg.confirm( 'Delete All Expired Requests', 'Are you sure that you want to delete all expired requests? You cannot undo this operation.', function (selection) { + this.reload(); if (selection === 'yes') { Ext.Msg.alert('TODO', 'TODO: Delete all the expired requests'); } @@ -238,6 +351,17 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { }); XDMoD.Module.DataExport.RequestsGrid.superclass.initComponent.call(this); + }, + + showLoadingMask: function () { + + }, + + hideLoadingMask: function () { + }, + + reload: function () { + this.store.reload(); } }); @@ -263,6 +387,7 @@ XDMoD.Module.DataExport.RequestsStore = Ext.extend(Ext.data.JsonStore, { constructor: function (config) { config = config || {}; Ext.apply(config, { + url: 'rest/v1/warehouse/export/requests', root: 'data', fields: [ { @@ -318,11 +443,7 @@ XDMoD.Module.DataExport.RequestsStore = Ext.extend(Ext.data.JsonStore, { return 'Submitted'; } } - ], - proxy: new Ext.data.HttpProxy({ - method: 'GET', - url: 'rest/v1/warehouse/export/requests' - }) + ] }); XDMoD.Module.DataExport.RequestsStore.superclass.constructor.call(this, config); From 0df25f5a09e7a66660bfe31f3d28798c7cd3fd7e Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Tue, 28 May 2019 16:26:36 -0400 Subject: [PATCH 062/217] Current working state, added state requirements to transitions, and added state to user request list report. --- classes/DataWarehouse/Export/QueryHandler.php | 114 ++++++++---- tests/component/lib/Export/ExportDBTest.php | 164 ++++++++++++++++-- 2 files changed, 233 insertions(+), 45 deletions(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index 8398eb46be..458d1ebd5b 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -18,6 +18,25 @@ * TODO, possibly: return list of ids for records that need to be marked 'export_expired' * * timestamp and current datetime: always use the database's value + * + * Recognized states enforced in this class: + * + * State export_succeeded export_created_datetime export_expired + * ----- ---------------- ----------------------- -------------- + * Submitted NULL NULL FALSE + * Available TRUE NOT NULL FALSE + * Expired TRUE NOT NULL TRUE + * Failed FALSE NULL FALSE + * (Deleted) not present in database or filesystem + * + * State transitions enforced in this class: + * + * Submitted -> Available -> Expired + * | + * v + * Failed + * + * ...any state can transition to Deleted. * ========================================================================================== */ @@ -30,6 +49,9 @@ class QueryHandler { private $pdo; // populated in the constructor + // Definition of Submitted state: + private $whereSubmitted = "WHERE export_succeeded is NULL and export_created_datetime is NULL and export_expired = FALSE "; + function __construct() { @@ -39,18 +61,21 @@ function __construct() /* transition between request record states */ - // create request record for specified export request - public function createRequestRecord($userId, $realm, $startDate, $endDate) + // Create request record for specified export request + // Result is a single request in Submitted state. + public function createRequestRecord($userId, $realm, $startDate, $endDate, $format) { $sql = "INSERT INTO batch_export_requests - (requested_datetime, user_id, realm, start_date, end_date) + (requested_datetime, user_id, realm, start_date, end_date, export_file_format) VALUES - (NOW(), :user_id, :realm, :start_date, :end_date)"; + (NOW(), :user_id, :realm, :start_date, :end_date, :export_file_format)"; $params = array('user_id' => $userId, 'realm' => $realm, 'start_date' => $startDate, - 'end_date' => $endDate); + 'end_date' => $endDate, + 'export_file_format' => $format + ); // return the id for the inserted record $id = $this->pdo->insert($sql, $params); @@ -58,22 +83,24 @@ public function createRequestRecord($userId, $realm, $startDate, $endDate) return($id); } - // transition specified export request to Failed state from Submitted. + // Transition specified export request from Submitted state to Failed state. public function submittedToFailed($id) { $sql = "UPDATE batch_export_requests - SET export_succeeded=0 - WHERE id=:id"; + SET export_succeeded=0 " . + $this->whereSubmitted . + "AND id=:id"; $params = array('id' => $id); // Return count of affected rows--should be 1. $result = $this->pdo->execute($sql, $params); - return(count($result)==1); + //return(count($result)==1); + return($result); } - // transition specified export request to Available state from Submitted. + // transition specified export request from Submitted state to Available state. // All time is current time as relative to database. public function submittedToAvailable($id) { @@ -81,11 +108,10 @@ public function submittedToAvailable($id) $expires_in_days = \xd_utilities\getConfiguration('data_warehouse_export','retention_duration'); $sql = "UPDATE batch_export_requests - SET - export_created_datetime=CAST(NOW() as DATETIME), + SET export_created_datetime=CAST(NOW() as DATETIME), export_expires_datetime=DATE_ADD(CAST(NOW() as DATETIME), INTERVAL :expires_in_days DAY), - export_succeeded=1 - WHERE id=:id"; + export_succeeded=1 " . + $this->whereSubmitted . "AND id=:id"; $params = array('expires_in_days' => $expires_in_days, 'id' => $id); @@ -96,14 +122,14 @@ public function submittedToAvailable($id) return(count($result)==1); } - // transition specified export request to Expired state from Available. - // Note that when the table grows large we don't want to do date comparisons... - // Therefore we should create an export_expired flag on the db table. + // Transition specified export request from Available state to Expired state. public function availableToExpired($id) { $sql = "UPDATE batch_export_requests SET export_expired=1 - WHERE id=:id"; + WHERE id=:id AND + export_succeeded = TRUE AND export_created_datetime IS NOT NULL + AND export_expired = FALSE"; $params = array('id' => $id); @@ -118,9 +144,7 @@ public function availableToExpired($id) // Return count of export requests presently in Submitted state. public function countSubmittedRecords() { - $sql = "SELECT COUNT(id) - FROM batch_export_requests - WHERE export_succeeded IS NULL"; + $sql = "SELECT COUNT(id) FROM batch_export_requests " . $this->whereSubmitted; // Return count of rows. try { @@ -135,16 +159,12 @@ public function countSubmittedRecords() // Return details of all export requests presently in Submitted state. public function listSubmittedRecords() { - $sql = "SELECT id, realm, start_date, end_date, - export_file_format, requested_datetime - FROM batch_export_requests - WHERE export_succeeded IS NULL"; + $sql = "SELECT id, realm, start_date, end_date, export_file_format, requested_datetime + FROM batch_export_requests " . $this->whereSubmitted; // Return query results. $result = $this->pdo->query($sql); - // TODO: Logic to return neat table of records and states - // (or in another function that handles that part?) return($result); } @@ -170,9 +190,45 @@ public function listRequestsForUser($user_id) // Return query results. $result = $this->pdo->query($sql, $params); - // TODO: Logic to return neat table of records and states - // (or in another function that handles that part?) return $result; } + + // Return details (including state) of all export requests made by specified user. + public function listUserRequestsByState($user_id) + { + $attributes = "SELECT id, realm, start_date, end_date, export_succeeded, export_expired, export_expires_datetime, export_created_datetime, export_file_format, requested_datetime, "; + $fromTable = "FROM batch_export_requests "; + //$whereSubmitted = "WHERE export_succeeded is NULL and export_created_datetime is NULL and export_expired = FALSE "; + $whereAvailable = "WHERE export_succeeded = TRUE and export_created_datetime is NOT NULL and export_expired = FALSE "; + $whereExpired = "WHERE export_succeeded = TRUE and export_created_datetime is NOT NULL and export_expired = TRUE "; + $whereFailed = "WHERE export_succeeded = FALSE and export_created_datetime is NULL and export_expired = FALSE "; + $userClause = "AND user_id = :user_id "; + + $sql = $attributes . "'Submitted' as state " . $fromTable . $this->whereSubmitted . $userClause . "UNION " . + $attributes . "'Available' as state " . $fromTable . $whereAvailable . $userClause . "UNION " . + $attributes . "'Expired' as state " . $fromTable . $whereExpired . $userClause . "UNION " . + $attributes . "'Failed' as state " . $fromTable . $whereFailed . $userClause . "ORDER BY requested_datetime"; + + $params = array('user_id' => $user_id); + + // Return query results. + $result = $this->pdo->query($sql, $params); + return $result; + } + + // Delete specified record, regardless of its state. + // Only the user who submittedthe request may delete it. + public function deleteRequest($id, $user) + { + // delete record, providing that requesting user owns specified record + $sql = "DELETE FROM batch_export_requests WHERE id=:request_id AND user_id=:user_id"; + $params = array('request_id' => $id, + 'user_id' => $user + ); + + // Return count of deleted rows--should be 1 if successful. + $result = $this->pdo->execute($sql, $params); + return($result); + } } diff --git a/tests/component/lib/Export/ExportDBTest.php b/tests/component/lib/Export/ExportDBTest.php index 8aaf5e588d..48eeccc7ae 100644 --- a/tests/component/lib/Export/ExportDBTest.php +++ b/tests/component/lib/Export/ExportDBTest.php @@ -15,7 +15,7 @@ class ExportDBTest extends BaseTest { static $dbh = null; static $maxId = null; - private static $userName = self::NORMAL_USER_USER_NAME; + //private static $userName = self::NORMAL_USER_USER_NAME; private static $debug = TRUE; public function testCountSubmitted() @@ -29,15 +29,25 @@ public function testCountSubmitted() if (self::$debug) print("\n".__FUNCTION__.": submittedRecords=$submittedCount\n"); } + // Specify a user that actually exists in moddb.Users table. private function acquireUserId() { $userId = static::$dbh->query('SELECT MIN(id) AS id FROM Users')[0]['id']; if ($userId == NULL) { - $userId = XDUser::getUserByUserName(self::$userName); + // TODO throw an exception, we have no users in the db } return $userId; } + /* + // Specify a different user that actually exists in moddb.Users table. + private function acquireMaxUserId() + { + return static::$dbh->query('SELECT MAX(id) AS id FROM Users')[0]['id']; + } + */ + + private function findSubmittedRecord() { $query = new QueryHandler(); @@ -46,7 +56,7 @@ private function findSubmittedRecord() // Find or create a record in submitted status to transition $maxSubmitted = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where export_succeeded IS NULL')[0]['id']; if ($maxSubmitted == NULL) { - $maxSubmitted = $query->createRequestRecord($userId, 'Jobs', '2017-01-01','2017-08-01'); + $maxSubmitted = $query->createRequestRecord($userId, 'Jobs', '2017-01-01','2017-08-01','CSV'); } if (self::$debug) print("\n".__FUNCTION__.": maxSubmitted ID=$maxSubmitted\n"); @@ -54,6 +64,21 @@ private function findSubmittedRecord() return($maxSubmitted); } + private function findAvailableRecord() + { + $query = new QueryHandler(); + + // Find or create a record in available status to transition + $maxAvailable = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where + export_succeeded IS TRUE + AND export_expired IS FALSE')[0]['id']; + if ($maxAvailable == NULL) { + $maxSubmitted = $this->findSubmittedRecord(); + $maxAvailable = $query->submittedToAvailable($maxSubmitted); + } + return($maxAvailable); + } + // Create two new records to enable testing of transition to Available, and transition to Expired: public function testNewRecordCreation() { @@ -65,11 +90,11 @@ public function testNewRecordCreation() $initialCount = $query->countSubmittedRecords(); // add new record and verify - $requestId = $query->createRequestRecord($userId, 'Jobs', '2019-01-01','2019-03-01'); + $requestId = $query->createRequestRecord($userId, 'Jobs', '2019-01-01','2019-03-01','CSV'); $this->assertNotNull($requestId); // add another new record and verify - $requestId2 = $query->createRequestRecord($userId, 'Accounts', '2016-12-01','2017-01-01'); + $requestId2 = $query->createRequestRecord($userId, 'Accounts', '2016-12-01','2017-01-01','JSON'); $this->assertNotNull($requestId2 ); $this->assertTrue($requestId2-$requestId==1); @@ -85,6 +110,39 @@ public function testNewRecordCreation() if (self::$debug) print("\n".__FUNCTION__.": initialCount=$initialCount finalCount=$finalCount requestId=$requestId requestId2=$requestId2\n"); } + // Test should fail. Do not allow this transition. + public function testAvailableToFailed() + { + $query = new QueryHandler(); + + // Find or create a record in available status + $maxAvailable = $this->findAvailableRecord(); + if (self::$debug) print("\n".__FUNCTION__.": max=$maxAvailable\n"); + + // Attempt to transition the record + $result = $query->submittedToFailed($maxAvailable); + $test = static::$dbh->query("SELECT export_succeeded FROM batch_export_requests where id=:id", + array('id'=>$maxAvailable))[0]['export_succeeded']; + + // Assert that: + // + // Exactly zero records were transitioned + $this->assertTrue($result==0); + + // That record is marked export_succeeded=TRUE + $this->assertEquals($test, 1); + + // That record has a non-null export created datetime + $export_created = static::$dbh->query("SELECT export_created_datetime FROM batch_export_requests where id=:id", + array('id'=>$maxAvailable))[0]['export_created_datetime']; + + // That record is marked export_created_datetime=NOT NULL + // todo: has a datetime + //$this->assertEquals($test, 1); + + // debug + if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxAvailable\n"); + } public function testSubmittedToFailed() { $query = new QueryHandler(); @@ -135,13 +193,7 @@ public function testAvailableToExpired() $query = new QueryHandler(); // Find or create a record in available status to transition - $maxAvailable = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where - export_succeeded IS TRUE - AND export_expired IS FALSE')[0]['id']; - if ($maxAvailable == NULL) { - $maxSubmitted = $this->findSubmittedRecord(); - $maxAvailable = $query->submittedToAvailable($maxSubmitted); - } + $maxAvailable = $this->findAvailableRecord(); $result = $query->availableToExpired($maxAvailable); $test = static::$dbh->query("SELECT export_expired FROM batch_export_requests where id=:id", @@ -167,14 +219,16 @@ public function testSubmittedRecordFieldList() 'id', 'realm', 'start_date', - 'end_date' + 'end_date', + 'export_file_format', + 'requested_datetime' ); $actual = $query->listSubmittedRecords(); if (count($actual) > 0) { - // check that you get the same fields back from the query... + // assert that the expected fields are returned from the query $this->assertEquals($expectedKeys, array_keys($actual[0])); } } @@ -193,7 +247,9 @@ public function testUserRecordFieldList() 'export_succeeded', 'export_expired', 'export_expires_datetime', - 'export_created_datetime' + 'export_created_datetime', + 'export_file_format', + 'requested_datetime' ); // Requests via this user have been created as part of these tests @@ -201,19 +257,95 @@ public function testUserRecordFieldList() if (count($actual) > 0) { - // check that you get the same fields back from the query... + // assert that the expected fields are returned from the query + $this->assertEquals($expectedKeys, array_keys($actual[0])); + } + } + + public function testUserRecordReportStates() + { + $query = new QueryHandler(); + $userId = $this->acquireUserId(); //XDUser::getUserByUserName(self::$userName); + + // Expect these keys from the associative array + $expectedKeys = array( + 'id', + 'realm', + 'start_date', + 'end_date', + 'export_succeeded', + 'export_expired', + 'export_expires_datetime', + 'export_created_datetime', + 'export_file_format', + 'requested_datetime', + 'state' + ); + + // Requests via this user have been created as part of these tests + $actual = $query->listUserRequestsByState($userId); + + if (count($actual) > 0) { + + // assert that the expected fields are returned from the query $this->assertEquals($expectedKeys, array_keys($actual[0])); } + + + } + + // verify that user that created request can delete it + public function testRecordDeleteCorrectUser() + { + $query = new QueryHandler(); + $userId = $this->acquireUserId(); //XDUser::getUserByUserName(self::$userName); + + // Requests via this user have been created as part of these tests + $actual = $query->listRequestsForUser($userId); + + if (count($actual) > 0) { + + // pick the first such request and try to delete it: + $testVal = $query->deleteRequest($actual[0]['id'], $userId); + + // assert that the delete affected 1 row + $this->assertEquals($testVal, 1); + if (self::$debug) print("\n".__FUNCTION__.": deleted record? $testVal\n"); + } + } + + // verify that user that did not create request cannot delete it + public function testRecordDeleteIncorrectUser() + { + $query = new QueryHandler(); + $userId = $this->acquireUserId(); + $wrongUserId = \XDUser::PUBLIC_USER; + + // Requests via user with $userId have been created as part of these tests + $actual = $query->listRequestsForUser($userId); + + // Provided that different users created and try to delete: + if (count($actual) > 0 && $userId != $wrongUserId) { + + // pick the first such request and try to delete it: + $testVal = $query->deleteRequest($actual[0]['id'], $wrongUserId); + + // assert that the delete attempt affected 0 rows + $this->assertEquals($testVal, 0); + if (self::$debug) print("\n".__FUNCTION__.": deleted record? $testVal\n"); + } } // determine initial max id to enable cleanup after testing public static function setUpBeforeClass() { + // so long as you use PUBLIC_USER_NAME or the like parent::setUpBeforeClass(); static::$dbh = DB::factory('database'); static::$maxId = static::$dbh->query('SELECT COALESCE(MAX(id), 0) AS id FROM batch_export_requests')[0]['id']; } + // TODO: put it back... // Reset the table to where it started public static function tearDownAfterClass() { From 9f343e176d99cb9e9f4b749e269ae6a165a8a196 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 29 May 2019 07:39:23 -0400 Subject: [PATCH 063/217] Fix style issues and improve error reporting --- html/gui/js/modules/DataExport.js | 47 +++++++++++++++++++------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index f64634c88a..2b0719bdd9 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -163,7 +163,7 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { disabled: true, scope: this, handler: function () { - this.getForm().submit() + this.getForm().submit(); } } ] @@ -173,29 +173,32 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { this.getForm().on('actionfailed', function (form, action) { if (action.failureType === Ext.form.Action.CLIENT_INVALID) { - Ext.Msg.alert('Error', 'Validation failed, please check input values.'); + // It shouldn't be possible to submit and invalid form, but it + // does happen display an error message. + Ext.Msg.alert('Error', 'Validation failed, please check input values and resubmit.'); } else if (action.failureType === Ext.form.Action.CONNECT_FAILURE || action.failureType === Ext.form.Action.SERVER_INVALID) { var response = action.response; Ext.Msg.alert( response.statusText || 'Error', JSON.parse(response.responseText).message || 'Unknown Error' ); - } else if (action.failureType === '') { + } else if (action.failureType === Ext.form.Action.LOAD_FAILURE) { + // This error occurs when the server doesn't return anything. + Ext.Msg.alert('Submission Error', 'Failed to submit request, try again later.'); } else { + Ext.Msg.alert('Unknown Error', 'An unknown error occured, try again later.'); } }); - - //this.on('clientvalidation', this.validateForm, this); }, - validateStartDate: function (startDate) { + validateStartDate: function (date) { try { - startDate = this.parseDate(startDate); + startDate = this.parseDate(date); } catch (e) { return e.message; } - var endDate = this.getForm().getFieldValues()['end_date']; + var endDate = this.getForm().getFieldValues().end_date; if (endDate === '') { return true; @@ -212,14 +215,14 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { return true; }, - validateEndDate: function (endDate) { + validateEndDate: function (date) { try { - endDate = this.parseDate(endDate); + endDate = this.parseDate(date); } catch (e) { return e.message; } - var startDate = this.getForm().getFieldValues()['start_date']; + var startDate = this.getForm().getFieldValues().start_date; if (startDate === '') { return true; @@ -241,12 +244,12 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { return date; } - var format = 'Y-m-d'; - var parsedDate = Date.parseDate(date, format); + var format = 'Y-m-d'; + var parsedDate = Date.parseDate(date, format); if (parsedDate === undefined) { throw new Error(date + ' is not a valid date - it must be in the format ' + format); - } + } return parsedDate; } @@ -384,8 +387,8 @@ XDMoD.Module.DataExport.exportStatusHelpText = * Data warehouse export batch requests data store. */ XDMoD.Module.DataExport.RequestsStore = Ext.extend(Ext.data.JsonStore, { - constructor: function (config) { - config = config || {}; + constructor: function (c) { + var config = c || {}; Ext.apply(config, { url: 'rest/v1/warehouse/export/requests', root: 'data', @@ -434,12 +437,20 @@ XDMoD.Module.DataExport.RequestsStore = Ext.extend(Ext.data.JsonStore, { { name: 'state', convert: function (v, record) { - // TODO - return 'Available'; + // TODO: Replace this with data from the server. + if (record.export_expired === '1') { return 'Expired'; } + if (record.export_created_datetime !== '') { + if (Date.now() < Date.parseDate(record.export_expires_datetime, 'Y-m-d H:i:s')) { + return 'Available'; + } else { + return 'Expired'; + } + } + return 'Submitted'; } } From 244b0790ba36440625909a8c7de19fbc67bf641f Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 29 May 2019 08:57:03 -0400 Subject: [PATCH 064/217] Add missing variable declarations and fix logic --- html/gui/js/modules/DataExport.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 2b0719bdd9..df2a66cb59 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -192,6 +192,7 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { }, validateStartDate: function (date) { + var startDate; try { startDate = this.parseDate(date); } catch (e) { @@ -216,6 +217,7 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { }, validateEndDate: function (date) { + var endDate; try { endDate = this.parseDate(date); } catch (e) { @@ -444,11 +446,7 @@ XDMoD.Module.DataExport.RequestsStore = Ext.extend(Ext.data.JsonStore, { } if (record.export_created_datetime !== '') { - if (Date.now() < Date.parseDate(record.export_expires_datetime, 'Y-m-d H:i:s')) { - return 'Available'; - } else { - return 'Expired'; - } + return 'Available'; } return 'Submitted'; From 985842df80a345cb9772ea541d7ecdb95570c54b Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Wed, 29 May 2019 14:27:50 -0400 Subject: [PATCH 065/217] Added state requirements to all transitions in DB access class. Added state to output from user request list report. Added tests for all state transitions. --- classes/DataWarehouse/Export/QueryHandler.php | 62 ++- tests/component/lib/Export/ExportDBTest.php | 366 +++++++++++++----- 2 files changed, 285 insertions(+), 143 deletions(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index 458d1ebd5b..cfa7cfbb21 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -3,22 +3,12 @@ * * Class governing database access by Data Warehouse Export batch script * - * TODO: Notes and choices: evaluate these: - * Perhaps no: make this class a singleton--does it make sense? - * or do I need to have multiple instances--and who instantiates them? - * say I'm using it for a rest endpoint--what I do need to know? - * For database access and to issue queries: - * use CCR\DB namespace, which contains PDODB class that we want. - * deciding against e.g. making the state transitions reflected by classes. Too much. - * think about and safe access... - * do I need to do more to secure vars coming in - * determine current testing practices and write tests to them - * these need to be component tests - * * TODO, possibly: return list of ids for records that need to be marked 'export_expired' * * timestamp and current datetime: always use the database's value * + * ------------------------------------------------------------------------------------------ + * * Recognized states enforced in this class: * * State export_succeeded export_created_datetime export_expired @@ -29,6 +19,8 @@ * Failed FALSE NULL FALSE * (Deleted) not present in database or filesystem * + * ------------------------------------------------------------------------------------------ + * * State transitions enforced in this class: * * Submitted -> Available -> Expired @@ -37,6 +29,7 @@ * Failed * * ...any state can transition to Deleted. + * * ========================================================================================== */ @@ -47,7 +40,8 @@ class QueryHandler { - private $pdo; // populated in the constructor + // database handle, populated in the constructor + private $pdo; // Definition of Submitted state: private $whereSubmitted = "WHERE export_succeeded is NULL and export_created_datetime is NULL and export_expired = FALSE "; @@ -59,9 +53,9 @@ function __construct() $this->pdo = DB::factory('database'); } - /* transition between request record states */ + /* ******** Transition between request record states ******** */ - // Create request record for specified export request + // Create request record for specified export request. // Result is a single request in Submitted state. public function createRequestRecord($userId, $realm, $startDate, $endDate, $format) { @@ -79,7 +73,6 @@ public function createRequestRecord($userId, $realm, $startDate, $endDate, $form // return the id for the inserted record $id = $this->pdo->insert($sql, $params); - return($id); } @@ -93,14 +86,12 @@ public function submittedToFailed($id) $params = array('id' => $id); - // Return count of affected rows--should be 1. + // Return count of affected rows--should be 1 if successful. $result = $this->pdo->execute($sql, $params); - - //return(count($result)==1); return($result); } - // transition specified export request from Submitted state to Available state. + // Transition specified export request from Submitted state to Available state. // All time is current time as relative to database. public function submittedToAvailable($id) { @@ -116,10 +107,9 @@ public function submittedToAvailable($id) $params = array('expires_in_days' => $expires_in_days, 'id' => $id); - // Return count of affected rows--should be 1. + // Return count of affected rows--should be 1 if successful. $result = $this->pdo->execute($sql, $params); - - return(count($result)==1); + return($result); } // Transition specified export request from Available state to Expired state. @@ -133,15 +123,14 @@ public function availableToExpired($id) $params = array('id' => $id); - // Return count of affected rows--should be 1. + // Return count of affected rows--should be 1 if successful. $result = $this->pdo->execute($sql, $params); - - return(count($result)==1); + return($result); } - /* list request records states */ + /* ******** List request records and states ******** */ - // Return count of export requests presently in Submitted state. + // Return count of all export requests presently in Submitted state. public function countSubmittedRecords() { $sql = "SELECT COUNT(id) FROM batch_export_requests " . $this->whereSubmitted; @@ -164,11 +153,10 @@ public function listSubmittedRecords() // Return query results. $result = $this->pdo->query($sql); - return($result); } - // Return details of all export requests made by specified user. + // Return details of export requests made by specified user. public function listRequestsForUser($user_id) { $sql = "SELECT id, @@ -189,16 +177,14 @@ public function listRequestsForUser($user_id) // Return query results. $result = $this->pdo->query($sql, $params); - - return $result; + return($result); } - // Return details (including state) of all export requests made by specified user. + // Return details (including state) of export requests made by specified user. public function listUserRequestsByState($user_id) { $attributes = "SELECT id, realm, start_date, end_date, export_succeeded, export_expired, export_expires_datetime, export_created_datetime, export_file_format, requested_datetime, "; $fromTable = "FROM batch_export_requests "; - //$whereSubmitted = "WHERE export_succeeded is NULL and export_created_datetime is NULL and export_expired = FALSE "; $whereAvailable = "WHERE export_succeeded = TRUE and export_created_datetime is NOT NULL and export_expired = FALSE "; $whereExpired = "WHERE export_succeeded = TRUE and export_created_datetime is NOT NULL and export_expired = TRUE "; $whereFailed = "WHERE export_succeeded = FALSE and export_created_datetime is NULL and export_expired = FALSE "; @@ -213,11 +199,13 @@ public function listUserRequestsByState($user_id) // Return query results. $result = $this->pdo->query($sql, $params); - return $result; + return($result); } - // Delete specified record, regardless of its state. - // Only the user who submittedthe request may delete it. + /* ******** Delete single user-submitted request record ******** */ + + // Delete specified record from the database, regardless of its state. + // Only the user who submitted the request may delete it. public function deleteRequest($id, $user) { // delete record, providing that requesting user owns specified record diff --git a/tests/component/lib/Export/ExportDBTest.php b/tests/component/lib/Export/ExportDBTest.php index 48eeccc7ae..f5893b4034 100644 --- a/tests/component/lib/Export/ExportDBTest.php +++ b/tests/component/lib/Export/ExportDBTest.php @@ -8,208 +8,363 @@ use \XDUser; +// Test access to batch_export_requests table, via QueryHandler class. class ExportDBTest extends BaseTest - // Test access to batch_export_requests table, via QueryHandler class. - // Clean up table to initial state following testing. - // $debug variable, if TRUE, enables print of output from public functions, { static $dbh = null; + + // Used for: Clean up table to initial state following testing. static $maxId = null; - //private static $userName = self::NORMAL_USER_USER_NAME; - private static $debug = TRUE; - public function testCountSubmitted() - { - $query = new QueryHandler(); - $submittedCount = $query->countSubmittedRecords(); - $this->assertNotNull($submittedCount); - $this->assertTrue($submittedCount>=0); + // $debug variable, if TRUE, enables print of output from public functions, + private static $debug = FALSE; - // debug - if (self::$debug) print("\n".__FUNCTION__.": submittedRecords=$submittedCount\n"); - } + /* *********** PRIVATE HELPER METHODS *********** */ - // Specify a user that actually exists in moddb.Users table. + // Acquire an existing userId for test record creation, deletion, transitions private function acquireUserId() { - $userId = static::$dbh->query('SELECT MIN(id) AS id FROM Users')[0]['id']; - if ($userId == NULL) { - // TODO throw an exception, we have no users in the db - } - return $userId; - } - - /* - // Specify a different user that actually exists in moddb.Users table. - private function acquireMaxUserId() - { - return static::$dbh->query('SELECT MAX(id) AS id FROM Users')[0]['id']; + return XDUser::getUserByUserName(self::NORMAL_USER_USER_NAME)->getUserID(); } - */ - private function findSubmittedRecord() { - $query = new QueryHandler(); - $userId = $this->acquireUserId(); //XDUser::getUserByUserName(self::$userName); - - // Find or create a record in submitted status to transition + // Find or create a record in Submitted status $maxSubmitted = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where export_succeeded IS NULL')[0]['id']; + if ($maxSubmitted == NULL) { + $query = new QueryHandler(); + $userId = $this->acquireUserId(); $maxSubmitted = $query->createRequestRecord($userId, 'Jobs', '2017-01-01','2017-08-01','CSV'); } - - if (self::$debug) print("\n".__FUNCTION__.": maxSubmitted ID=$maxSubmitted\n"); - return($maxSubmitted); } private function findAvailableRecord() { - $query = new QueryHandler(); - - // Find or create a record in available status to transition + // Find or create a record in Available status $maxAvailable = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where export_succeeded IS TRUE AND export_expired IS FALSE')[0]['id']; if ($maxAvailable == NULL) { + $query = new QueryHandler(); $maxSubmitted = $this->findSubmittedRecord(); $maxAvailable = $query->submittedToAvailable($maxSubmitted); } return($maxAvailable); } + private function findExpiredRecord() + { + // Find or create a record in Expired status + $maxExpired = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where + export_expired IS TRUE')[0]['id']; + if ($maxExpired == NULL) { + $query = new QueryHandler(); + $maxAvailable = $this->findAvailableRecord(); + $maxExpired = $query->availableToExpired($maxAvailable); + } + return($maxExpired); + } + + private function findFailedRecord() + { + // Find or create a record in Failed status + $maxFailed = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where + export_succeeded IS FALSE')[0]['id']; + if ($maxFailed == NULL) { + $query = new QueryHandler(); + $maxSubmitted = $this->findSubmittedRecord(); + $maxFailed = $query->submittedToFailed($maxSubmitted); + } + return($maxFailed); + } + + private function flattenRecords($inArr) + { + $ids = Array(); + foreach($inArr as $arr) + { + $ids[] = $arr['id']; + } + return $ids; + } + + private function listSubmittedRecords() + { + // List ids of records in Submitted state + $submittedArr = static::$dbh->query('SELECT id FROM batch_export_requests where export_succeeded IS NULL'); + $retval = $this->flattenRecords($submittedArr); + return($retval); + } + + private function listAvailableRecords() + { + // List ids of records in Available state + $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests where export_succeeded IS TRUE and export_expired IS FALSE'); + $retval = $this->flattenRecords($availableArr); + return($retval); + } + + private function listExpiredRecords() + { + // List ids of records in Expired state + $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests where export_succeeded IS TRUE and export_expired IS TRUE'); + $retval = $this->flattenRecords($availableArr); + return($retval); + } + + private function listFailedRecords() + { + // List ids of records in Failed state + $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests where export_succeeded IS FALSE and export_expired IS FALSE'); + $retval = $this->flattenRecords($availableArr); + return($retval); + } + + /* *********** PUBLIC TESTS *********** */ + + public function testCountSubmitted() + { + $query = new QueryHandler(); + $submittedCount = $query->countSubmittedRecords(); + + $submittedList = $this->listSubmittedRecords(); + + $this->assertEquals($submittedCount, count($submittedList)); + $this->assertNotNull($submittedCount); + $this->assertTrue($submittedCount>=0); + + // debug + if (self::$debug) print("\n".__FUNCTION__.": submittedRecords=$submittedCount\n"); + } + // Create two new records to enable testing of transition to Available, and transition to Expired: public function testNewRecordCreation() { $query = new QueryHandler(); - $userId = $this->acquireUserId(); //XDUser::getUserByUserName(self::$userName); - //if (self::$debug) print("\n".__FUNCTION__.": userName=".self::$userName." userId=$userId\n"); + $userId = $this->acquireUserId(); - // find the count + // Find the count $initialCount = $query->countSubmittedRecords(); - // add new record and verify + // Add new record and verify $requestId = $query->createRequestRecord($userId, 'Jobs', '2019-01-01','2019-03-01','CSV'); $this->assertNotNull($requestId); - // add another new record and verify + // Add another new record and verify $requestId2 = $query->createRequestRecord($userId, 'Accounts', '2016-12-01','2017-01-01','JSON'); - $this->assertNotNull($requestId2 ); - $this->assertTrue($requestId2-$requestId==1); - // determine final count + // Determine final count $finalCount = $query->countSubmittedRecords(); - // verify final count - // should have added 2 records. + // Verify final count. Should have added 2 records. $this->assertTrue($finalCount-$initialCount==2); + // Verify newly created records are found in list of Submitted status records + $allSubmitted = $this->listSubmittedRecords(); + $this->assertContains($requestId2, $allSubmitted); + $this->assertContains($requestId, $allSubmitted); + // debug if (self::$debug) print("\n".__FUNCTION__.": initialCount=$initialCount finalCount=$finalCount requestId=$requestId requestId2=$requestId2\n"); } - // Test should fail. Do not allow this transition. - public function testAvailableToFailed() + public function testSubmittedToFailed() { $query = new QueryHandler(); - // Find or create a record in available status - $maxAvailable = $this->findAvailableRecord(); - if (self::$debug) print("\n".__FUNCTION__.": max=$maxAvailable\n"); + // Find or create a record in submitted status to transition + $maxSubmitted = $this->findSubmittedRecord(); - // Attempt to transition the record - $result = $query->submittedToFailed($maxAvailable); - $test = static::$dbh->query("SELECT export_succeeded FROM batch_export_requests where id=:id", - array('id'=>$maxAvailable))[0]['export_succeeded']; + // Transition the record to Failed + $result = $query->submittedToFailed($maxSubmitted); // Assert that: - // - // Exactly zero records were transitioned - $this->assertTrue($result==0); - - // That record is marked export_succeeded=TRUE - $this->assertEquals($test, 1); - - // That record has a non-null export created datetime - $export_created = static::$dbh->query("SELECT export_created_datetime FROM batch_export_requests where id=:id", - array('id'=>$maxAvailable))[0]['export_created_datetime']; + // Exactly one record was transitioned + $this->assertTrue($result==1); - // That record is marked export_created_datetime=NOT NULL - // todo: has a datetime - //$this->assertEquals($test, 1); + // This record is now marked Failed + $allFailed = $this->listFailedRecords(); + $this->assertContains($maxSubmitted, $allFailed); // debug - if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxAvailable\n"); + if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxSubmitted\n"); } - public function testSubmittedToFailed() + + public function testSubmittedToAvailable() { $query = new QueryHandler(); // Find or create a record in submitted status to transition $maxSubmitted = $this->findSubmittedRecord(); - // Transition record - $result = $query->submittedToFailed($maxSubmitted); - $test = static::$dbh->query("SELECT export_succeeded FROM batch_export_requests where id=:id", - array('id'=>$maxSubmitted))[0]['export_succeeded']; + $result = $query->submittedToAvailable($maxSubmitted); // Assert that: // Exactly one record was transitioned $this->assertTrue($result==1); - // That record is marked export_succeeded=FALSE - $this->assertEquals($test, 0); + // That record is marked Available + $allAvailable = $this->listAvailableRecords(); + $this->assertContains($maxSubmitted, $allAvailable); // debug if (self::$debug) print("\n".__FUNCTION__.": transitioned Id=$maxSubmitted\n"); } - public function testSubmittedToAvailable() + public function testSubmittedToExpired() { $query = new QueryHandler(); - // Find or create a record in submitted status to transition + // Find or create a record in Submitted status to transition $maxSubmitted = $this->findSubmittedRecord(); - $result = $query->submittedToAvailable($maxSubmitted); - $test = static::$dbh->query("SELECT export_succeeded FROM batch_export_requests where id=:id", - array('id'=>$maxSubmitted))[0]['export_succeeded']; + $result = $query->availableToExpired($maxSubmitted); // Assert that: - // Exactly one record was transitioned - $this->assertTrue($result==1); + // Exactly zero records transitioned + $this->assertTrue($result==0); - // That record is marked export_succeeded=TRUE - $this->assertEquals($test, 1); + // That record is still marked Submitted + $allSubmitted = $this->listSubmittedRecords(); + $this->assertContains($maxSubmitted, $allSubmitted); // debug - if (self::$debug) print("\n".__FUNCTION__.": transitioned Id=$maxSubmitted\n"); + if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxSubmitted\n"); } + public function testAvailableToExpired() { $query = new QueryHandler(); - // Find or create a record in available status to transition + // Find or create a record in Available status to transition $maxAvailable = $this->findAvailableRecord(); $result = $query->availableToExpired($maxAvailable); - $test = static::$dbh->query("SELECT export_expired FROM batch_export_requests where id=:id", - array('id'=>$maxAvailable))[0]['export_expired']; // Assert that: // Exactly one record was transitioned $this->assertTrue($result==1); // That record is marked export_expired=TRUE - $this->assertEquals($test, 1); + $allExpired = $this->listExpiredRecords(); + $this->assertContains($maxAvailable, $allExpired); // debug if (self::$debug) print("\n".__FUNCTION__.": transitioned Id=$maxAvailable\n"); } + public function testAvailableToFailed() + { + $query = new QueryHandler(); + + // Find or create a record in Available status + $maxAvailable = $this->findAvailableRecord(); + + // Attempt to transition the record to Failed + $result = $query->submittedToFailed($maxAvailable); + + // Assert that: + // Exactly zero records were transitioned + $this->assertTrue($result==0); + + // This record is still marked Available + $allAvailable = $this->listAvailableRecords(); + $this->assertContains($maxAvailable, $allAvailable); + + // debug + if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxAvailable\n"); + } + + public function testExpiredToFailed() + { + $query = new QueryHandler(); + + // Find or create a record in Expired status + $maxExpired = $this->findExpiredRecord(); + + // Attempt to transition the record to Failed + $result = $query->submittedToFailed($maxExpired); + + // Assert that: + // Exactly zero records were transitioned + $this->assertTrue($result==0); + + // This record is still marked Expired + $allExpired = $this->listExpiredRecords(); + $this->assertContains($maxExpired, $allExpired); + + // debug + if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxExpired\n"); + } + + public function testFailedToExpired() + { + $query = new QueryHandler(); + + // Find or create a record in Failed status to transition + $maxFailed = $this->findFailedRecord(); + + $result = $query->availableToExpired($maxFailed); + + // Assert that: + // Exactly zero records transitioned + $this->assertTrue($result==0); + + // That record is marked Failed + $allFailed = $this->listFailedRecords(); + $this->assertContains($maxFailed, $allFailed); + + // debug + if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxFailed\n"); + } + + public function testExpiredToAvailable() + { + $query = new QueryHandler(); + + // Find or create a record in Expired status to transition + $maxExpired = $this->findExpiredRecord(); + + $result = $query->submittedToAvailable($maxExpired); + + // Assert that: + // Exactly zero records transitioned + $this->assertTrue($result==0); + + // That record is marked Expired + $allExpired = $this->listExpiredRecords(); + $this->assertContains($maxExpired, $allExpired); + + // debug + if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxExpired\n"); + } + + public function testFailedToAvailable() + { + $query = new QueryHandler(); + + // Find or create a record in Failed status to transition + $maxFailed = $this->findFailedRecord(); + + $result = $query->submittedToAvailable($maxFailed); + + // Assert that: + // Exactly zero records transitioned + $this->assertTrue($result==0); + + // That record is marked Failed + $allFailed = $this->listFailedRecords(); + $this->assertContains($maxFailed, $allFailed); + + // debug + if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxFailed\n"); + } + + // Verify field list returned from listSubmittedRecords() public function testSubmittedRecordFieldList() { $query = new QueryHandler(); @@ -233,10 +388,11 @@ public function testSubmittedRecordFieldList() } } + // Verify field list returned from listRequestsForUser() public function testUserRecordFieldList() { $query = new QueryHandler(); - $userId = $this->acquireUserId(); //XDUser::getUserByUserName(self::$userName); + $userId = $this->acquireUserId(); // Expect these keys from the associative array $expectedKeys = array( @@ -262,10 +418,11 @@ public function testUserRecordFieldList() } } + // Verify field list returned from listUserRequestsByState() public function testUserRecordReportStates() { $query = new QueryHandler(); - $userId = $this->acquireUserId(); //XDUser::getUserByUserName(self::$userName); + $userId = $this->acquireUserId(); // Expect these keys from the associative array $expectedKeys = array( @@ -290,21 +447,19 @@ public function testUserRecordReportStates() // assert that the expected fields are returned from the query $this->assertEquals($expectedKeys, array_keys($actual[0])); } - - } - // verify that user that created request can delete it + // Verify that user that created request can delete it public function testRecordDeleteCorrectUser() { $query = new QueryHandler(); - $userId = $this->acquireUserId(); //XDUser::getUserByUserName(self::$userName); + $userId = $this->acquireUserId(); // Requests via this user have been created as part of these tests $actual = $query->listRequestsForUser($userId); - if (count($actual) > 0) { - + if (count($actual) > 0) + { // pick the first such request and try to delete it: $testVal = $query->deleteRequest($actual[0]['id'], $userId); @@ -314,12 +469,12 @@ public function testRecordDeleteCorrectUser() } } - // verify that user that did not create request cannot delete it + // Verify that user that did not create request cannot delete it public function testRecordDeleteIncorrectUser() { $query = new QueryHandler(); $userId = $this->acquireUserId(); - $wrongUserId = \XDUser::PUBLIC_USER; + $wrongUserId = XDUser::getUserByUserName(self::CENTER_STAFF_USER_NAME)->getUserID(); // Requests via user with $userId have been created as part of these tests $actual = $query->listRequestsForUser($userId); @@ -339,14 +494,13 @@ public function testRecordDeleteIncorrectUser() // determine initial max id to enable cleanup after testing public static function setUpBeforeClass() { - // so long as you use PUBLIC_USER_NAME or the like + // setup needed to use PUBLIC_USER_NAME or the like parent::setUpBeforeClass(); static::$dbh = DB::factory('database'); static::$maxId = static::$dbh->query('SELECT COALESCE(MAX(id), 0) AS id FROM batch_export_requests')[0]['id']; } - // TODO: put it back... - // Reset the table to where it started + // Reset the batch_export_requests database table to where it started public static function tearDownAfterClass() { static::$dbh->execute('DELETE FROM batch_export_requests WHERE id > :id', array('id' => static::$maxId)); From 45150597e6da4ec367f47388f7386544f7831d8c Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Thu, 30 May 2019 09:04:18 -0400 Subject: [PATCH 066/217] Style fixes for Export database class tests. --- tests/component/lib/Export/ExportDBTest.php | 121 +++++++++++++------- 1 file changed, 80 insertions(+), 41 deletions(-) diff --git a/tests/component/lib/Export/ExportDBTest.php b/tests/component/lib/Export/ExportDBTest.php index f5893b4034..c7c7e4fa1b 100644 --- a/tests/component/lib/Export/ExportDBTest.php +++ b/tests/component/lib/Export/ExportDBTest.php @@ -7,17 +7,16 @@ use \DataWarehouse\Export\QueryHandler; use \XDUser; - // Test access to batch_export_requests table, via QueryHandler class. class ExportDBTest extends BaseTest { - static $dbh = null; + private static $dbh = null; // Used for: Clean up table to initial state following testing. - static $maxId = null; + private static $maxId = null; - // $debug variable, if TRUE, enables print of output from public functions, - private static $debug = FALSE; + // $debug variable, if true, enables print of output from public functions, + private static $debug = false; /* *********** PRIVATE HELPER METHODS *********** */ @@ -30,12 +29,12 @@ private function acquireUserId() private function findSubmittedRecord() { // Find or create a record in Submitted status - $maxSubmitted = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where export_succeeded IS NULL')[0]['id']; + $maxSubmitted = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE export_succeeded IS NULL')[0]['id']; - if ($maxSubmitted == NULL) { + if ($maxSubmitted == null) { $query = new QueryHandler(); $userId = $this->acquireUserId(); - $maxSubmitted = $query->createRequestRecord($userId, 'Jobs', '2017-01-01','2017-08-01','CSV'); + $maxSubmitted = $query->createRequestRecord($userId, 'Jobs', '2017-01-01', '2017-08-01','CSV'); } return($maxSubmitted); } @@ -43,10 +42,10 @@ private function findSubmittedRecord() private function findAvailableRecord() { // Find or create a record in Available status - $maxAvailable = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where + $maxAvailable = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE export_succeeded IS TRUE AND export_expired IS FALSE')[0]['id']; - if ($maxAvailable == NULL) { + if ($maxAvailable == null) { $query = new QueryHandler(); $maxSubmitted = $this->findSubmittedRecord(); $maxAvailable = $query->submittedToAvailable($maxSubmitted); @@ -57,9 +56,9 @@ private function findAvailableRecord() private function findExpiredRecord() { // Find or create a record in Expired status - $maxExpired = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where + $maxExpired = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE export_expired IS TRUE')[0]['id']; - if ($maxExpired == NULL) { + if ($maxExpired == null) { $query = new QueryHandler(); $maxAvailable = $this->findAvailableRecord(); $maxExpired = $query->availableToExpired($maxAvailable); @@ -70,9 +69,9 @@ private function findExpiredRecord() private function findFailedRecord() { // Find or create a record in Failed status - $maxFailed = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests where + $maxFailed = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE export_succeeded IS FALSE')[0]['id']; - if ($maxFailed == NULL) { + if ($maxFailed == null) { $query = new QueryHandler(); $maxSubmitted = $this->findSubmittedRecord(); $maxFailed = $query->submittedToFailed($maxSubmitted); @@ -82,7 +81,7 @@ private function findFailedRecord() private function flattenRecords($inArr) { - $ids = Array(); + $ids = array(); foreach($inArr as $arr) { $ids[] = $arr['id']; @@ -93,7 +92,7 @@ private function flattenRecords($inArr) private function listSubmittedRecords() { // List ids of records in Submitted state - $submittedArr = static::$dbh->query('SELECT id FROM batch_export_requests where export_succeeded IS NULL'); + $submittedArr = static::$dbh->query('SELECT id FROM batch_export_requests WHERE export_succeeded IS NULL'); $retval = $this->flattenRecords($submittedArr); return($retval); } @@ -101,7 +100,7 @@ private function listSubmittedRecords() private function listAvailableRecords() { // List ids of records in Available state - $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests where export_succeeded IS TRUE and export_expired IS FALSE'); + $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests WHERE export_succeeded IS TRUE and export_expired IS FALSE'); $retval = $this->flattenRecords($availableArr); return($retval); } @@ -109,7 +108,7 @@ private function listAvailableRecords() private function listExpiredRecords() { // List ids of records in Expired state - $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests where export_succeeded IS TRUE and export_expired IS TRUE'); + $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests WHERE export_succeeded IS TRUE and export_expired IS TRUE'); $retval = $this->flattenRecords($availableArr); return($retval); } @@ -117,7 +116,7 @@ private function listExpiredRecords() private function listFailedRecords() { // List ids of records in Failed state - $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests where export_succeeded IS FALSE and export_expired IS FALSE'); + $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests WHERE export_succeeded IS FALSE and export_expired IS FALSE'); $retval = $this->flattenRecords($availableArr); return($retval); } @@ -136,10 +135,13 @@ public function testCountSubmitted() $this->assertTrue($submittedCount>=0); // debug - if (self::$debug) print("\n".__FUNCTION__.": submittedRecords=$submittedCount\n"); + if (self::$debug) + { + print("\n".__FUNCTION__.": submittedRecords=$submittedCount\n"); + } } - // Create two new records to enable testing of transition to Available, and transition to Expired: + // Create two new records in Submitted state. public function testNewRecordCreation() { $query = new QueryHandler(); @@ -149,11 +151,11 @@ public function testNewRecordCreation() $initialCount = $query->countSubmittedRecords(); // Add new record and verify - $requestId = $query->createRequestRecord($userId, 'Jobs', '2019-01-01','2019-03-01','CSV'); + $requestId = $query->createRequestRecord($userId, 'Jobs', '2019-01-01', '2019-03-01','CSV'); $this->assertNotNull($requestId); // Add another new record and verify - $requestId2 = $query->createRequestRecord($userId, 'Accounts', '2016-12-01','2017-01-01','JSON'); + $requestId2 = $query->createRequestRecord($userId, 'Accounts', '2016-12-01', '2017-01-01','JSON'); $this->assertNotNull($requestId2 ); // Determine final count @@ -168,7 +170,10 @@ public function testNewRecordCreation() $this->assertContains($requestId, $allSubmitted); // debug - if (self::$debug) print("\n".__FUNCTION__.": initialCount=$initialCount finalCount=$finalCount requestId=$requestId requestId2=$requestId2\n"); + if (self::$debug) + { + print("\n".__FUNCTION__.": initialCount=$initialCount finalCount=$finalCount requestId=$requestId requestId2=$requestId2\n"); + } } public function testSubmittedToFailed() @@ -190,7 +195,10 @@ public function testSubmittedToFailed() $this->assertContains($maxSubmitted, $allFailed); // debug - if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxSubmitted\n"); + if (self::$debug) + { + print("\n".__FUNCTION__.": NON transitioned Id=$maxSubmitted\n"); + } } public function testSubmittedToAvailable() @@ -211,7 +219,10 @@ public function testSubmittedToAvailable() $this->assertContains($maxSubmitted, $allAvailable); // debug - if (self::$debug) print("\n".__FUNCTION__.": transitioned Id=$maxSubmitted\n"); + if (self::$debug) + { + print("\n".__FUNCTION__.": transitioned Id=$maxSubmitted\n"); + } } public function testSubmittedToExpired() @@ -232,10 +243,12 @@ public function testSubmittedToExpired() $this->assertContains($maxSubmitted, $allSubmitted); // debug - if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxSubmitted\n"); + if (self::$debug) + { + print("\n".__FUNCTION__.": NON transitioned Id=$maxSubmitted\n"); + } } - public function testAvailableToExpired() { $query = new QueryHandler(); @@ -254,7 +267,10 @@ public function testAvailableToExpired() $this->assertContains($maxAvailable, $allExpired); // debug - if (self::$debug) print("\n".__FUNCTION__.": transitioned Id=$maxAvailable\n"); + if (self::$debug) + { + print("\n".__FUNCTION__.": transitioned Id=$maxAvailable\n"); + } } public function testAvailableToFailed() @@ -276,7 +292,10 @@ public function testAvailableToFailed() $this->assertContains($maxAvailable, $allAvailable); // debug - if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxAvailable\n"); + if (self::$debug) + { + print("\n".__FUNCTION__.": NON transitioned Id=$maxAvailable\n"); + } } public function testExpiredToFailed() @@ -298,7 +317,10 @@ public function testExpiredToFailed() $this->assertContains($maxExpired, $allExpired); // debug - if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxExpired\n"); + if (self::$debug) + { + print("\n".__FUNCTION__.": NON transitioned Id=$maxExpired\n"); + } } public function testFailedToExpired() @@ -319,7 +341,10 @@ public function testFailedToExpired() $this->assertContains($maxFailed, $allFailed); // debug - if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxFailed\n"); + if (self::$debug) + { + print("\n".__FUNCTION__.": NON transitioned Id=$maxFailed\n"); + } } public function testExpiredToAvailable() @@ -340,7 +365,10 @@ public function testExpiredToAvailable() $this->assertContains($maxExpired, $allExpired); // debug - if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxExpired\n"); + if (self::$debug) + { + print("\n".__FUNCTION__.": NON transitioned Id=$maxExpired\n"); + } } public function testFailedToAvailable() @@ -361,7 +389,10 @@ public function testFailedToAvailable() $this->assertContains($maxFailed, $allFailed); // debug - if (self::$debug) print("\n".__FUNCTION__.": NON transitioned Id=$maxFailed\n"); + if (self::$debug) + { + print("\n".__FUNCTION__.": NON transitioned Id=$maxFailed\n"); + } } // Verify field list returned from listSubmittedRecords() @@ -465,7 +496,11 @@ public function testRecordDeleteCorrectUser() // assert that the delete affected 1 row $this->assertEquals($testVal, 1); - if (self::$debug) print("\n".__FUNCTION__.": deleted record? $testVal\n"); + + if (self::$debug) + { + print("\n".__FUNCTION__.": deleted record? $testVal\n"); + } } } @@ -479,7 +514,7 @@ public function testRecordDeleteIncorrectUser() // Requests via user with $userId have been created as part of these tests $actual = $query->listRequestsForUser($userId); - // Provided that different users created and try to delete: + // Provided that we have specified two different users here: if (count($actual) > 0 && $userId != $wrongUserId) { // pick the first such request and try to delete it: @@ -487,23 +522,27 @@ public function testRecordDeleteIncorrectUser() // assert that the delete attempt affected 0 rows $this->assertEquals($testVal, 0); - if (self::$debug) print("\n".__FUNCTION__.": deleted record? $testVal\n"); + + if (self::$debug) + { + print("\n".__FUNCTION__.": deleted record? $testVal\n"); + } } } - // determine initial max id to enable cleanup after testing public static function setUpBeforeClass() { - // setup needed to use PUBLIC_USER_NAME or the like + // setup needed to use NORMAL_USER_USER_NAME or the like parent::setUpBeforeClass(); + + // determine initial max id to enable cleanup after testing static::$dbh = DB::factory('database'); static::$maxId = static::$dbh->query('SELECT COALESCE(MAX(id), 0) AS id FROM batch_export_requests')[0]['id']; } - // Reset the batch_export_requests database table to where it started public static function tearDownAfterClass() { + // Reset the batch_export_requests database table to its initial contents static::$dbh->execute('DELETE FROM batch_export_requests WHERE id > :id', array('id' => static::$maxId)); } - } From 14931083a66034693eb6ca3e95797a368b2f0502 Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Thu, 30 May 2019 09:23:11 -0400 Subject: [PATCH 067/217] Updated name of data_warehouse_export config field indicating number of days that elapse before automatic deletion of exported files. --- classes/DataWarehouse/Export/QueryHandler.php | 2 +- configuration/portal_settings.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index cfa7cfbb21..2df901ac5b 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -96,7 +96,7 @@ public function submittedToFailed($id) public function submittedToAvailable($id) { // read export retention duration from config file. Value is stored in days. - $expires_in_days = \xd_utilities\getConfiguration('data_warehouse_export','retention_duration'); + $expires_in_days = \xd_utilities\getConfiguration('data_warehouse_export','retention_duration_days'); $sql = "UPDATE batch_export_requests SET export_created_datetime=CAST(NOW() as DATETIME), diff --git a/configuration/portal_settings.ini b/configuration/portal_settings.ini index 38347e97f0..0cd15ffc40 100644 --- a/configuration/portal_settings.ini +++ b/configuration/portal_settings.ini @@ -157,5 +157,5 @@ sacct = "sacct" ; Exported data files will be stored in this directory export_directory = "/path/to/export/directory" ; Length of time in days that files will be retained before automatic deletion. -retention_duration = 30 +retention_duration_days = 30 From 2e2bf1d9003c8264ef5a71bdb7d132897d1e8e71 Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Thu, 30 May 2019 10:06:57 -0400 Subject: [PATCH 068/217] Added default path to data warehouse export file storage. --- configuration/portal_settings.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/portal_settings.ini b/configuration/portal_settings.ini index 0cd15ffc40..585bcf4090 100644 --- a/configuration/portal_settings.ini +++ b/configuration/portal_settings.ini @@ -155,7 +155,7 @@ sacct = "sacct" ; [data_warehouse_export] ; Exported data files will be stored in this directory -export_directory = "/path/to/export/directory" +export_directory = "/var/spool/xdmod/export" ; Length of time in days that files will be retained before automatic deletion. retention_duration_days = 30 From ae96ece796e853c1e4819a391b62005366aa36e3 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 30 May 2019 14:58:54 -0400 Subject: [PATCH 069/217] Remove unnecessary mask functions --- html/gui/js/modules/DataExport.js | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index df2a66cb59..d0eb0387e9 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -262,9 +262,6 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { */ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { initComponent: function () { - this.store.on('beforeload', this.showLoadingMask, this); - this.store.on('load', this.hideLoadingMask, this); - Ext.apply(this, { loadMask: true, tools: [ @@ -358,13 +355,6 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { XDMoD.Module.DataExport.RequestsGrid.superclass.initComponent.call(this); }, - showLoadingMask: function () { - - }, - - hideLoadingMask: function () { - }, - reload: function () { this.store.reload(); } From d320e8ce15e52ca0b90658d32dfefb637ec31b63 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 30 May 2019 14:59:12 -0400 Subject: [PATCH 070/217] Improve field and data store configurations --- html/gui/js/modules/DataExport.js | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index d0eb0387e9..7bff1d1895 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -96,7 +96,7 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { items: [ { xtype: 'combo', - name: 'realm', + hiddenName: 'realm', fieldLabel: 'Realm', emptyText: 'Select a realm', valueField: 'id', @@ -109,9 +109,12 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { xtype: 'jsonstore', autoLoad: true, autoDestroy: true, + url: 'rest/v1/warehouse/export/realms', root: 'data', - fields: ['id', 'name'], - url: 'rest/v1/warehouse/export/realms' + fields: [ + {name: 'id', type: 'string'}, + {name: 'name', type: 'string'} + ] } }, { @@ -137,20 +140,11 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { name: 'format', fieldLabel: 'Format', emptyText: 'Select an export format', - valueField: 'id', - displayField: 'name', allowBlank: false, editable: false, triggerAction: 'all', mode: 'local', - store: { - xtype: 'arraystore', - fields: ['id', 'name'], - data: [ - ['csv', 'CSV'], - ['json', 'JSON'] - ] - } + store: ['CSV', 'JSON'] } ] } @@ -435,7 +429,7 @@ XDMoD.Module.DataExport.RequestsStore = Ext.extend(Ext.data.JsonStore, { return 'Expired'; } - if (record.export_created_datetime !== '') { + if (record.export_created_datetime !== null) { return 'Available'; } From 39bb62e7910309e601cacacb0efe7cd083465c9a Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 30 May 2019 15:00:00 -0400 Subject: [PATCH 071/217] Add some validation and update controller --- .../Controllers/BaseControllerProvider.php | 46 +++++++++++++++++ .../WarehouseExportControllerProvider.php | 49 +++++++++++++++---- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/classes/Rest/Controllers/BaseControllerProvider.php b/classes/Rest/Controllers/BaseControllerProvider.php index ee3985ea5d..867a7ada03 100644 --- a/classes/Rest/Controllers/BaseControllerProvider.php +++ b/classes/Rest/Controllers/BaseControllerProvider.php @@ -562,6 +562,52 @@ protected function getDateTimeFromUnixParam(Request $request, $name, $mandatory ); } + /** + * Attempt to get a date parameter value from a request where it is + * submitted as a ISO 8601 (YYYY-MM-DD) date. + * + * @param Request $request The request to extract the parameter from. + * @param string $name The name of the parameter. + * @param boolean $mandatory (Optional) If true, an exception will be + * thrown if the parameter is missing from the + * request. (Defaults to false.) + * @param mixed $default (Optional) The value to return if the + * parameter was not specified and the parameter + * is not mandatory. (Defaults to null.) + * @return mixed If available and valid, the parameter value + * as a DateTime. Otherwise, if it is missing + * and not mandatory, the given default. + * + * @throws BadRequestHttpException If the parameter was not available + * and the parameter was deemed mandatory, + * or if the parameter value could not be + * converted to a DateTime. + */ + protected function getDateFromISO8601Param( + Request $request, + $name, + $mandatory = false, + $default = null + ) { + return $this->getParam( + $request, + $name, + $mandatory, + $default, + FILTER_CALLBACK, + [ + 'options' => function ($value) { + $value_dt = \DateTime::createFromFormat('Y-m-d', $value); + if ($value_dt === false) { + return null; + } + return $value_dt; + }, + ], + 'ISO 8601 Date' + ); + } + /** * Get the best match for the acceptable content type for the request, given a * list of supported content types. diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index ced264aa46..a760e5ba78 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -6,6 +6,7 @@ use Silex\Application; use Silex\ControllerCollection; use Symfony\Component\HttpFoundation\Request; +use Symfony\Component\HttpKernel\Exception\BadRequestHttpException; class WarehouseExportControllerProvider extends BaseControllerProvider { @@ -79,22 +80,52 @@ public function getRequests(Request $request, Application $app) public function createRequest(Request $request, Application $app) { $user = $this->authorize($request); + $realm = $this->getStringParam($request, 'realm', true); - // TODO: Validate input. - $realm = $this->getStringParam($request, 'realm'); - $startDate = $this->getStringParam($request, 'start_date'); - $endDate = $this->getStringParam($request, 'end_date'); - $format = $this->getStringParam($request, 'format'); + // TODO: Validate realm from user ACLs. + if (!in_array( + $realm, + [ + 'jobs', + 'supremm', + 'accounts', + 'allocations', + 'requests', + 'resource_allocations' + ] + )) { + throw new BadRequestHttpException('Invalid realm'); + } + + $startDate = $this->getDateFromISO8601Param($request, 'start_date', true); + $endDate = $this->getDateFromISO8601Param($request, 'end_date', true); + + $interval = $startDate->diff($endDate); + + if ($interval === false) { + throw new BadRequestHttpException('Failed to calculate date interval'); + } + + if ($interval->invert === 1) { + throw new BadRequestHttpException('Start date must be before end date'); + } + + $format = strtoupper($this->getStringParam($request, 'format', true)); + + if (!in_array($format, ['CSV', 'JSON'])) { + throw new BadRequestHttpException('format must be CSV or JSON'); + } $handler = new QueryHandler(); - $handler->createRequestRecord( + $id = $handler->createRequestRecord( $user->getUserId(), $realm, - $startDate, - $endDate + $startDate->format('Y-m-d'), + $endDate->format('Y-m-d'), + $format ); - return $app->json(['success' => true]); + return $app->json(['success' => true, 'data' => ['id' => $id]]); } } From f155380f7eec42afcf74fcf266c205b8cf096e4f Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 07:05:58 -0400 Subject: [PATCH 072/217] Fix style issue --- html/gui/js/modules/DataExport.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 7bff1d1895..10dfb56b39 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -112,8 +112,8 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { url: 'rest/v1/warehouse/export/realms', root: 'data', fields: [ - {name: 'id', type: 'string'}, - {name: 'name', type: 'string'} + { name: 'id', type: 'string' }, + { name: 'name', type: 'string' } ] } }, From 6aaf03e82a4552dfada67b7cc831fc6c60fcd021 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 07:11:25 -0400 Subject: [PATCH 073/217] Update template and comments --- configuration/portal_settings.ini | 6 ++---- templates/portal_settings.template | 7 +++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/configuration/portal_settings.ini b/configuration/portal_settings.ini index 585bcf4090..ae0480e2d1 100644 --- a/configuration/portal_settings.ini +++ b/configuration/portal_settings.ini @@ -151,11 +151,9 @@ database = "mod_hpcdb" [slurm] sacct = "sacct" -; configuration for XDMoD data warehouse export functionality -; +; Configuration for data warehouse export functionality. [data_warehouse_export] -; Exported data files will be stored in this directory +; Exported data files will be stored in this directory. export_directory = "/var/spool/xdmod/export" ; Length of time in days that files will be retained before automatic deletion. retention_duration_days = 30 - diff --git a/templates/portal_settings.template b/templates/portal_settings.template index 0f6bf0505d..ef07c822b5 100644 --- a/templates/portal_settings.template +++ b/templates/portal_settings.template @@ -150,3 +150,10 @@ database = "mod_hpcdb" [slurm] sacct = "[:slurm_sacct:]" + +; Configuration for data warehouse export functionality. +[data_warehouse_export] +; Exported data files will be stored in this directory. +export_directory = "[:data_warehouse_export_export_directory:]" +; Length of time in days that files will be retained before automatic deletion. +retention_duration_days = [:data_warehouse_export_retention_duration_days:] From b5d040bd2931cae80827c96a8b5327f1909c4e95 Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Fri, 31 May 2019 09:57:31 -0400 Subject: [PATCH 074/217] Ensure TRUE and FALSE are 1 and 0 in SQL for tests --- tests/component/lib/Export/ExportDBTest.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/component/lib/Export/ExportDBTest.php b/tests/component/lib/Export/ExportDBTest.php index c7c7e4fa1b..d281fc4ffd 100644 --- a/tests/component/lib/Export/ExportDBTest.php +++ b/tests/component/lib/Export/ExportDBTest.php @@ -43,8 +43,8 @@ private function findAvailableRecord() { // Find or create a record in Available status $maxAvailable = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE - export_succeeded IS TRUE - AND export_expired IS FALSE')[0]['id']; + export_succeeded = 1 + AND export_expired = 0')[0]['id']; if ($maxAvailable == null) { $query = new QueryHandler(); $maxSubmitted = $this->findSubmittedRecord(); @@ -57,7 +57,7 @@ private function findExpiredRecord() { // Find or create a record in Expired status $maxExpired = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE - export_expired IS TRUE')[0]['id']; + export_expired = 1')[0]['id']; if ($maxExpired == null) { $query = new QueryHandler(); $maxAvailable = $this->findAvailableRecord(); @@ -70,7 +70,7 @@ private function findFailedRecord() { // Find or create a record in Failed status $maxFailed = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE - export_succeeded IS FALSE')[0]['id']; + export_succeeded = 0')[0]['id']; if ($maxFailed == null) { $query = new QueryHandler(); $maxSubmitted = $this->findSubmittedRecord(); @@ -100,7 +100,7 @@ private function listSubmittedRecords() private function listAvailableRecords() { // List ids of records in Available state - $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests WHERE export_succeeded IS TRUE and export_expired IS FALSE'); + $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests WHERE export_succeeded = 1 and export_expired = 0'); $retval = $this->flattenRecords($availableArr); return($retval); } @@ -108,7 +108,7 @@ private function listAvailableRecords() private function listExpiredRecords() { // List ids of records in Expired state - $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests WHERE export_succeeded IS TRUE and export_expired IS TRUE'); + $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests WHERE export_succeeded = 1 and export_expired = 1'); $retval = $this->flattenRecords($availableArr); return($retval); } @@ -116,7 +116,7 @@ private function listExpiredRecords() private function listFailedRecords() { // List ids of records in Failed state - $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests WHERE export_succeeded IS FALSE and export_expired IS FALSE'); + $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests WHERE export_succeeded = 0 and export_expired = 0'); $retval = $this->flattenRecords($availableArr); return($retval); } From 60c0d45e080e95a5a93ac57bfddc48fd40229cf6 Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Fri, 31 May 2019 10:34:44 -0400 Subject: [PATCH 075/217] doing some stuff to my tests... --- tests/component/lib/Export/ExportDBTest.php | 195 ++++++++++---------- 1 file changed, 102 insertions(+), 93 deletions(-) diff --git a/tests/component/lib/Export/ExportDBTest.php b/tests/component/lib/Export/ExportDBTest.php index d281fc4ffd..45fa485281 100644 --- a/tests/component/lib/Export/ExportDBTest.php +++ b/tests/component/lib/Export/ExportDBTest.php @@ -123,25 +123,7 @@ private function listFailedRecords() /* *********** PUBLIC TESTS *********** */ - public function testCountSubmitted() - { - $query = new QueryHandler(); - $submittedCount = $query->countSubmittedRecords(); - - $submittedList = $this->listSubmittedRecords(); - - $this->assertEquals($submittedCount, count($submittedList)); - $this->assertNotNull($submittedCount); - $this->assertTrue($submittedCount>=0); - - // debug - if (self::$debug) - { - print("\n".__FUNCTION__.": submittedRecords=$submittedCount\n"); - } - } - - // Create two new records in Submitted state. + // Create three new records in Submitted state. public function testNewRecordCreation() { $query = new QueryHandler(); @@ -158,65 +140,89 @@ public function testNewRecordCreation() $requestId2 = $query->createRequestRecord($userId, 'Accounts', '2016-12-01', '2017-01-01','JSON'); $this->assertNotNull($requestId2 ); + // Add another new record and verify + $requestId3 = $query->createRequestRecord($userId, 'Jobs', '2014-01-05', '2014-01-26','CSV'); + $this->assertNotNull($requestId3 ); + // Determine final count $finalCount = $query->countSubmittedRecords(); - // Verify final count. Should have added 2 records. - $this->assertTrue($finalCount-$initialCount==2); + // Verify final count. Should have added 3 records. + $this->assertTrue($finalCount-$initialCount==3); // Verify newly created records are found in list of Submitted status records $allSubmitted = $this->listSubmittedRecords(); + $this->assertContains($requestId3, $allSubmitted); $this->assertContains($requestId2, $allSubmitted); $this->assertContains($requestId, $allSubmitted); // debug if (self::$debug) { - print("\n".__FUNCTION__.": initialCount=$initialCount finalCount=$finalCount requestId=$requestId requestId2=$requestId2\n"); + print("\n".__FUNCTION__.": initialCount=$initialCount finalCount=$finalCount requestId=$requestId + requestId2=$requestId2 requestId3=$requestId3\n"); } } - public function testSubmittedToFailed() + public function testCountSubmitted() { $query = new QueryHandler(); + $submittedCount = $query->countSubmittedRecords(); - // Find or create a record in submitted status to transition - $maxSubmitted = $this->findSubmittedRecord(); - - // Transition the record to Failed - $result = $query->submittedToFailed($maxSubmitted); - - // Assert that: - // Exactly one record was transitioned - $this->assertTrue($result==1); + $submittedList = $this->listSubmittedRecords(); - // This record is now marked Failed - $allFailed = $this->listFailedRecords(); - $this->assertContains($maxSubmitted, $allFailed); + $this->assertEquals($submittedCount, count($submittedList)); + $this->assertNotNull($submittedCount); + $this->assertTrue($submittedCount>=0); // debug if (self::$debug) { - print("\n".__FUNCTION__.": NON transitioned Id=$maxSubmitted\n"); + print("\n".__FUNCTION__.": submittedRecords=$submittedCount\n"); } } - public function testSubmittedToAvailable() + // Verify field list returned from listSubmittedRecords() + public function testSubmittedRecordFieldList() + { + $query = new QueryHandler(); + + // Expect these keys from the associative array + $expectedKeys = array( + 'id', + 'realm', + 'start_date', + 'end_date', + 'export_file_format', + 'requested_datetime' + ); + + // List all records in Submitted state: + $actual = $query->listSubmittedRecords(); + + if (count($actual) > 0) { + + // assert that the expected fields are returned from the query + $this->assertEquals($expectedKeys, array_keys($actual[0])); + } + } + public function testSubmittedToFailed() { $query = new QueryHandler(); // Find or create a record in submitted status to transition $maxSubmitted = $this->findSubmittedRecord(); - $result = $query->submittedToAvailable($maxSubmitted); + // Transition the record to Failed + $result = $query->submittedToFailed($maxSubmitted); // Assert that: // Exactly one record was transitioned $this->assertTrue($result==1); - // That record is marked Available - $allAvailable = $this->listAvailableRecords(); - $this->assertContains($maxSubmitted, $allAvailable); + // This record is now marked Failed + $allFailed = $this->listFailedRecords(); + $this->assertContains($maxSubmitted, $allFailed); // debug if (self::$debug) @@ -249,27 +255,27 @@ public function testSubmittedToExpired() } } - public function testAvailableToExpired() + public function testSubmittedToAvailable() { $query = new QueryHandler(); - // Find or create a record in Available status to transition - $maxAvailable = $this->findAvailableRecord(); + // Find or create a record in submitted status to transition + $maxSubmitted = $this->findSubmittedRecord(); - $result = $query->availableToExpired($maxAvailable); + $result = $query->submittedToAvailable($maxSubmitted); // Assert that: // Exactly one record was transitioned $this->assertTrue($result==1); - // That record is marked export_expired=TRUE - $allExpired = $this->listExpiredRecords(); - $this->assertContains($maxAvailable, $allExpired); + // That record is marked Available + $allAvailable = $this->listAvailableRecords(); + $this->assertContains($maxSubmitted, $allAvailable); // debug if (self::$debug) { - print("\n".__FUNCTION__.": transitioned Id=$maxAvailable\n"); + print("\n".__FUNCTION__.": transitioned Id=$maxSubmitted\n"); } } @@ -298,6 +304,31 @@ public function testAvailableToFailed() } } + public function testAvailableToExpired() + { + $query = new QueryHandler(); + + // Find or create a record in Available status to transition + $maxAvailable = $this->findAvailableRecord(); + + $result = $query->availableToExpired($maxAvailable); + + // Assert that: + // Exactly one record was transitioned + $this->assertTrue($result==1); + + // That record is marked export_expired=TRUE + $allExpired = $this->listExpiredRecords(); + $this->assertContains($maxAvailable, $allExpired); + + // debug + if (self::$debug) + { + print("\n".__FUNCTION__.": transitioned Id=$maxAvailable\n"); + } + } + + public function testExpiredToFailed() { $query = new QueryHandler(); @@ -395,29 +426,6 @@ public function testFailedToAvailable() } } - // Verify field list returned from listSubmittedRecords() - public function testSubmittedRecordFieldList() - { - $query = new QueryHandler(); - - // Expect these keys from the associative array - $expectedKeys = array( - 'id', - 'realm', - 'start_date', - 'end_date', - 'export_file_format', - 'requested_datetime' - ); - - $actual = $query->listSubmittedRecords(); - - if (count($actual) > 0) { - - // assert that the expected fields are returned from the query - $this->assertEquals($expectedKeys, array_keys($actual[0])); - } - } // Verify field list returned from listRequestsForUser() public function testUserRecordFieldList() @@ -480,56 +488,57 @@ public function testUserRecordReportStates() } } - // Verify that user that created request can delete it - public function testRecordDeleteCorrectUser() + // Verify that user that did not create request cannot delete it + public function testRecordDeleteIncorrectUser() { $query = new QueryHandler(); $userId = $this->acquireUserId(); + $wrongUserId = XDUser::getUserByUserName(self::CENTER_STAFF_USER_NAME)->getUserID(); - // Requests via this user have been created as part of these tests + // Requests via user with $userId have been created as part of these tests $actual = $query->listRequestsForUser($userId); - if (count($actual) > 0) - { + // Provided that we have specified two different users here: + if (count($actual) > 0 && $userId != $wrongUserId) { + // pick the first such request and try to delete it: - $testVal = $query->deleteRequest($actual[0]['id'], $userId); + $testVal = $query->deleteRequest($actual[0]['id'], $wrongUserId); - // assert that the delete affected 1 row - $this->assertEquals($testVal, 1); + // assert that the delete attempt affected 0 rows + $this->assertEquals($testVal, 0); if (self::$debug) { - print("\n".__FUNCTION__.": deleted record? $testVal\n"); + print("\n".__FUNCTION__.": deleted record id=".$actual[0]['id']." ? $testVal\n"); } } } - // Verify that user that did not create request cannot delete it - public function testRecordDeleteIncorrectUser() + // Verify that user that created request can delete it + public function testRecordDeleteCorrectUser() { $query = new QueryHandler(); $userId = $this->acquireUserId(); - $wrongUserId = XDUser::getUserByUserName(self::CENTER_STAFF_USER_NAME)->getUserID(); - // Requests via user with $userId have been created as part of these tests + // Requests via this user have been created as part of these tests $actual = $query->listRequestsForUser($userId); - // Provided that we have specified two different users here: - if (count($actual) > 0 && $userId != $wrongUserId) { - + if (count($actual) > 0) + { // pick the first such request and try to delete it: - $testVal = $query->deleteRequest($actual[0]['id'], $wrongUserId); + $testVal = $query->deleteRequest($actual[0]['id'], $userId); - // assert that the delete attempt affected 0 rows - $this->assertEquals($testVal, 0); + // assert that the delete affected 1 row + $this->assertEquals($testVal, 1); if (self::$debug) { - print("\n".__FUNCTION__.": deleted record? $testVal\n"); + print("\n".__FUNCTION__.": deleted record id=".$actual[0]['id']." ? $testVal\n"); } } } + public static function setUpBeforeClass() { // setup needed to use NORMAL_USER_USER_NAME or the like @@ -543,6 +552,6 @@ public static function setUpBeforeClass() public static function tearDownAfterClass() { // Reset the batch_export_requests database table to its initial contents - static::$dbh->execute('DELETE FROM batch_export_requests WHERE id > :id', array('id' => static::$maxId)); + // static::$dbh->execute('DELETE FROM batch_export_requests WHERE id > :id', array('id' => static::$maxId)); } } From fa08b80fbd1cdf696f81a98745d6c10ad80cfe4e Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 11:00:43 -0400 Subject: [PATCH 076/217] Use request state data from internal API --- .../WarehouseExportControllerProvider.php | 2 +- html/gui/js/modules/DataExport.js | 14 +------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index a760e5ba78..008e63b103 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -70,7 +70,7 @@ public function getRequests(Request $request, Application $app) { $user = $this->authorize($request); $handler = new QueryHandler(); - $results = $handler->listRequestsForUser($user->getUserId()); + $results = $handler->listUserRequestsByState($user->getUserId()); return $app->json(['success' => true, 'data' => $results]); } diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 10dfb56b39..44de222f79 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -422,19 +422,7 @@ XDMoD.Module.DataExport.RequestsStore = Ext.extend(Ext.data.JsonStore, { }, { name: 'state', - convert: function (v, record) { - // TODO: Replace this with data from the server. - - if (record.export_expired === '1') { - return 'Expired'; - } - - if (record.export_created_datetime !== null) { - return 'Available'; - } - - return 'Submitted'; - } + type: 'string' } ] }); From 1b100242e6ed697e489de6f871cb9a01e0e9e4d2 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 11:53:34 -0400 Subject: [PATCH 077/217] Change state column colors and action buttons --- html/gui/js/modules/DataExport.js | 85 +++++++++++++++++++++++++++---- 1 file changed, 74 insertions(+), 11 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 44de222f79..dcd2db9159 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -273,7 +273,23 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { }, { header: 'State', - dataIndex: 'state' + dataIndex: 'state', + renderer: function (value, metaData) { + switch (value) { + case 'Available': + metaData.attr = 'style="background-color:#040"'; + break; + case 'Expired': + case 'Failed': + metaData.attr = 'style="background-color:#f00"'; + break; + case 'Submitted': + metaData.attr = 'style="background-color:yellow"'; + break; + default: + } + return value; + } }, { header: 'Realm', @@ -303,16 +319,39 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { }, { header: 'Actions', - xtype: 'templatecolumn', - tpl: new Ext.XTemplate( - '', - '', - ' ', - '', - '', - ' ', - '' - ) + xtype: 'actioncolumn', + dataIndex: 'state', + scope: this, + items: [ + { + icon: 'gui/images/report_generator/delete_report.png', + tooltip: 'Delete Request', + iconCls: 'data-export-action-icon', + handler: function (grid, rowIndex) { + this.deleteRequest(grid.store.getAt(rowIndex)); + } + }, + { + icon: 'gui/images/report_generator/download_report.png', + tooltip: 'Download Exported Data', + getClass: function (state, metaData) { + return 'data-export-action-icon' + (state !== 'Available' ? '-hidden': ''); + }, + handler: function (grid, rowIndex) { + this.downloadRequest(grid.store.getAt(rowIndex)); + } + }, + { + icon: 'gui/images/arrow_redo.png', + tooltip: 'Resumbit Request', + getClass: function (state, metaData) { + return 'data-export-action-icon' + (state !== 'Expired' && state !== 'Failed' ? '-hidden': ''); + }, + handler: function (grid, rowIndex) { + this.resubmitRequest(grid.store.getAt(rowIndex)); + } + } + ] } ], bbar: [ @@ -351,6 +390,30 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { reload: function () { this.store.reload(); + }, + + deleteRequest: function (record) { + Ext.Msg.confirm( + 'Delete Request', + 'Are you sure that you want to delete this request? You cannot undo this operation.', + function (selection) { + if (selection === 'yes') { + console.log({ 'delete': record }); + Ext.Msg.alert('TODO', 'TODO: Delete the request'); + } + }, + this + ); + }, + + downloadRequest: function (record) { + // TODO + console.log({ download: record }); + }, + + resubmitRequest: function (record) { + // TODO + console.log({ resubmit: record }); } }); From 8ab56f134bbae128bac7d8f84843b2bec79b5921 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 11:55:58 -0400 Subject: [PATCH 078/217] Refactor error handler and "Delete all ..." button --- html/gui/js/modules/DataExport.js | 60 ++++++++++++++++--------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index dcd2db9159..1871cadee6 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -166,21 +166,24 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { XDMoD.Module.DataExport.RequestForm.superclass.initComponent.call(this); this.getForm().on('actionfailed', function (form, action) { - if (action.failureType === Ext.form.Action.CLIENT_INVALID) { - // It shouldn't be possible to submit and invalid form, but it - // does happen display an error message. - Ext.Msg.alert('Error', 'Validation failed, please check input values and resubmit.'); - } else if (action.failureType === Ext.form.Action.CONNECT_FAILURE || action.failureType === Ext.form.Action.SERVER_INVALID) { - var response = action.response; - Ext.Msg.alert( - response.statusText || 'Error', - JSON.parse(response.responseText).message || 'Unknown Error' - ); - } else if (action.failureType === Ext.form.Action.LOAD_FAILURE) { - // This error occurs when the server doesn't return anything. - Ext.Msg.alert('Submission Error', 'Failed to submit request, try again later.'); - } else { - Ext.Msg.alert('Unknown Error', 'An unknown error occured, try again later.'); + switch (action.failureType) { + case Ext.form.Action.CLIENT_INVALID: + // It shouldn't be possible to submit and invalid form, but it + // does happen display an error message. + Ext.Msg.alert('Error', 'Validation failed, please check input values and resubmit.'); + break; + case Ext.form.Action.CONNECT_FAILURE: + case Ext.form.Action.SERVER_INVALID: + var response = action.response; + Ext.Msg.alert( + response.statusText || 'Error', + JSON.parse(response.responseText).message || 'Unknown Error' + ); + case Ext.form.Action.LOAD_FAILURE: + // This error occurs when the server doesn't return anything. + Ext.Msg.alert('Submission Error', 'Failed to submit request, try again later.'); + default: + Ext.Msg.alert('Unknown Error', 'An unknown error occured, try again later.'); } }); }, @@ -359,19 +362,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { xtype: 'button', text: 'Delete all expired requests', scope: this, - handler: function () { - Ext.Msg.confirm( - 'Delete All Expired Requests', - 'Are you sure that you want to delete all expired requests? You cannot undo this operation.', - function (selection) { - this.reload(); - if (selection === 'yes') { - Ext.Msg.alert('TODO', 'TODO: Delete all the expired requests'); - } - }, - this - ); - } + handler: this.deleteExpiredRequests }, '->', { @@ -392,6 +383,19 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { this.store.reload(); }, + deleteExpiredRequests: function () { + Ext.Msg.confirm( + 'Delete All Expired Requests', + 'Are you sure that you want to delete all expired requests? You cannot undo this operation.', + function (selection) { + if (selection === 'yes') { + Ext.Msg.alert('TODO', 'TODO: Delete all the expired requests'); + } + }, + this + ); + }, + deleteRequest: function (record) { Ext.Msg.confirm( 'Delete Request', From 61f2b9da0d4472831ac932df7ce25daf5c45a794 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 12:00:23 -0400 Subject: [PATCH 079/217] Add styles --- html/gui/css/viewer.css | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/html/gui/css/viewer.css b/html/gui/css/viewer.css index be672dfed0..2d864a8870 100644 --- a/html/gui/css/viewer.css +++ b/html/gui/css/viewer.css @@ -781,3 +781,12 @@ div.chart_thumb a { .jobviewer_helpcontent strong { font-weight: bold; } + +/* Data warehouse export module */ +img.data-export-action-icon { + margin-right: 3px; +} + +img.data-export-action-icon-hidden { + display: none; +} From 0820727af8a60127484a2e33fd093857a65566ee Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 12:00:52 -0400 Subject: [PATCH 080/217] Remove unused arguments --- html/gui/js/modules/DataExport.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 1871cadee6..a019c628d3 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -337,7 +337,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { { icon: 'gui/images/report_generator/download_report.png', tooltip: 'Download Exported Data', - getClass: function (state, metaData) { + getClass: function (state) { return 'data-export-action-icon' + (state !== 'Available' ? '-hidden': ''); }, handler: function (grid, rowIndex) { @@ -347,7 +347,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { { icon: 'gui/images/arrow_redo.png', tooltip: 'Resumbit Request', - getClass: function (state, metaData) { + getClass: function (state) { return 'data-export-action-icon' + (state !== 'Expired' && state !== 'Failed' ? '-hidden': ''); }, handler: function (grid, rowIndex) { From 4ac325ec4b2aca9c1e97a6c557dab15e88855a78 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 12:50:29 -0400 Subject: [PATCH 081/217] Add missing break statements --- html/gui/js/modules/DataExport.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index a019c628d3..8f08ff6088 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -179,9 +179,11 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { response.statusText || 'Error', JSON.parse(response.responseText).message || 'Unknown Error' ); + break; case Ext.form.Action.LOAD_FAILURE: // This error occurs when the server doesn't return anything. Ext.Msg.alert('Submission Error', 'Failed to submit request, try again later.'); + break; default: Ext.Msg.alert('Unknown Error', 'An unknown error occured, try again later.'); } From e16d78000a58e0be2abc9e2001afb4f9ddf50337 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 12:50:58 -0400 Subject: [PATCH 082/217] Fix style --- html/gui/js/modules/DataExport.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 8f08ff6088..1d1dbcb5d7 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -340,7 +340,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { icon: 'gui/images/report_generator/download_report.png', tooltip: 'Download Exported Data', getClass: function (state) { - return 'data-export-action-icon' + (state !== 'Available' ? '-hidden': ''); + return 'data-export-action-icon' + (state !== 'Available' ? '-hidden' : ''); }, handler: function (grid, rowIndex) { this.downloadRequest(grid.store.getAt(rowIndex)); @@ -350,7 +350,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { icon: 'gui/images/arrow_redo.png', tooltip: 'Resumbit Request', getClass: function (state) { - return 'data-export-action-icon' + (state !== 'Expired' && state !== 'Failed' ? '-hidden': ''); + return 'data-export-action-icon' + (state !== 'Expired' && state !== 'Failed' ? '-hidden' : ''); }, handler: function (grid, rowIndex) { this.resubmitRequest(grid.store.getAt(rowIndex)); From 76e90e87c147d9bffaa69c036f6e2b91f862a307 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 12:51:18 -0400 Subject: [PATCH 083/217] Remove console debugging --- html/gui/js/modules/DataExport.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 1d1dbcb5d7..793cf1e961 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -404,7 +404,6 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { 'Are you sure that you want to delete this request? You cannot undo this operation.', function (selection) { if (selection === 'yes') { - console.log({ 'delete': record }); Ext.Msg.alert('TODO', 'TODO: Delete the request'); } }, @@ -414,12 +413,10 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { downloadRequest: function (record) { // TODO - console.log({ download: record }); }, resubmitRequest: function (record) { // TODO - console.log({ resubmit: record }); } }); From f2b0e9fc3f627e1a945bc28b24494e7c6550fc34 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 12:51:56 -0400 Subject: [PATCH 084/217] Refactor realm store usage --- html/gui/js/modules/DataExport.js | 52 ++++++++++++++++++------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 793cf1e961..0f7ca4cb87 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -18,11 +18,21 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { initComponent: function () { this.requestsStore = new XDMoD.Module.DataExport.RequestsStore(); + this.realmsStore = new Ext.data.JsonStore({ + url: 'rest/v1/warehouse/export/realms', + root: 'data', + fields: [ + { name: 'id', type: 'string' }, + { name: 'name', type: 'string' } + ] + }); + this.requestForm = new XDMoD.Module.DataExport.RequestForm({ title: 'Create Bulk Data Export Request', bodyStyle: 'padding: 5px 5px 0 5px', border: false, - region: 'north' + region: 'north', + realmsStore: this.realmsStore }); this.requestsGrid = new XDMoD.Module.DataExport.RequestsGrid({ @@ -30,11 +40,20 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { region: 'center', margins: '2 2 2 0', pageSize: this.defaultPageSize, + realmsStore: this.realmsStore, store: this.requestsStore }); - this.requestsGrid.on('afterrender', this.requestsStore.load, this.requestsStore, { single: true }); - this.requestForm.on('actioncomplete', this.requestsGrid.reload, this.requestsGrid); + // Defer loading of realms so they are not loaded immediately. + this.on('beforerender', this.realmsStore.load, this.realmsStore, { single: true }); + + // Load the requests after the realms have loaded. This is necessary so + // that the realm name can be determined from its ID when displayed in + // the grid. + this.realmsStore.on('load', this.requestsStore.load, this.requestsStore, { single: true }); + + // Reload the requests every time a new request is submitted. + this.requestForm.on('actioncomplete', this.requestsStore.reload, this.requestsStore); this.items = [ { @@ -105,17 +124,7 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { editable: false, triggerAction: 'all', mode: 'local', - store: { - xtype: 'jsonstore', - autoLoad: true, - autoDestroy: true, - url: 'rest/v1/warehouse/export/realms', - root: 'data', - fields: [ - { name: 'id', type: 'string' }, - { name: 'name', type: 'string' } - ] - } + store: this.realmsStore }, { xtype: 'datefield', @@ -298,7 +307,11 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { }, { header: 'Realm', - dataIndex: 'realm' + dataIndex: 'realmId', + scope: this, + renderer: function (value) { + return this.realmsStore.getById(value).get('name'); + } }, { header: 'Data Start Date', @@ -381,10 +394,6 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { XDMoD.Module.DataExport.RequestsGrid.superclass.initComponent.call(this); }, - reload: function () { - this.store.reload(); - }, - deleteExpiredRequests: function () { Ext.Msg.confirm( 'Delete All Expired Requests', @@ -450,8 +459,9 @@ XDMoD.Module.DataExport.RequestsStore = Ext.extend(Ext.data.JsonStore, { type: 'int' }, { - name: 'realm', - type: 'string' + name: 'realmId', + type: 'string', + mapping: 'realm' }, { name: 'start_date', From d0a9429bdea3c802bfae7c09dab9474308114c43 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 12:53:53 -0400 Subject: [PATCH 085/217] Add eslint exceptions --- html/gui/js/modules/DataExport.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 0f7ca4cb87..5a25c8f9e7 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -291,14 +291,14 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { renderer: function (value, metaData) { switch (value) { case 'Available': - metaData.attr = 'style="background-color:#040"'; + metaData.attr = 'style="background-color:#040"'; // eslint-disable-line no-param-reassign break; case 'Expired': case 'Failed': - metaData.attr = 'style="background-color:#f00"'; + metaData.attr = 'style="background-color:#f00"'; // eslint-disable-line no-param-reassign break; case 'Submitted': - metaData.attr = 'style="background-color:yellow"'; + metaData.attr = 'style="background-color:yellow"'; // eslint-disable-line no-param-reassign break; default: } From 67343a0f2810f022c252cb58e4b46302ed2eb126 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 14:03:40 -0400 Subject: [PATCH 086/217] Implement more REST endpoints --- .../WarehouseExportControllerProvider.php | 182 +++++++++++++++++- 1 file changed, 177 insertions(+), 5 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 008e63b103..db380d41ab 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -2,11 +2,15 @@ namespace Rest\Controllers; +use CCR\DB; use DataWarehouse\Export\QueryHandler; +use Exception; use Silex\Application; use Silex\ControllerCollection; use Symfony\Component\HttpFoundation\Request; +use Symfony\Component\HttpKernel\Exception\AccessDeniedHttpException; use Symfony\Component\HttpKernel\Exception\BadRequestHttpException; +use Symfony\Component\HttpKernel\Exception\NotFoundHttpException; class WarehouseExportControllerProvider extends BaseControllerProvider { @@ -22,9 +26,21 @@ public function setupRoutes( ) { $root = $this->prefix; $current = get_class($this); + $conversions = '\Rest\Utilities\Conversions'; + $controller->get("$root/realms", "$current::getRealms"); $controller->get("$root/requests", "$current::getRequests"); $controller->post("$root/request", "$current::createRequest"); + + $controller->get("$root/request/{id}", "$current::getRequest") + ->assert('id', '\d+') + ->convert('id', "$conversions::toInt"); + + $controller->delete("$root/request/{id}", "$current::deleteRequest") + ->assert('id', '\d+') + ->convert('id', "$conversions::toInt"); + + $controller->delete("$root/requests/{id}", "$current::deleteRequests"); } /** @@ -32,8 +48,8 @@ public function setupRoutes( * * @param Request $request * @param Application $app - * @return array - * @throws AccessDeniedException + * @return \Symfony\Component\HttpFoundation\JsonResponse + * @throws \Symfony\Component\HttpKernel\Exception\UnauthorizedHttpException */ public function getRealms(Request $request, Application $app) { @@ -63,8 +79,8 @@ public function getRealms(Request $request, Application $app) * * @param Request $request * @param Application $app - * @return array - * @throws AccessDeniedException + * @return \Symfony\Component\HttpFoundation\JsonResponse + * @throws \Symfony\Component\HttpKernel\Exception\UnauthorizedHttpException */ public function getRequests(Request $request, Application $app) { @@ -76,6 +92,12 @@ public function getRequests(Request $request, Application $app) /** * Create a new export request for the current user. + * + * @param Request $request + * @param Application $app + * @return \Symfony\Component\HttpFoundation\JsonResponse + * @throws \Symfony\Component\HttpKernel\Exception\UnauthorizedHttpException + * @throws BadRequestHttpException */ public function createRequest(Request $request, Application $app) { @@ -126,6 +148,156 @@ public function createRequest(Request $request, Application $app) $format ); - return $app->json(['success' => true, 'data' => ['id' => $id]]); + return $app->json([ + 'success' => true, + 'message' => 'Created export request', + 'data' => ['id' => $id] + ]); + } + + /** + * Get the requested data. + * + * @param Request $request + * @param Application $app + * @param int $id + * @return \Symfony\Component\HttpFoundation\BinaryFileResponse + * @throws \Symfony\Component\HttpKernel\Exception\UnauthorizedHttpException + * @throws AccessDeniedHttpException + * @throws NotFoundHttpException + * @throws BadRequestHttpException + */ + public function getRequest(Request $request, Application $app, $id) + { + $user = $this->authorize($request); + $handler = new QueryHandler(); + + $requests = array_filter( + $handler->listUserRequestsByState($user->getUserId()), + function ($request) use ($id) { + return $request['id'] === $id; + } + ); + + if (count($requests) === 0) { + throw new NotFoundHttpException(); + } + + $request = $requests[0]; + + if ($request['state'] !== 'Available') { + throw new BadRequestHttpException('Requested data is not available'); + } + + try { + $exportDir = xd_utilities\getConfiguration('data_warehouse_export', 'export_directory'); + } catch (Exception $e) { + throw new BadRequestHttpException('Export directory is not configured'); + } + + $file = sprintf( + '%s/%s.%s', + $exportDir, + $id, + strtolower($request['format']) + ); + + if (!is_readable($file)) { + throw new AccessDeniedHttpException(); + } + + return $this->sendFile($file); + } + + /** + * Delete a single request. + * + * @param Request $request + * @param Application $app + * @param int $id + * @return \Symfony\Component\HttpFoundation\JsonResponse + * @throws \Symfony\Component\HttpKernel\Exception\UnauthorizedHttpException + * @throws NotFoundHttpException + */ + public function deleteRequest(Request $request, Application $app, $id) + { + $user = $this->authorize($request); + $handler = new QueryHandler(); + $count = $handler->deleteRequest($id, $user->getUserId()); + + if ($count === 0) { + throw new NotFoundHttpException(); + } + + return $app->json([ + 'success' => true, + 'message' => 'Deleted export request' + ]); + } + + /** + * Delete multiple requests. + * + * The request body content must be a JSON encoded array of request IDs. + * + * @param Request $request + * @param Application $app + * @return \Symfony\Component\HttpFoundation\JsonResponse + * @throws \Symfony\Component\HttpKernel\Exception\UnauthorizedHttpException + * @throws NotFoundHttpException + */ + public function deleteRequests(Request $request, Application $app) + { + $user = $this->authorize($request); + $handler = new QueryHandler(); + + $requestIds = []; + + try { + $requestIds = json_decode($request->getContent()); + + if ($requestIds === null) { + throw new Exception('Failed to decode JSON'); + } + + if (!is_array($requestIds)) { + throw new Exception('Request IDs must be in an array'); + } + + foreach ($requestIds as $id) { + if (!is_int($id)) { + throw new Exception('Request IDs must integers'); + } + } + } catch (Exception $e) { + throw new BadRequestHttpException( + 'Malformed HTTP request content: ' . $e->getMessage() + ); + } + + try { + $dbh = DB::factory('database'); + $dbh->beginTransaction(); + + foreach ($requestIds as $id) { + $count = $handler->deleteRequest($id, $user->getUserId()); + if ($count === 0) { + throw new NotFoundHttpException(); + } + } + + $dbh->commit(); + } catch (NotFoundHttpException $e) { + $dbh->rollBack(); + throw $e; + } catch (Exception $e) { + $dbh->rollBack(); + throw new BadRequestHttpException('Failed to delete export requests'); + } + + return $app->json([ + 'success' => true, + 'message' => 'Deleted export requests' + ]); } } From 69a38600b49b5b0f2b03302de524661eca5fd687 Mon Sep 17 00:00:00 2001 From: Jeanette Sperhac Date: Fri, 31 May 2019 15:57:05 -0400 Subject: [PATCH 087/217] Rearrange and simplify tests for data warehouse export query class --- tests/component/lib/Export/ExportDBTest.php | 323 +++++++++++--------- 1 file changed, 171 insertions(+), 152 deletions(-) diff --git a/tests/component/lib/Export/ExportDBTest.php b/tests/component/lib/Export/ExportDBTest.php index 45fa485281..8790ab80d1 100644 --- a/tests/component/lib/Export/ExportDBTest.php +++ b/tests/component/lib/Export/ExportDBTest.php @@ -28,97 +28,63 @@ private function acquireUserId() private function findSubmittedRecord() { - // Find or create a record in Submitted status - $maxSubmitted = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE export_succeeded IS NULL')[0]['id']; - - if ($maxSubmitted == null) { - $query = new QueryHandler(); - $userId = $this->acquireUserId(); - $maxSubmitted = $query->createRequestRecord($userId, 'Jobs', '2017-01-01', '2017-08-01','CSV'); - } - return($maxSubmitted); + // Find a record in Submitted status + return static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE export_succeeded IS NULL')[0]['id']; } private function findAvailableRecord() { - // Find or create a record in Available status - $maxAvailable = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE + // Find a record in Available status + return static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE export_succeeded = 1 AND export_expired = 0')[0]['id']; - if ($maxAvailable == null) { - $query = new QueryHandler(); - $maxSubmitted = $this->findSubmittedRecord(); - $maxAvailable = $query->submittedToAvailable($maxSubmitted); - } - return($maxAvailable); } private function findExpiredRecord() { - // Find or create a record in Expired status - $maxExpired = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE + // Find a record in Expired status + return static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE export_expired = 1')[0]['id']; - if ($maxExpired == null) { - $query = new QueryHandler(); - $maxAvailable = $this->findAvailableRecord(); - $maxExpired = $query->availableToExpired($maxAvailable); - } - return($maxExpired); } private function findFailedRecord() { - // Find or create a record in Failed status - $maxFailed = static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE + // Find a record in Failed status + return static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE export_succeeded = 0')[0]['id']; - if ($maxFailed == null) { - $query = new QueryHandler(); - $maxSubmitted = $this->findSubmittedRecord(); - $maxFailed = $query->submittedToFailed($maxSubmitted); - } - return($maxFailed); } - private function flattenRecords($inArr) + private function countSubmittedRecords() { - $ids = array(); - foreach($inArr as $arr) - { - $ids[] = $arr['id']; - } - return $ids; + // Count records in Submitted state + return static::$dbh->query('SELECT COUNT(id) AS count FROM batch_export_requests WHERE export_succeeded IS NULL')[0]['count']; } - private function listSubmittedRecords() + private function countAvailableRecords() { - // List ids of records in Submitted state - $submittedArr = static::$dbh->query('SELECT id FROM batch_export_requests WHERE export_succeeded IS NULL'); - $retval = $this->flattenRecords($submittedArr); - return($retval); + // Count records in Available state + return static::$dbh->query('SELECT COUNT(id) AS count FROM batch_export_requests WHERE export_succeeded = 1 and export_expired = 0')[0]['count']; } - private function listAvailableRecords() + private function countExpiredRecords() { - // List ids of records in Available state - $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests WHERE export_succeeded = 1 and export_expired = 0'); - $retval = $this->flattenRecords($availableArr); - return($retval); + // Count records in Expired state + return static::$dbh->query('SELECT COUNT(id) AS count FROM batch_export_requests WHERE export_succeeded = 1 and export_expired = 1')[0]['count']; } - private function listExpiredRecords() + private function countFailedRecords() { - // List ids of records in Expired state - $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests WHERE export_succeeded = 1 and export_expired = 1'); - $retval = $this->flattenRecords($availableArr); - return($retval); + // List ids of records in Failed state + return static::$dbh->query('SELECT COUNT(id) AS count FROM batch_export_requests WHERE export_succeeded = 0 and export_expired = 0')[0]['count']; } - private function listFailedRecords() + private function countUserRequests() { - // List ids of records in Failed state - $availableArr = static::$dbh->query('SELECT id FROM batch_export_requests WHERE export_succeeded = 0 and export_expired = 0'); - $retval = $this->flattenRecords($availableArr); - return($retval); + // Determine number of requests placed by this user + $params= array('user_id' => $this->acquireUserId()); + $sql = 'SELECT COUNT(id) AS count FROM batch_export_requests WHERE user_id=:user_id'; + $retval = static::$dbh->query($sql, $params); + return $retval[0]['count']; } /* *********** PUBLIC TESTS *********** */ @@ -129,7 +95,7 @@ public function testNewRecordCreation() $query = new QueryHandler(); $userId = $this->acquireUserId(); - // Find the count + // Find the Submitted record count $initialCount = $query->countSubmittedRecords(); // Add new record and verify @@ -146,15 +112,13 @@ public function testNewRecordCreation() // Determine final count $finalCount = $query->countSubmittedRecords(); + $finalCountTest = $this->countSubmittedRecords(); - // Verify final count. Should have added 3 records. + // Verify final Submitted count. Should have added 3 records. $this->assertTrue($finalCount-$initialCount==3); - // Verify newly created records are found in list of Submitted status records - $allSubmitted = $this->listSubmittedRecords(); - $this->assertContains($requestId3, $allSubmitted); - $this->assertContains($requestId2, $allSubmitted); - $this->assertContains($requestId, $allSubmitted); + // Verify test and class methods return same Submitted counts + $this->assertEquals($finalCount, $finalCountTest); // debug if (self::$debug) @@ -164,14 +128,14 @@ public function testNewRecordCreation() } } + // Verify counts of Submitted records public function testCountSubmitted() { $query = new QueryHandler(); $submittedCount = $query->countSubmittedRecords(); + $submittedCountTest = $this->countSubmittedRecords(); - $submittedList = $this->listSubmittedRecords(); - - $this->assertEquals($submittedCount, count($submittedList)); + $this->assertEquals($submittedCount, $submittedCountTest); $this->assertNotNull($submittedCount); $this->assertTrue($submittedCount>=0); @@ -200,29 +164,36 @@ public function testSubmittedRecordFieldList() // List all records in Submitted state: $actual = $query->listSubmittedRecords(); - if (count($actual) > 0) { + // assert that the expected fields are returned from the query + $this->assertEquals($expectedKeys, array_keys($actual[0])); - // assert that the expected fields are returned from the query - $this->assertEquals($expectedKeys, array_keys($actual[0])); - } + // assert that the expected number of records is returned from the query + $this->assertEquals($this->countSubmittedRecords(), count($actual)); } + public function testSubmittedToFailed() { $query = new QueryHandler(); - // Find or create a record in submitted status to transition - $maxSubmitted = $this->findSubmittedRecord(); + // initial counts + $submittedCountInitial = $this->countSubmittedRecords(); + $failedCountInitial = $this->countFailedRecords(); - // Transition the record to Failed + // Find a record in submitted status to transition + $maxSubmitted = $this->findSubmittedRecord(); $result = $query->submittedToFailed($maxSubmitted); + // final counts + $submittedCountFinal = $this->countSubmittedRecords(); + $failedCountFinal = $this->countFailedRecords(); + // Assert that: // Exactly one record was transitioned $this->assertTrue($result==1); - // This record is now marked Failed - $allFailed = $this->listFailedRecords(); - $this->assertContains($maxSubmitted, $allFailed); + // There is one fewer Submitted record, one more Failed. + $this->assertTrue($submittedCountFinal + 1 == $submittedCountInitial); + $this->assertTrue($failedCountFinal - 1 == $failedCountInitial); // debug if (self::$debug) @@ -235,18 +206,25 @@ public function testSubmittedToExpired() { $query = new QueryHandler(); - // Find or create a record in Submitted status to transition - $maxSubmitted = $this->findSubmittedRecord(); + // initial counts + $submittedCountInitial = $this->countSubmittedRecords(); + $expiredCountInitial = $this->countExpiredRecords(); + // Find a record in Submitted status to transition + $maxSubmitted = $this->findSubmittedRecord(); $result = $query->availableToExpired($maxSubmitted); + // final counts + $submittedCountFinal = $this->countSubmittedRecords(); + $expiredCountFinal = $this->countExpiredRecords(); + // Assert that: // Exactly zero records transitioned $this->assertTrue($result==0); - // That record is still marked Submitted - $allSubmitted = $this->listSubmittedRecords(); - $this->assertContains($maxSubmitted, $allSubmitted); + // No change in Submitted and Expired state counts occurred + $this->assertTrue($submittedCountFinal == $submittedCountInitial); + $this->assertTrue($expiredCountFinal == $expiredCountInitial); // debug if (self::$debug) @@ -259,18 +237,25 @@ public function testSubmittedToAvailable() { $query = new QueryHandler(); - // Find or create a record in submitted status to transition - $maxSubmitted = $this->findSubmittedRecord(); + // initial counts + $submittedCountInitial = $this->countSubmittedRecords(); + $availCountInitial = $this->countAvailableRecords(); + // Find a record in Submitted status to transition + $maxSubmitted = $this->findSubmittedRecord(); $result = $query->submittedToAvailable($maxSubmitted); + // final counts + $submittedCountFinal = $this->countSubmittedRecords(); + $availCountFinal = $this->countAvailableRecords(); + // Assert that: // Exactly one record was transitioned $this->assertTrue($result==1); - // That record is marked Available - $allAvailable = $this->listAvailableRecords(); - $this->assertContains($maxSubmitted, $allAvailable); + // There is one fewer Submitted record, one more Available. + $this->assertTrue($submittedCountFinal + 1 == $submittedCountInitial); + $this->assertTrue($availCountFinal - 1 == $availCountInitial); // debug if (self::$debug) @@ -283,19 +268,25 @@ public function testAvailableToFailed() { $query = new QueryHandler(); - // Find or create a record in Available status - $maxAvailable = $this->findAvailableRecord(); + // initial counts + $availCountInitial = $this->countAvailableRecords(); + $failCountInitial = $this->countFailedRecords(); - // Attempt to transition the record to Failed + // Find a record in Available status to transition + $maxAvailable = $this->findAvailableRecord(); $result = $query->submittedToFailed($maxAvailable); + // final counts + $availCountFinal = $this->countAvailableRecords(); + $failCountFinal = $this->countFailedRecords(); + // Assert that: // Exactly zero records were transitioned $this->assertTrue($result==0); - // This record is still marked Available - $allAvailable = $this->listAvailableRecords(); - $this->assertContains($maxAvailable, $allAvailable); + // No change in Available and Failed state counts occurred + $this->assertTrue($availCountFinal == $availCountInitial); + $this->assertTrue($failCountFinal == $failCountInitial); // debug if (self::$debug) @@ -308,18 +299,25 @@ public function testAvailableToExpired() { $query = new QueryHandler(); - // Find or create a record in Available status to transition - $maxAvailable = $this->findAvailableRecord(); + // initial counts + $availCountInitial = $this->countAvailableRecords(); + $expiredCountInitial = $this->countExpiredRecords(); + // Find a record in Available status to transition + $maxAvailable = $this->findAvailableRecord(); $result = $query->availableToExpired($maxAvailable); + // final counts + $availCountFinal = $this->countAvailableRecords(); + $expiredCountFinal = $this->countExpiredRecords(); + // Assert that: // Exactly one record was transitioned $this->assertTrue($result==1); - // That record is marked export_expired=TRUE - $allExpired = $this->listExpiredRecords(); - $this->assertContains($maxAvailable, $allExpired); + // There is one fewer Available record, one more Expired. + $this->assertTrue($availCountFinal + 1 == $availCountInitial); + $this->assertTrue($expiredCountFinal - 1 == $expiredCountInitial); // debug if (self::$debug) @@ -328,24 +326,29 @@ public function testAvailableToExpired() } } - public function testExpiredToFailed() { $query = new QueryHandler(); - // Find or create a record in Expired status - $maxExpired = $this->findExpiredRecord(); + // initial counts + $expiredCountInitial = $this->countExpiredRecords(); + $failCountInitial = $this->countFailedRecords(); - // Attempt to transition the record to Failed + // Find a record in Expired status to transition + $maxExpired = $this->findExpiredRecord(); $result = $query->submittedToFailed($maxExpired); + // final counts + $expiredCountFinal = $this->countExpiredRecords(); + $failCountFinal = $this->countFailedRecords(); + // Assert that: // Exactly zero records were transitioned $this->assertTrue($result==0); - // This record is still marked Expired - $allExpired = $this->listExpiredRecords(); - $this->assertContains($maxExpired, $allExpired); + // No change in Expired and Failed state counts occurred + $this->assertTrue($expiredCountFinal == $expiredCountInitial); + $this->assertTrue($failCountFinal == $failCountInitial); // debug if (self::$debug) @@ -358,18 +361,25 @@ public function testFailedToExpired() { $query = new QueryHandler(); + // initial counts + $expiredCountInitial = $this->countExpiredRecords(); + $failCountInitial = $this->countFailedRecords(); + // Find or create a record in Failed status to transition $maxFailed = $this->findFailedRecord(); - $result = $query->availableToExpired($maxFailed); + // final counts + $expiredCountFinal = $this->countExpiredRecords(); + $failCountFinal = $this->countFailedRecords(); + // Assert that: // Exactly zero records transitioned $this->assertTrue($result==0); - // That record is marked Failed - $allFailed = $this->listFailedRecords(); - $this->assertContains($maxFailed, $allFailed); + // No change in Expired and Failed state counts occurred + $this->assertTrue($expiredCountFinal == $expiredCountInitial); + $this->assertTrue($failCountFinal == $failCountInitial); // debug if (self::$debug) @@ -382,18 +392,25 @@ public function testExpiredToAvailable() { $query = new QueryHandler(); - // Find or create a record in Expired status to transition - $maxExpired = $this->findExpiredRecord(); + // initial counts + $expiredCountInitial = $this->countExpiredRecords(); + $availCountInitial = $this->countAvailableRecords(); + // Find a record in Expired status to transition + $maxExpired = $this->findExpiredRecord(); $result = $query->submittedToAvailable($maxExpired); + // final counts + $expiredCountFinal = $this->countExpiredRecords(); + $availCountFinal = $this->countAvailableRecords(); + // Assert that: // Exactly zero records transitioned $this->assertTrue($result==0); - // That record is marked Expired - $allExpired = $this->listExpiredRecords(); - $this->assertContains($maxExpired, $allExpired); + // No change in Expired and Available state counts occurred + $this->assertTrue($expiredCountFinal == $expiredCountInitial); + $this->assertTrue($availCountFinal == $availCountInitial); // debug if (self::$debug) @@ -406,18 +423,25 @@ public function testFailedToAvailable() { $query = new QueryHandler(); - // Find or create a record in Failed status to transition - $maxFailed = $this->findFailedRecord(); + // initial counts + $availCountInitial = $this->countAvailableRecords(); + $failCountInitial = $this->countFailedRecords(); + // Find a record in Failed status to transition + $maxFailed = $this->findFailedRecord(); $result = $query->submittedToAvailable($maxFailed); + // final counts + $availCountFinal = $this->countAvailableRecords(); + $failCountFinal = $this->countFailedRecords(); + // Assert that: // Exactly zero records transitioned $this->assertTrue($result==0); - // That record is marked Failed - $allFailed = $this->listFailedRecords(); - $this->assertContains($maxFailed, $allFailed); + // No change in Available and Failed state counts occurred + $this->assertTrue($availCountFinal == $availCountInitial); + $this->assertTrue($failCountFinal == $failCountInitial); // debug if (self::$debug) @@ -426,7 +450,6 @@ public function testFailedToAvailable() } } - // Verify field list returned from listRequestsForUser() public function testUserRecordFieldList() { @@ -450,11 +473,11 @@ public function testUserRecordFieldList() // Requests via this user have been created as part of these tests $actual = $query->listRequestsForUser($userId); - if (count($actual) > 0) { + // assert that the expected fields are returned from the query + $this->assertEquals($expectedKeys, array_keys($actual[0])); - // assert that the expected fields are returned from the query - $this->assertEquals($expectedKeys, array_keys($actual[0])); - } + // assert that the expected number of records is returned from the query + $this->assertEquals($this->countUserRequests(), count($actual)); } // Verify field list returned from listUserRequestsByState() @@ -481,11 +504,11 @@ public function testUserRecordReportStates() // Requests via this user have been created as part of these tests $actual = $query->listUserRequestsByState($userId); - if (count($actual) > 0) { + // assert that the expected fields are returned from the query + $this->assertEquals($expectedKeys, array_keys($actual[0])); - // assert that the expected fields are returned from the query - $this->assertEquals($expectedKeys, array_keys($actual[0])); - } + // assert that the expected number of records is returned from the query + $this->assertEquals($this->countUserRequests(), count($actual)); } // Verify that user that did not create request cannot delete it @@ -495,21 +518,21 @@ public function testRecordDeleteIncorrectUser() $userId = $this->acquireUserId(); $wrongUserId = XDUser::getUserByUserName(self::CENTER_STAFF_USER_NAME)->getUserID(); - // Requests via user with $userId have been created as part of these tests - $actual = $query->listRequestsForUser($userId); + // Requests via $userId have been created as part of these tests + $maxSubmitted = $this->findSubmittedRecord(); // Provided that we have specified two different users here: - if (count($actual) > 0 && $userId != $wrongUserId) { + if ($userId != $wrongUserId) { - // pick the first such request and try to delete it: - $testVal = $query->deleteRequest($actual[0]['id'], $wrongUserId); + // try to delete the request: + $actual = $query->deleteRequest($maxSubmitted, $wrongUserId); // assert that the delete attempt affected 0 rows - $this->assertEquals($testVal, 0); + $this->assertEquals($actual, 0); if (self::$debug) { - print("\n".__FUNCTION__.": deleted record id=".$actual[0]['id']." ? $testVal\n"); + print("\n".__FUNCTION__.": deleted record id=".$maxSubmitted." ? $actual\n"); } } } @@ -520,25 +543,21 @@ public function testRecordDeleteCorrectUser() $query = new QueryHandler(); $userId = $this->acquireUserId(); - // Requests via this user have been created as part of these tests - $actual = $query->listRequestsForUser($userId); + // Requests via $userId have been created as part of these tests + $maxSubmitted = $this->findSubmittedRecord(); - if (count($actual) > 0) - { - // pick the first such request and try to delete it: - $testVal = $query->deleteRequest($actual[0]['id'], $userId); + // try to delete the request: + $actual = $query->deleteRequest($maxSubmitted, $userId); - // assert that the delete affected 1 row - $this->assertEquals($testVal, 1); + // assert that the delete affected 1 row + $this->assertEquals($actual, 1); - if (self::$debug) - { - print("\n".__FUNCTION__.": deleted record id=".$actual[0]['id']." ? $testVal\n"); - } + if (self::$debug) + { + print("\n".__FUNCTION__.": deleted record id=".$maxSubmitted." ? $actual\n"); } } - public static function setUpBeforeClass() { // setup needed to use NORMAL_USER_USER_NAME or the like @@ -552,6 +571,6 @@ public static function setUpBeforeClass() public static function tearDownAfterClass() { // Reset the batch_export_requests database table to its initial contents - // static::$dbh->execute('DELETE FROM batch_export_requests WHERE id > :id', array('id' => static::$maxId)); + static::$dbh->execute('DELETE FROM batch_export_requests WHERE id > :id', array('id' => static::$maxId)); } } From 490bcc2ea94c8bb9799c17c4d097c409376a726c Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Sun, 2 Jun 2019 15:30:05 -0400 Subject: [PATCH 088/217] Fix file download REST --- .../WarehouseExportControllerProvider.php | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index db380d41ab..1a75d4f896 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -11,6 +11,7 @@ use Symfony\Component\HttpKernel\Exception\AccessDeniedHttpException; use Symfony\Component\HttpKernel\Exception\BadRequestHttpException; use Symfony\Component\HttpKernel\Exception\NotFoundHttpException; +use xd_utilities; class WarehouseExportControllerProvider extends BaseControllerProvider { @@ -175,7 +176,7 @@ public function getRequest(Request $request, Application $app, $id) $requests = array_filter( $handler->listUserRequestsByState($user->getUserId()), function ($request) use ($id) { - return $request['id'] === $id; + return $request['id'] == $id; } ); @@ -196,17 +197,32 @@ function ($request) use ($id) { } $file = sprintf( - '%s/%s.%s', + '%s/%s.%s.zip', $exportDir, $id, - strtolower($request['format']) + strtolower($request['export_file_format']) ); if (!is_readable($file)) { throw new AccessDeniedHttpException(); } - return $this->sendFile($file); + $fileName = sprintf( + '%s--%s-%s.%s.zip', + $request['realm'], + $request['start_date'], + $request['end_date'], + strtolower($request['export_file_format']) + ); + + return $app->sendFile( + $file, + 200, + [ + 'Content-type' => 'application/zip', + 'Content-Disposition' => sprintf('attachment; filename="%s"', $fileName) + ] + ); } /** From 18b1347f1afae82c0465709f5e370a09b0a37c4f Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 14:31:12 -0400 Subject: [PATCH 089/217] Fix and rearrange route setup Also silence possible JSON decode warnings. --- .../Rest/Controllers/WarehouseExportControllerProvider.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 1a75d4f896..1ee014b1f3 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -30,8 +30,9 @@ public function setupRoutes( $conversions = '\Rest\Utilities\Conversions'; $controller->get("$root/realms", "$current::getRealms"); - $controller->get("$root/requests", "$current::getRequests"); $controller->post("$root/request", "$current::createRequest"); + $controller->get("$root/requests", "$current::getRequests"); + $controller->delete("$root/requests", "$current::deleteRequests"); $controller->get("$root/request/{id}", "$current::getRequest") ->assert('id', '\d+') @@ -40,8 +41,6 @@ public function setupRoutes( $controller->delete("$root/request/{id}", "$current::deleteRequest") ->assert('id', '\d+') ->convert('id', "$conversions::toInt"); - - $controller->delete("$root/requests/{id}", "$current::deleteRequests"); } /** @@ -270,7 +269,7 @@ public function deleteRequests(Request $request, Application $app) $requestIds = []; try { - $requestIds = json_decode($request->getContent()); + $requestIds = @json_decode($request->getContent()); if ($requestIds === null) { throw new Exception('Failed to decode JSON'); From 28357c8433acb13f92ecf333fe02780aa1c9304e Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 31 May 2019 14:32:15 -0400 Subject: [PATCH 090/217] Implement delete buttons --- html/gui/js/modules/DataExport.js | 39 +++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 5a25c8f9e7..49c5d556c7 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -400,7 +400,29 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { 'Are you sure that you want to delete all expired requests? You cannot undo this operation.', function (selection) { if (selection === 'yes') { - Ext.Msg.alert('TODO', 'TODO: Delete all the expired requests'); + var requestIds = []; + + this.store.each(function (record) { + if (record.get('state') === 'Expired') { + requestIds.push(record.get('id')); + } + }); + + Ext.Ajax.request({ + method: 'DELETE', + url: 'rest/v1/warehouse/export/requests', + jsonData: requestIds, + scope: this, + success: function () { + this.store.reload(); + }, + failure: function (response) { + Ext.Msg.alert( + response.statusText || 'Deletion Failure', + JSON.parse(response.responseText).message || 'Unknown Error' + ); + } + }); } }, this @@ -413,7 +435,20 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { 'Are you sure that you want to delete this request? You cannot undo this operation.', function (selection) { if (selection === 'yes') { - Ext.Msg.alert('TODO', 'TODO: Delete the request'); + Ext.Ajax.request({ + method: 'DELETE', + url: 'rest/v1/warehouse/export/request/' + record.get('id'), + scope: this, + success: function () { + this.store.reload(); + }, + failure: function (response) { + Ext.Msg.alert( + response.statusText || 'Deletion Failure', + JSON.parse(response.responseText).message || 'Unknown Error' + ); + } + }); } }, this From 1ac43ab43723736003d6e573b7eb64c726a359b2 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 3 Jun 2019 07:00:32 -0400 Subject: [PATCH 091/217] Fix icons and change green color --- html/gui/js/modules/DataExport.js | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 49c5d556c7..6e1a2c925c 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -291,7 +291,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { renderer: function (value, metaData) { switch (value) { case 'Available': - metaData.attr = 'style="background-color:#040"'; // eslint-disable-line no-param-reassign + metaData.attr = 'style="background-color:#0f0"'; // eslint-disable-line no-param-reassign break; case 'Expired': case 'Failed': @@ -340,11 +340,22 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { xtype: 'actioncolumn', dataIndex: 'state', scope: this, + // XXX: The first argument to the getClass callback should + // always contain the value specified by the dataIndex, but + // the ActionColumn renderer alters it and so is only + // accurate in the renderer callback. Store this value in + // the meta object so it can be used for the icons. + // + // See https://docs.sencha.com/extjs/3.4.0/source/Column2.html#Ext-grid-ActionColumn-method-constructor + renderer: function (state, meta) { + meta._rowState = state; // eslint-disable-line no-param-reassign + }, items: [ { icon: 'gui/images/report_generator/delete_report.png', tooltip: 'Delete Request', iconCls: 'data-export-action-icon', + scope: this, handler: function (grid, rowIndex) { this.deleteRequest(grid.store.getAt(rowIndex)); } @@ -352,8 +363,9 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { { icon: 'gui/images/report_generator/download_report.png', tooltip: 'Download Exported Data', - getClass: function (state) { - return 'data-export-action-icon' + (state !== 'Available' ? '-hidden' : ''); + scope: this, + getClass: function (v, meta) { + return 'data-export-action-icon' + (meta._rowState !== 'Available' ? '-hidden' : ''); }, handler: function (grid, rowIndex) { this.downloadRequest(grid.store.getAt(rowIndex)); @@ -361,9 +373,10 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { }, { icon: 'gui/images/arrow_redo.png', - tooltip: 'Resumbit Request', - getClass: function (state) { - return 'data-export-action-icon' + (state !== 'Expired' && state !== 'Failed' ? '-hidden' : ''); + tooltip: 'Resubmit Request', + scope: this, + getClass: function (v, meta) { + return 'data-export-action-icon' + (meta._rowState !== 'Expired' && meta._rowState !== 'Failed' ? '-hidden' : ''); }, handler: function (grid, rowIndex) { this.resubmitRequest(grid.store.getAt(rowIndex)); @@ -456,7 +469,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { }, downloadRequest: function (record) { - // TODO + window.open('rest/v1/warehouse/export/request/' + record.get('id')); }, resubmitRequest: function (record) { From 49e421544fce0820107256586ef646af3e7d67d8 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 3 Jun 2019 07:29:16 -0400 Subject: [PATCH 092/217] Change variable and property names Also remove redundant scope configuration and return the empty string in the action column renderer. --- html/gui/js/modules/DataExport.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 6e1a2c925c..d57b969bc6 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -344,18 +344,19 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { // always contain the value specified by the dataIndex, but // the ActionColumn renderer alters it and so is only // accurate in the renderer callback. Store this value in - // the meta object so it can be used for the icons. + // the metaData object so it can be used by the icons. // // See https://docs.sencha.com/extjs/3.4.0/source/Column2.html#Ext-grid-ActionColumn-method-constructor - renderer: function (state, meta) { - meta._rowState = state; // eslint-disable-line no-param-reassign + renderer: function (state, metaData) { + console.log(JSON.stringify(metaData)); + metaData.rowState = state; // eslint-disable-line no-param-reassign + return ''; }, items: [ { icon: 'gui/images/report_generator/delete_report.png', tooltip: 'Delete Request', iconCls: 'data-export-action-icon', - scope: this, handler: function (grid, rowIndex) { this.deleteRequest(grid.store.getAt(rowIndex)); } @@ -363,9 +364,8 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { { icon: 'gui/images/report_generator/download_report.png', tooltip: 'Download Exported Data', - scope: this, - getClass: function (v, meta) { - return 'data-export-action-icon' + (meta._rowState !== 'Available' ? '-hidden' : ''); + getClass: function (v, metaData) { + return 'data-export-action-icon' + (metaData.rowState !== 'Available' ? '-hidden' : ''); }, handler: function (grid, rowIndex) { this.downloadRequest(grid.store.getAt(rowIndex)); @@ -374,9 +374,8 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { { icon: 'gui/images/arrow_redo.png', tooltip: 'Resubmit Request', - scope: this, - getClass: function (v, meta) { - return 'data-export-action-icon' + (meta._rowState !== 'Expired' && meta._rowState !== 'Failed' ? '-hidden' : ''); + getClass: function (v, metaData) { + return 'data-export-action-icon' + (metaData.rowState !== 'Expired' && metaData.rowState !== 'Failed' ? '-hidden' : ''); }, handler: function (grid, rowIndex) { this.resubmitRequest(grid.store.getAt(rowIndex)); From e3a1fa40f9b7e5165a1d89dfec0a01244255d0b6 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 3 Jun 2019 11:55:35 -0400 Subject: [PATCH 093/217] Fix bug --- .../Rest/Controllers/WarehouseExportControllerProvider.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 1ee014b1f3..846d4f662b 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -183,7 +183,9 @@ function ($request) use ($id) { throw new NotFoundHttpException(); } - $request = $requests[0]; + // Using `array_shift` because `array_filter` preserves keys so the + // request may not be at index 0. + $request = array_shift($requests); if ($request['state'] !== 'Available') { throw new BadRequestHttpException('Requested data is not available'); From 4cde1435486fd7792ae5323b860f595f5273e654 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 3 Jun 2019 11:56:08 -0400 Subject: [PATCH 094/217] Improve exceptions --- .../WarehouseExportControllerProvider.php | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 846d4f662b..71780364c8 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -180,7 +180,7 @@ function ($request) use ($id) { ); if (count($requests) === 0) { - throw new NotFoundHttpException(); + throw new NotFoundHttpException('Export request not found'); } // Using `array_shift` because `array_filter` preserves keys so the @@ -194,7 +194,7 @@ function ($request) use ($id) { try { $exportDir = xd_utilities\getConfiguration('data_warehouse_export', 'export_directory'); } catch (Exception $e) { - throw new BadRequestHttpException('Export directory is not configured'); + throw new NotFoundHttpException('Export directory is not configured'); } $file = sprintf( @@ -204,8 +204,12 @@ function ($request) use ($id) { strtolower($request['export_file_format']) ); + if (!is_file($file)) { + throw new NotFoundHttpException('Exported data not found'); + } + if (!is_readable($file)) { - throw new AccessDeniedHttpException(); + throw new AccessDeniedHttpException('Exported data is not readable'); } $fileName = sprintf( @@ -243,7 +247,7 @@ public function deleteRequest(Request $request, Application $app, $id) $count = $handler->deleteRequest($id, $user->getUserId()); if ($count === 0) { - throw new NotFoundHttpException(); + throw new NotFoundHttpException('Export request not found'); } return $app->json([ @@ -278,12 +282,12 @@ public function deleteRequests(Request $request, Application $app) } if (!is_array($requestIds)) { - throw new Exception('Request IDs must be in an array'); + throw new Exception('Export request IDs must be in an array'); } foreach ($requestIds as $id) { if (!is_int($id)) { - throw new Exception('Request IDs must integers'); + throw new Exception('Export request IDs must integers'); } } } catch (Exception $e) { @@ -299,7 +303,7 @@ public function deleteRequests(Request $request, Application $app) foreach ($requestIds as $id) { $count = $handler->deleteRequest($id, $user->getUserId()); if ($count === 0) { - throw new NotFoundHttpException(); + throw new NotFoundHttpException('Export request not found'); } } From a518de75691f5739311279c2a547f58d9cc15c90 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 3 Jun 2019 12:14:23 -0400 Subject: [PATCH 095/217] Enable/disable "Delete all ..." button --- html/gui/js/modules/DataExport.js | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index d57b969bc6..28e5cabdbc 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -387,7 +387,9 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { bbar: [ { xtype: 'button', + id: 'delete-all-expired-requests-button', text: 'Delete all expired requests', + disabled: true, scope: this, handler: this.deleteExpiredRequests }, @@ -404,6 +406,25 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { }); XDMoD.Module.DataExport.RequestsGrid.superclass.initComponent.call(this); + + // Enable or disable the "Delete all ..." after the store loads. + this.store.on('load', function () { + Ext.getCmp('delete-all-expired-requests-button').setDisabled( + this.getExpiredRequestIds().length === 0 + ); + }, this); + }, + + getExpiredRequestIds: function () { + var requestIds = []; + + this.store.each(function (record) { + if (record.get('state') === 'Expired') { + requestIds.push(record.get('id')); + } + }); + + return requestIds; }, deleteExpiredRequests: function () { @@ -412,18 +433,10 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { 'Are you sure that you want to delete all expired requests? You cannot undo this operation.', function (selection) { if (selection === 'yes') { - var requestIds = []; - - this.store.each(function (record) { - if (record.get('state') === 'Expired') { - requestIds.push(record.get('id')); - } - }); - Ext.Ajax.request({ method: 'DELETE', url: 'rest/v1/warehouse/export/requests', - jsonData: requestIds, + jsonData: this.getExpiredRequestIds(), scope: this, success: function () { this.store.reload(); From b3d0fb9f807ae4ba1ed5501ae79ddf4d7756828c Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 3 Jun 2019 12:29:27 -0400 Subject: [PATCH 096/217] Change field name style --- html/gui/js/modules/DataExport.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 28e5cabdbc..46a0136a22 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -307,7 +307,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { }, { header: 'Realm', - dataIndex: 'realmId', + dataIndex: 'realm_id', scope: this, renderer: function (value) { return this.realmsStore.getById(value).get('name'); @@ -519,7 +519,7 @@ XDMoD.Module.DataExport.RequestsStore = Ext.extend(Ext.data.JsonStore, { type: 'int' }, { - name: 'realmId', + name: 'realm_id', type: 'string', mapping: 'realm' }, From 66487cf2e00db68b100ebaf471b7ccd5188e0b40 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 3 Jun 2019 12:35:37 -0400 Subject: [PATCH 097/217] Implement request resubmission --- html/gui/js/modules/DataExport.js | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 46a0136a22..d92a2a9391 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -485,7 +485,26 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { }, resubmitRequest: function (record) { - // TODO + Ext.Ajax.request({ + method: 'POST', + url: 'rest/v1/warehouse/export/request', + params: { + realm: record.get('realm_id'), + start_date: record.get('start_date').format('Y-m-d'), + end_date: record.get('end_date').format('Y-m-d'), + format: record.get('export_file_format') + }, + scope: this, + success: function () { + this.store.reload(); + }, + failure: function (response) { + Ext.Msg.alert( + response.statusText || 'Resubmission Failure', + JSON.parse(response.responseText).message || 'Unknown Error' + ); + } + }); } }); From 39fa80cdd889891d711d70199dbfb84688e8e1bd Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 3 Jun 2019 12:53:57 -0400 Subject: [PATCH 098/217] Add more start/end date validation --- .../Controllers/WarehouseExportControllerProvider.php | 10 ++++++++++ html/gui/js/modules/DataExport.js | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 71780364c8..74b057240b 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -4,6 +4,7 @@ use CCR\DB; use DataWarehouse\Export\QueryHandler; +use DateTime; use Exception; use Silex\Application; use Silex\ControllerCollection; @@ -121,6 +122,15 @@ public function createRequest(Request $request, Application $app) $startDate = $this->getDateFromISO8601Param($request, 'start_date', true); $endDate = $this->getDateFromISO8601Param($request, 'end_date', true); + $now = new DateTime(); + + if ($startDate > $now) { + throw new BadRequestHttpException('Start date cannot be in the future'); + } + + if ($endDate > $now) { + throw new BadRequestHttpException('End date cannot be in the future'); + } $interval = $startDate->diff($endDate); diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index d92a2a9391..38b70c28ab 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -213,6 +213,10 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { return true; } + if (startDate > Date.now()) { + return 'Start date cannot be in the future'; + } + if (startDate > endDate) { return 'Start date must be before the end date'; } @@ -238,6 +242,10 @@ XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, { return true; } + if (endDate > Date.now()) { + return 'End date cannot be in the future'; + } + if (startDate > endDate) { return 'End date must be after the start date'; } From 2f8e495f806bd1092984f1749be63feba3c4cf97 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 3 Jun 2019 13:17:58 -0400 Subject: [PATCH 099/217] Add mask when no requests exist --- html/gui/js/modules/DataExport.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 38b70c28ab..dfaeda26b6 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -415,11 +415,18 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { XDMoD.Module.DataExport.RequestsGrid.superclass.initComponent.call(this); - // Enable or disable the "Delete all ..." after the store loads. + // Update elements that should be enabled/disabled or masked/unmasked + // after the store loads. this.store.on('load', function () { Ext.getCmp('delete-all-expired-requests-button').setDisabled( this.getExpiredRequestIds().length === 0 ); + + if (this.store.getCount() === 0) { + this.el.mask('No Current Requests'); + } else { + this.el.unmask(); + } }, this); }, From 3a3d32746e3a8c83b72c9017c42c07ed714bfe17 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 3 Jun 2019 14:38:52 -0400 Subject: [PATCH 100/217] Fix data index --- html/gui/js/modules/DataExport.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index dfaeda26b6..dc89e4f8a6 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -339,7 +339,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { }, { header: 'Expiration Date', - dataIndex: 'expires_date', + dataIndex: 'export_expires_datetime', xtype: 'datecolumn', format: 'Y-m-d' }, From 0ab3c681e4ed1459209bd2cbbf3a2fd3ad44841f Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 12 Jun 2019 07:22:13 -0400 Subject: [PATCH 101/217] Fix style issues --- classes/DataWarehouse/Export/FileHandler.php | 0 classes/DataWarehouse/Export/MailHandler.php | 0 classes/DataWarehouse/Export/QueryHandler.php | 6 ++---- .../Rest/Controllers/WarehouseExportControllerProvider.php | 2 +- html/gui/js/modules/DataExport.js | 1 - tests/component/lib/Export/ExportDBTest.php | 6 +++--- 6 files changed, 6 insertions(+), 9 deletions(-) delete mode 100644 classes/DataWarehouse/Export/FileHandler.php delete mode 100644 classes/DataWarehouse/Export/MailHandler.php diff --git a/classes/DataWarehouse/Export/FileHandler.php b/classes/DataWarehouse/Export/FileHandler.php deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/classes/DataWarehouse/Export/MailHandler.php b/classes/DataWarehouse/Export/MailHandler.php deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index 2df901ac5b..1b5539d040 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -46,8 +46,7 @@ class QueryHandler // Definition of Submitted state: private $whereSubmitted = "WHERE export_succeeded is NULL and export_created_datetime is NULL and export_expired = FALSE "; - - function __construct() + public function __construct() { // Fetch the database handle $this->pdo = DB::factory('database'); @@ -96,7 +95,7 @@ public function submittedToFailed($id) public function submittedToAvailable($id) { // read export retention duration from config file. Value is stored in days. - $expires_in_days = \xd_utilities\getConfiguration('data_warehouse_export','retention_duration_days'); + $expires_in_days = \xd_utilities\getConfiguration('data_warehouse_export', 'retention_duration_days'); $sql = "UPDATE batch_export_requests SET export_created_datetime=CAST(NOW() as DATETIME), @@ -219,4 +218,3 @@ public function deleteRequest($id, $user) return($result); } } - diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 74b057240b..4ebc37b38e 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -54,7 +54,7 @@ public function setupRoutes( */ public function getRealms(Request $request, Application $app) { - $user = $this->authorize($request); + /* $user = */$this->authorize($request); // TODO: Determine realms using user ACLs. $realms = [ diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index dc89e4f8a6..d93a250c2a 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -356,7 +356,6 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { // // See https://docs.sencha.com/extjs/3.4.0/source/Column2.html#Ext-grid-ActionColumn-method-constructor renderer: function (state, metaData) { - console.log(JSON.stringify(metaData)); metaData.rowState = state; // eslint-disable-line no-param-reassign return ''; }, diff --git a/tests/component/lib/Export/ExportDBTest.php b/tests/component/lib/Export/ExportDBTest.php index 8790ab80d1..fb4f51358d 100644 --- a/tests/component/lib/Export/ExportDBTest.php +++ b/tests/component/lib/Export/ExportDBTest.php @@ -99,15 +99,15 @@ public function testNewRecordCreation() $initialCount = $query->countSubmittedRecords(); // Add new record and verify - $requestId = $query->createRequestRecord($userId, 'Jobs', '2019-01-01', '2019-03-01','CSV'); + $requestId = $query->createRequestRecord($userId, 'Jobs', '2019-01-01', '2019-03-01', 'CSV'); $this->assertNotNull($requestId); // Add another new record and verify - $requestId2 = $query->createRequestRecord($userId, 'Accounts', '2016-12-01', '2017-01-01','JSON'); + $requestId2 = $query->createRequestRecord($userId, 'Accounts', '2016-12-01', '2017-01-01', 'JSON'); $this->assertNotNull($requestId2 ); // Add another new record and verify - $requestId3 = $query->createRequestRecord($userId, 'Jobs', '2014-01-05', '2014-01-26','CSV'); + $requestId3 = $query->createRequestRecord($userId, 'Jobs', '2014-01-05', '2014-01-26', 'CSV'); $this->assertNotNull($requestId3 ); // Determine final count From 6f6f1b254565d13b52dd7ae2b4b59fa6ab861ce1 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 14 Jun 2019 08:19:45 -0400 Subject: [PATCH 102/217] First attempt at integrating ACLs --- .../WarehouseExportControllerProvider.php | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 4ebc37b38e..d87fc44794 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -6,6 +6,7 @@ use DataWarehouse\Export\QueryHandler; use DateTime; use Exception; +use Models\Services\Realms; use Silex\Application; use Silex\ControllerCollection; use Symfony\Component\HttpFoundation\Request; @@ -54,18 +55,26 @@ public function setupRoutes( */ public function getRealms(Request $request, Application $app) { - /* $user = */$this->authorize($request); + $user = $this->authorize($request); + $userRealms = Realms::getRealmsForUser($user); // XXX Returns data from moddb.realms.display column. - // TODO: Determine realms using user ACLs. + // TODO: Get list of exportable realms. $realms = [ ['id' => 'jobs', 'name' => 'Jobs'], ['id' => 'supremm', 'name' => 'SUPReMM'], ['id' => 'accounts', 'name' => 'Accounts'], ['id' => 'allocations', 'name' => 'Allocations'], ['id' => 'requests', 'name' => 'Requests'], - ['id' => 'resource_allocations', 'name' => 'Resource Allocations'] + ['id' => 'resourceallocations', 'name' => 'ResourceAllocations'] ]; + $realms = array_filter( + $realms, + function ($realm) use ($userRealms) { + return in_array($realm['name'], $userRealms); + } + ); + return $app->json( [ 'success' => true, @@ -103,22 +112,14 @@ public function getRequests(Request $request, Application $app) public function createRequest(Request $request, Application $app) { $user = $this->authorize($request); + $userRealms = Realms::getRealmsForUser($user); // XXX Returns data from moddb.realms.display column. + $realm = $this->getStringParam($request, 'realm', true); - // TODO: Validate realm from user ACLs. - if (!in_array( - $realm, - [ - 'jobs', - 'supremm', - 'accounts', - 'allocations', - 'requests', - 'resource_allocations' - ] - )) { - throw new BadRequestHttpException('Invalid realm'); - } + // TODO: Check that realm is in list of exportable realms. + //if (!in_array($realm, $userRealms)) { + // throw new BadRequestHttpException('Invalid realm'); + //} $startDate = $this->getDateFromISO8601Param($request, 'start_date', true); $endDate = $this->getDateFromISO8601Param($request, 'end_date', true); From 5f11e2ecd0470f84fc5f0979266c857ce5cd2f67 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 14 Jun 2019 08:52:35 -0400 Subject: [PATCH 103/217] Add more data to REST responses --- .../WarehouseExportControllerProvider.php | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index d87fc44794..05ace94725 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -97,7 +97,13 @@ public function getRequests(Request $request, Application $app) $user = $this->authorize($request); $handler = new QueryHandler(); $results = $handler->listUserRequestsByState($user->getUserId()); - return $app->json(['success' => true, 'data' => $results]); + return $app->json( + [ + 'success' => true, + 'data' => $results, + 'total' => count($results) + ] + ); } /** @@ -162,7 +168,8 @@ public function createRequest(Request $request, Application $app) return $app->json([ 'success' => true, 'message' => 'Created export request', - 'data' => ['id' => $id] + 'data' => [['id' => $id]], + 'total' => 1 ]); } @@ -263,7 +270,9 @@ public function deleteRequest(Request $request, Application $app, $id) return $app->json([ 'success' => true, - 'message' => 'Deleted export request' + 'message' => 'Deleted export request', + 'data' => [['id' => $id]], + 'total' => 1 ]); } @@ -329,7 +338,14 @@ public function deleteRequests(Request $request, Application $app) return $app->json([ 'success' => true, - 'message' => 'Deleted export requests' + 'message' => 'Deleted export requests', + 'data' => array_map( + function ($id) { + return ['id' => $id]; + }, + $requestIds + ), + 'total' => count($requestIds) ]); } } From 82bbe6a9e82a7faea45e46be14fcaa2570684711 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 14 Jun 2019 14:44:12 -0400 Subject: [PATCH 104/217] Add REST tests --- .../input/create-request.json | 2 + .../input/delete-request.json | 2 + .../input/delete-requests.json | 2 + .../warehouse-export/input/get-realms.json | 38 +++ .../warehouse-export/input/get-request.json | 1 + .../warehouse-export/input/get-requests.json | 1 + .../DELETE-request.schema.json | 26 ++ .../DELETE-requests.schema.json | 26 ++ .../warehouse-export/GET-realms.schema.json | 42 +++ .../warehouse-export/GET-requests.schema.json | 117 ++++++++ .../warehouse-export/POST-request.schema.json | 26 ++ .../schema/warehouse-export/error.schema.json | 18 ++ .../Rest/WarehouseExportControllerTest.php | 274 ++++++++++++++++++ .../integration/lib/TestHarness/TestFiles.php | 14 + .../lib/TestHarness/XdmodTestHelper.php | 11 +- 15 files changed, 598 insertions(+), 2 deletions(-) create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse-export/input/create-request.json create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-request.json create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-requests.json create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-realms.json create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-request.json create mode 100644 tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-requests.json create mode 100644 tests/artifacts/xdmod/schema/warehouse-export/DELETE-request.schema.json create mode 100644 tests/artifacts/xdmod/schema/warehouse-export/DELETE-requests.schema.json create mode 100644 tests/artifacts/xdmod/schema/warehouse-export/GET-realms.schema.json create mode 100644 tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json create mode 100644 tests/artifacts/xdmod/schema/warehouse-export/POST-request.schema.json create mode 100644 tests/artifacts/xdmod/schema/warehouse-export/error.schema.json create mode 100644 tests/integration/lib/Rest/WarehouseExportControllerTest.php diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/create-request.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/create-request.json new file mode 100644 index 0000000000..0d4f101c7a --- /dev/null +++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/create-request.json @@ -0,0 +1,2 @@ +[ +] diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-request.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-request.json new file mode 100644 index 0000000000..0d4f101c7a --- /dev/null +++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-request.json @@ -0,0 +1,2 @@ +[ +] diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-requests.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-requests.json new file mode 100644 index 0000000000..0d4f101c7a --- /dev/null +++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-requests.json @@ -0,0 +1,2 @@ +[ +] diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-realms.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-realms.json new file mode 100644 index 0000000000..b0d6dc9863 --- /dev/null +++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-realms.json @@ -0,0 +1,38 @@ +{ + "Normal user": [ + "usr", + 200, + "GET-realms", + ["jobs"] + ], + "PI": [ + "pi", + 200, + "GET-realms", + ["jobs"] + ], + "Center staff": [ + "cs", + 200, + "GET-realms", + ["jobs"] + ], + "Center director": [ + "cd", + 200, + "GET-realms", + ["jobs"] + ], + "Administrative user": [ + "mgr", + 200, + "GET-realms", + ["jobs"] + ], + "Public user": [ + "pub", + 401, + "error", + [] + ] +} diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-request.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-request.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-request.json @@ -0,0 +1 @@ +[] diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-requests.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-requests.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-requests.json @@ -0,0 +1 @@ +[] diff --git a/tests/artifacts/xdmod/schema/warehouse-export/DELETE-request.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/DELETE-request.schema.json new file mode 100644 index 0000000000..3ff5b76e73 --- /dev/null +++ b/tests/artifacts/xdmod/schema/warehouse-export/DELETE-request.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Response from DELETE rest/warehouse/export/request/{id}", + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "message": { + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object" + } + }, + "total": { + "type": "integer" + } + }, + "required": [ + "success", + "message" + ] +} diff --git a/tests/artifacts/xdmod/schema/warehouse-export/DELETE-requests.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/DELETE-requests.schema.json new file mode 100644 index 0000000000..f467bd4985 --- /dev/null +++ b/tests/artifacts/xdmod/schema/warehouse-export/DELETE-requests.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Response from DELETE rest/warehouse/export/requests", + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "message": { + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object" + } + }, + "total": { + "type": "integer" + } + }, + "required": [ + "success", + "message" + ] +} diff --git a/tests/artifacts/xdmod/schema/warehouse-export/GET-realms.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/GET-realms.schema.json new file mode 100644 index 0000000000..58d7caac95 --- /dev/null +++ b/tests/artifacts/xdmod/schema/warehouse-export/GET-realms.schema.json @@ -0,0 +1,42 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Response from GET rest/warehouse/export/realms", + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/realm" + } + }, + "total": { + "type": "integer" + } + }, + "required": [ + "success", + "data" + ], + "additionalProperties": false, + "definitions": { + "realm": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + "required": [ + "id", + "name" + ], + "additionalProperties": false + } + } +} diff --git a/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json new file mode 100644 index 0000000000..18efdbd042 --- /dev/null +++ b/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json @@ -0,0 +1,117 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Response from GET rest/warehouse/export/requests", + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/request" + } + }, + "total": { + "type": "integer" + } + }, + "required": [ + "success", + "data" + ], + "additionalProperties": false, + "definitions": { + "request": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "realm": { + "type": "string" + }, + "start_date": { + "type": "string", + "format": "date" + }, + "end_date": { + "type": "string", + "format": "date" + }, + "export_succeeded": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "integer", + "minimum": 0, + "maximum": 1 + } + ] + }, + "export_expired": { + "type": "integer", + "minimum": 0, + "maximum": 1 + }, + "export_expires_datetime": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string", + "pattern": "^\\d{4}-\\d{2}-\\d{2}$" + } + ] + }, + "export_created_datetime": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "string", + "pattern": "^\\d{4}-\\d{2}-\\d{2}$" + } + ] + }, + "export_file_format": { + "enum": [ + "CSV", + "JSON" + ] + }, + "requested_datetime": { + "type": "string", + "pattern": "^\\d{4}-\\d{2}-\\d{2}$" + }, + "state": { + "type": "string", + "enum": [ + "Submitted", + "Available", + "Failed", + "Expired" + ] + } + }, + "required": [ + "id", + "realm", + "start_date", + "end_date", + "export_succeeded", + "export_expired", + "export_expires_datetime", + "export_created_datetime", + "export_file_format", + "requested_datetime", + "state" + ], + "additionalProperties": false + } + } +} diff --git a/tests/artifacts/xdmod/schema/warehouse-export/POST-request.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/POST-request.schema.json new file mode 100644 index 0000000000..cf034bd6af --- /dev/null +++ b/tests/artifacts/xdmod/schema/warehouse-export/POST-request.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Response from POST rest/warehouse/export/request", + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "message": { + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object" + } + }, + "total": { + "type": "integer" + } + }, + "required": [ + "success", + "message" + ] +} diff --git a/tests/artifacts/xdmod/schema/warehouse-export/error.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/error.schema.json new file mode 100644 index 0000000000..8f23aba744 --- /dev/null +++ b/tests/artifacts/xdmod/schema/warehouse-export/error.schema.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Generic error response", + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "message": { + "type": "string" + } + }, + "required": [ + "success", + "message" + ], + "additionalProperties": true +} diff --git a/tests/integration/lib/Rest/WarehouseExportControllerTest.php b/tests/integration/lib/Rest/WarehouseExportControllerTest.php new file mode 100644 index 0000000000..687b1f2cf5 --- /dev/null +++ b/tests/integration/lib/Rest/WarehouseExportControllerTest.php @@ -0,0 +1,274 @@ + null, + 'usr' => 'normaluser', + 'pi' => 'principal', + 'cs' => 'centerstaff', + 'cd' => 'centerdirector', + 'mgr' => 'admin' + ]; + + /** + * User for each role. + * @var XDUser[] + */ + private static $users = []; + + /** + * Instances of XdmodTestHelper for each user role. + * @var XdmodTestHelper[] + */ + private static $helpers = []; + + /** + * Database handle. + * @var iDatabase + */ + private static $dbh; + + /** + * Database handle. + * @var QueryHandler + */ + private static $queryHandler; + + /** + * JSON schema validator. + * @var Validator + */ + private static $schemaValidator; + + /** + * JSON schema objects. + * @var stdClass[] + */ + private static $schemaCache = []; + + /** + * @var TestFiles + */ + private static $testFiles; + + /** + * @return TestFiles + */ + private static function getTestFiles() + { + if (!isset(self::$testFiles)) { + self::$testFiles = new TestFiles(__DIR__ . '/../../../'); + } + + return self::$testFiles; + } + + /** + * Instantiate fixtures and authenticate helpers. + */ + public static function setUpBeforeClass() + { + foreach (self::$userRoles as $role => $username) { + self::$helpers[$role] = new XdmodTestHelper(); + + if ($role !== 'pub') { + self::$users[$role] = XDUser::getUserByUserName($username); + self::$helpers[$role]->authenticate($role); + } + } + + self::$dbh = DB::factory('database'); + self::$schemaValidator = new Validator(); + self::$queryHandler = new QueryHandler(); + } + + /** + * Logout and unset fixtures. + */ + public static function tearDownAfterClass() + { + foreach (self::$helpers as $helper) { + $helper->logout(); + } + + self::$users = null; + self::$helpers = null; + self::$dbh = null; + self::$schemaValidator = null; + self::$queryHandler = null; + self::$testFiles = null; + } + + /** + * Load a JSON schema file. + * + * @return stdClass + */ + private static function getSchema($schema) + { + if (!array_key_exists($schema, self::$schemaCache)) { + static::$schemaCache[$schema] = self::getTestFiles()->loadJsonFile( + 'schema', + $schema . '.schema', + 'warehouse-export', + false + ); + } + + return self::$schemaCache[$schema]; + } + + /** + * Validate content against a JSON schema. + * + * Test the results of the validation with an assertion. + * + * @param mixed $content The content to validate. + * @param string $schema The name of the schema file (without ".schema.json"). + * @param string $message The message to use in the assertion. + */ + public function validateAgainstSchema( + $content, + $schema, + $message = 'Validated against JSON schema' + ) { + // The content may have been decoded as an associative array so it needs + // to be encoded and decoded again as a stdClass before it is validated. + $normalizedContent = json_decode(json_encode($content)); + + self::$schemaValidator->check( + $normalizedContent, + self::getSchema($schema) + ); + $this->assertTrue(self::$schemaValidator->isValid(), $message); + + foreach ( self::$schemaValidator->getErrors() as $error) { + $this->assertTrue( + false, + sprintf("[%s] %s\n", $error['property'], $error['message']) + ); + } + } + + /** + * Test getting the list of exportable realms. + * + * @dataProvider getRealmsProvider + */ + public function testGetRealms($role, $httpCode, $schema, $realms) + { + list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/realms'); + $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code'); + $this->validateAgainstSchema($content, $schema); + } + + /** + * Test creating a new export request. + * + * @dataProvider createRequestProvider + */ + public function testCreateRequest($role, $params, $httpCode, $schema) + { + list($content, $info, $headers) = self::$helpers[$role]->post('rest/warehouse/export/request', $params); + $this->markTestIncomplete('This test has not been implemented yet.'); + } + + /** + * Test getting the list of export requests. + * + * @depends testCreateRequest + * @dataProvider getRequestsProvider + */ + public function testGetRequests() + { + list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/requests'); + $this->markTestIncomplete('This test has not been implemented yet.'); + } + + /** + * Test getting the exported data. + * + * @depends testCreateRequest + * #dataProvider getRequestProvider + */ + public function testGetRequest() + { + list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/request/' . $id); + $this->markTestIncomplete('This test has not been implemented yet.'); + } + + /** + * Test deleting an export request. + * + * @depends testCreateRequest + * #dataProvider deleteRequestProvider + */ + public function testDeleteRequest() + { + list($content, $info, $headers) = self::$helpers[$role]->delete('rest/warehouse/export/request/' . $id); + $this->markTestIncomplete('This test has not been implemented yet.'); + } + + /** + * Test deleting multiple export requests at a time. + */ + public function testDeleteRequests() + { + $data = json_encode([]); + //list($content, $info, $headers) = self::$helpers[$role]->delete('rest/warehouse/export/requests', $data); + $this->markTestIncomplete('This test has not been implemented yet.'); + } + + public function getRealmsProvider() + { + return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'get-realms', 'input'); + } + + public function createRequestProvider() + { + return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'create-request', 'input'); + } + + public function getRequestsProvider() + { + return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'get-requests', 'input'); + } + + public function getRequestProvider() + { + return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'get-request', 'input'); + } + + public function deleteRequestProvider() + { + return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'delete-request', 'input'); + } + + public function deleteRequestsProvider() + { + return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'delete-requests', 'input'); + } +} diff --git a/tests/integration/lib/TestHarness/TestFiles.php b/tests/integration/lib/TestHarness/TestFiles.php index 570b7f585a..9f5899c542 100644 --- a/tests/integration/lib/TestHarness/TestFiles.php +++ b/tests/integration/lib/TestHarness/TestFiles.php @@ -2,6 +2,8 @@ namespace TestHarness; +use CCR\Json; + class TestFiles { const TEST_ARTIFACT_OUTPUT_PATH = './artifacts'; @@ -63,4 +65,16 @@ public function getFile($testGroup, $fileName, $type = 'output', $extension = '. ) )); } + + public function loadJsonFile( + $testGroup, + $fileName, + $type = '', + $assoc = true + ) { + return Json::loadfile( + $this->getFile($testGroup, $fileName, $type), + $assoc + ); + } } diff --git a/tests/integration/lib/TestHarness/XdmodTestHelper.php b/tests/integration/lib/TestHarness/XdmodTestHelper.php index abc74282d2..24be90a179 100644 --- a/tests/integration/lib/TestHarness/XdmodTestHelper.php +++ b/tests/integration/lib/TestHarness/XdmodTestHelper.php @@ -273,7 +273,7 @@ private function docurl() return array($content, $curlinfo, $this->responseHeaders); } - public function delete($path, $params = null) + public function delete($path, $params = null, $data = null) { $url = $this->siteurl . $path; if ($params !== null) { @@ -281,7 +281,14 @@ public function delete($path, $params = null) } curl_setopt($this->curl, CURLOPT_URL, $url); - curl_setopt($this->curl, CURLOPT_POST, false); + + if ($data === null) { + curl_setopt($this->curl, CURLOPT_POST, false); + } else { + curl_setopt($this->curl, CURLOPT_POST, true); + curl_setopt($this->curl, CURLOPT_POSTFIELDS, $data); + } + curl_setopt($this->curl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_setopt($this->curl, CURLOPT_HTTPHEADER, $this->getheaders()); From f8c9482d80439e7b279a167194f4042c9321fd18 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 21 Jun 2019 08:55:19 -0400 Subject: [PATCH 105/217] Change ordering --- classes/DataWarehouse/Export/QueryHandler.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index 1b5539d040..936ee43536 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -148,7 +148,7 @@ public function countSubmittedRecords() public function listSubmittedRecords() { $sql = "SELECT id, realm, start_date, end_date, export_file_format, requested_datetime - FROM batch_export_requests " . $this->whereSubmitted; + FROM batch_export_requests " . $this->whereSubmitted . ' ORDER BY requested_datetime, id'; // Return query results. $result = $this->pdo->query($sql); @@ -170,7 +170,7 @@ public function listRequestsForUser($user_id) requested_datetime FROM batch_export_requests WHERE user_id=:user_id - ORDER BY id"; + ORDER BY requested_datetime, id"; $params = array('user_id' => $user_id); @@ -192,7 +192,7 @@ public function listUserRequestsByState($user_id) $sql = $attributes . "'Submitted' as state " . $fromTable . $this->whereSubmitted . $userClause . "UNION " . $attributes . "'Available' as state " . $fromTable . $whereAvailable . $userClause . "UNION " . $attributes . "'Expired' as state " . $fromTable . $whereExpired . $userClause . "UNION " . - $attributes . "'Failed' as state " . $fromTable . $whereFailed . $userClause . "ORDER BY requested_datetime"; + $attributes . "'Failed' as state " . $fromTable . $whereFailed . $userClause . "ORDER BY requested_datetime, id"; $params = array('user_id' => $user_id); From 1f083c849e16e147de6269403ac1d8e09ee917b0 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 21 Jun 2019 08:55:34 -0400 Subject: [PATCH 106/217] Implement more tests --- .../WarehouseExportControllerProvider.php | 12 ++ .../input/create-request.json | 176 +++++++++++++++- .../input/delete-request.json | 37 +++- .../input/delete-requests.json | 19 +- .../warehouse-export/input/get-realms.json | 35 +++- .../warehouse-export/input/get-requests.json | 80 +++++++- .../warehouse-export/GET-requests.schema.json | 6 +- .../Rest/WarehouseExportControllerTest.php | 194 +++++++++++++++--- .../lib/TestHarness/XdmodTestHelper.php | 45 ++-- 9 files changed, 548 insertions(+), 56 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 05ace94725..1bfe1e6e37 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -122,6 +122,18 @@ public function createRequest(Request $request, Application $app) $realm = $this->getStringParam($request, 'realm', true); + // TODO: Get list of exportable realms. + $realms = [ + 'jobs', + 'supremm', + 'accounts', + 'allocations', + 'requests', + 'resourceallocations' + ]; + if (!in_array($realm, $realms)) { + throw new BadRequestHttpException('Invalid realm'); + } // TODO: Check that realm is in list of exportable realms. //if (!in_array($realm, $userRealms)) { // throw new BadRequestHttpException('Invalid realm'); diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/create-request.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/create-request.json index 0d4f101c7a..6564af9e7e 100644 --- a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/create-request.json +++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/create-request.json @@ -1,2 +1,174 @@ -[ -] +{ + "Normal user": [ + "usr", + { + "realm": "jobs", + "start_date": "2018-01-01", + "end_date": "2018-12-31", + "format": "CSV" + }, + 200, + "POST-request" + ], + "Normal user (JSON format)": [ + "usr", + { + "realm": "jobs", + "start_date": "2017-02-01", + "end_date": "2017-02-28", + "format": "JSON" + }, + 200, + "POST-request" + ], + "PI": [ + "pi", + { + "realm": "jobs", + "start_date": "2017-01-01", + "end_date": "2017-12-31", + "format": "CSV" + }, + 200, + "POST-request" + ], + "Center director": [ + "cd", + { + "realm": "jobs", + "start_date": "2016-01-01", + "end_date": "2016-12-31", + "format": "CSV" + }, + 200, + "POST-request" + ], + "Missing realm": [ + "usr", + { + "start_date": "2018-01-01", + "end_date": "2018-12-31", + "format": "CSV" + }, + 400, + "error" + ], + "Missing start date": [ + "usr", + { + "realm": "jobs", + "end_date": "2018-12-31", + "format": "CSV" + }, + 400, + "error" + ], + "Missing end date": [ + "usr", + { + "realm": "jobs", + "start_date": "2018-01-01", + "format": "CSV" + }, + 400, + "error" + ], + "Missing format": [ + "usr", + { + "realm": "jobs", + "start_date": "2018-01-01", + "end_date": "2018-12-31" + }, + 400, + "error" + ], + "Invalid realm": [ + "usr", + { + "realm": "foo", + "start_date": "2018-01-01", + "end_date": "2018-12-31", + "format": "CSV" + }, + 400, + "error" + ], + "Invalid start date": [ + "usr", + { + "realm": "jobs", + "start_date": "1 January 2018", + "end_date": "2018-12-31", + "format": "CSV" + }, + 400, + "error" + ], + "Invalid end date": [ + "usr", + { + "realm": "jobs", + "start_date": "2018-01-01", + "end_date": "31 December 2018", + "format": "CSV" + }, + 400, + "error" + ], + "Invalid format": [ + "usr", + { + "realm": "jobs", + "start_date": "2018-01-01", + "end_date": "2018-12-31", + "format": "bar" + }, + 400, + "error" + ], + "Future start date": [ + "usr", + { + "realm": "jobs", + "start_date": "9999-12-31", + "end_date": "2018-12-31", + "format": "CSV" + }, + 400, + "error" + ], + "Future end date": [ + "usr", + { + "realm": "jobs", + "start_date": "2018-01-01", + "end_date": "9999-12-31", + "format": "CSV" + }, + 400, + "error" + ], + "Start date before end date": [ + "usr", + { + "realm": "jobs", + "start_date": "2018-12-31", + "end_date": "2018-01-01", + "format": "CSV" + }, + 400, + "error" + ], + "Public user": [ + "pub", + { + "realm": "jobs", + "start_date": "2018-01-01", + "end_date": "2018-12-31", + "format": "CSV" + }, + 401, + "error" + ] +} diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-request.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-request.json index 0d4f101c7a..cc68732f07 100644 --- a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-request.json +++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-request.json @@ -1,2 +1,35 @@ -[ -] +{ + "Normal user": [ + "usr", + { + "realm": "jobs", + "start_date": "2018-01-01", + "end_date": "2018-12-31", + "format": "CSV" + }, + 200, + "DELETE-request" + ], + "PI": [ + "pi", + { + "realm": "jobs", + "start_date": "2018-01-01", + "end_date": "2018-12-31", + "format": "CSV" + }, + 200, + "DELETE-request" + ], + "Center director": [ + "cd", + { + "realm": "jobs", + "start_date": "2018-01-01", + "end_date": "2018-12-31", + "format": "CSV" + }, + 200, + "DELETE-request" + ] +} diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-requests.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-requests.json index 0d4f101c7a..e83a6aef5b 100644 --- a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-requests.json +++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-requests.json @@ -1,2 +1,17 @@ -[ -] +{ + "Normal user": [ + "usr", + 200, + "DELETE-requests" + ], + "PI": [ + "pi", + 200, + "DELETE-requests" + ], + "Center director": [ + "cd", + 200, + "DELETE-requests" + ] +} diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-realms.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-realms.json index b0d6dc9863..e4c16f7d97 100644 --- a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-realms.json +++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-realms.json @@ -3,31 +3,56 @@ "usr", 200, "GET-realms", - ["jobs"] + [ + { + "id": "jobs", + "name": "Jobs" + } + ] ], "PI": [ "pi", 200, "GET-realms", - ["jobs"] + [ + { + "id": "jobs", + "name": "Jobs" + } + ] ], "Center staff": [ "cs", 200, "GET-realms", - ["jobs"] + [ + { + "id": "jobs", + "name": "Jobs" + } + ] ], "Center director": [ "cd", 200, "GET-realms", - ["jobs"] + [ + { + "id": "jobs", + "name": "Jobs" + } + ] ], "Administrative user": [ "mgr", 200, "GET-realms", - ["jobs"] + [ + { + "id": "jobs", + "name": "Jobs" + } + ] ], "Public user": [ "pub", diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-requests.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-requests.json index fe51488c70..cc4993d48b 100644 --- a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-requests.json +++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-requests.json @@ -1 +1,79 @@ -[] +{ + "Normal user": [ + "usr", + 200, + "GET-requests", + [ + { + "realm": "jobs", + "start_date": "2018-01-01", + "end_date": "2018-12-31", + "export_succeeded": null, + "export_expired": "0", + "export_expires_datetime": null, + "export_created_datetime": null, + "export_file_format": "CSV", + "state": "Submitted" + }, + { + "realm": "jobs", + "start_date": "2017-02-01", + "end_date": "2017-02-28", + "export_succeeded": null, + "export_expired": "0", + "export_expires_datetime": null, + "export_created_datetime": null, + "export_file_format": "JSON", + "state": "Submitted" + } + ] + ], + "PI": [ + "pi", + 200, + "GET-requests", + [ + { + "realm": "jobs", + "start_date": "2017-01-01", + "end_date": "2017-12-31", + "export_succeeded": null, + "export_expired": "0", + "export_expires_datetime": null, + "export_created_datetime": null, + "export_file_format": "CSV", + "state": "Submitted" + } + ] + ], + "Center staff": [ + "cs", + 200, + "GET-requests", + [] + ], + "Center director": [ + "cd", + 200, + "GET-requests", + [ + { + "realm": "jobs", + "start_date": "2016-01-01", + "end_date": "2016-12-31", + "export_succeeded": null, + "export_expired": "0", + "export_expires_datetime": null, + "export_created_datetime": null, + "export_file_format": "CSV", + "state": "Submitted" + } + ] + ], + "Public user": [ + "pub", + 401, + "error", + [] + ] +} diff --git a/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json index 18efdbd042..219dfaf2ac 100644 --- a/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json +++ b/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json @@ -63,7 +63,7 @@ }, { "type": "string", - "pattern": "^\\d{4}-\\d{2}-\\d{2}$" + "pattern": "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$" } ] }, @@ -74,7 +74,7 @@ }, { "type": "string", - "pattern": "^\\d{4}-\\d{2}-\\d{2}$" + "pattern": "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$" } ] }, @@ -86,7 +86,7 @@ }, "requested_datetime": { "type": "string", - "pattern": "^\\d{4}-\\d{2}-\\d{2}$" + "pattern": "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$" }, "state": { "type": "string", diff --git a/tests/integration/lib/Rest/WarehouseExportControllerTest.php b/tests/integration/lib/Rest/WarehouseExportControllerTest.php index 687b1f2cf5..a0870eb4da 100644 --- a/tests/integration/lib/Rest/WarehouseExportControllerTest.php +++ b/tests/integration/lib/Rest/WarehouseExportControllerTest.php @@ -3,15 +3,19 @@ namespace IntegrationTests\Rest; use CCR\DB; +use DataWarehouse\Export\QueryHandler; +use Exception; +use JsonSchema\Constraints\Constraint; use JsonSchema\Validator; use PHPUnit_Framework_TestCase; -use TestHarness\XdmodTestHelper; use TestHarness\TestFiles; -use DataWarehouse\Export\QueryHandler; +use TestHarness\XdmodTestHelper; use XDUser; /** * Test data warehouse export REST endpoints. + * + * @coversDefaultClass \Rest\Controllers\WarehouseExportControllerProvider */ class WarehouseExportControllerTest extends PHPUnit_Framework_TestCase { @@ -101,6 +105,12 @@ public static function setUpBeforeClass() } self::$dbh = DB::factory('database'); + + list($row) = self::$dbh->query('SELECT COUNT(*) AS count FROM batch_export_requests'); + if ($row['count'] > 0) { + error_log(sprintf('Expected 0 rows in moddb.batch_export_requests, found %d', $row['count'])); + } + self::$schemaValidator = new Validator(); self::$queryHandler = new QueryHandler(); } @@ -114,6 +124,9 @@ public static function tearDownAfterClass() $helper->logout(); } + // Delete any requests that weren't already deleted. + self::$dbh->execute('DELETE FROM batch_export_requests'); + self::$users = null; self::$helpers = null; self::$dbh = null; @@ -153,93 +166,218 @@ private static function getSchema($schema) public function validateAgainstSchema( $content, $schema, - $message = 'Validated against JSON schema' + $message = 'Validate against JSON schema' ) { // The content may have been decoded as an associative array so it needs // to be encoded and decoded again as a stdClass before it is validated. $normalizedContent = json_decode(json_encode($content)); - self::$schemaValidator->check( + // Data (numbers, etc.) are returned from MySQL as strings and likewise + // returned from the REST endpoint as string. Using + // CHECK_MODE_COERCE_TYPES to allow these values. + self::$schemaValidator->validate( $normalizedContent, - self::getSchema($schema) + self::getSchema($schema), + Constraint::CHECK_MODE_COERCE_TYPES ); - $this->assertTrue(self::$schemaValidator->isValid(), $message); - foreach ( self::$schemaValidator->getErrors() as $error) { - $this->assertTrue( - false, - sprintf("[%s] %s\n", $error['property'], $error['message']) - ); - } + $errors = self::$schemaValidator->getErrors(); + $this->assertCount( + 0, + $errors, + $message . "\n" . implode("\n", array_map( + function ($error) { + return sprintf("[%s] %s", $error['property'], $error['message']); + }, + $errors + )) + ); } /** * Test getting the list of exportable realms. * + * @param string $role Role to use during test. + * @param int $httpCode Expected HTTP response code. + * @param string $schema Name of JSON schema file that will be used + * to validate returned data. + * @param array $realms The name of the realms that are expected to + * be in the returned data. + * @covers ::getRealms * @dataProvider getRealmsProvider */ - public function testGetRealms($role, $httpCode, $schema, $realms) + public function testGetRealms($role, $httpCode, $schema, array $realms) { list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/realms'); $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code'); $this->validateAgainstSchema($content, $schema); + + // Only check data for successful requests. + if ($httpCode == 200) { + $this->assertEquals($realms, $content['data'], 'Data contains realms'); + } } /** * Test creating a new export request. * + * @param string $role Role to use during test. + * @param int $httpCode Expected HTTP response code. + * @param string $schema Name of JSON schema file that will be used + * to validate returned data. + * @covers ::createRequest * @dataProvider createRequestProvider */ - public function testCreateRequest($role, $params, $httpCode, $schema) + public function testCreateRequest($role, array $params, $httpCode, $schema) { - list($content, $info, $headers) = self::$helpers[$role]->post('rest/warehouse/export/request', $params); - $this->markTestIncomplete('This test has not been implemented yet.'); + list($content, $info, $headers) = self::$helpers[$role]->post('rest/warehouse/export/request', null, $params); + $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code'); + $this->validateAgainstSchema($content, $schema); } /** * Test getting the list of export requests. * + * @param string $role Role to use during test. + * @param int $httpCode Expected HTTP response code. + * @param string $schema Name of JSON schema file that will be used + * to validate returned data. + * @param array $requests Export requests expected to exist. + * @covers ::getRequests * @depends testCreateRequest * @dataProvider getRequestsProvider */ - public function testGetRequests() - { + public function testGetRequests( + $role, + $httpCode, + $schema, + array $requests + ) { list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/requests'); - $this->markTestIncomplete('This test has not been implemented yet.'); + $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code'); + $this->validateAgainstSchema($content, $schema); + + // Only check data for successful requests. + if ($httpCode == 200) { + $this->assertArraySubset($requests, $content['data'], 'Data contains requests'); + } } /** * Test getting the exported data. * + * @covers ::getRequest * @depends testCreateRequest * #dataProvider getRequestProvider */ + //public function testGetRequest($role, $params, $httpCode) public function testGetRequest() { - list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/request/' . $id); + //list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/request/' . $id); $this->markTestIncomplete('This test has not been implemented yet.'); } /** * Test deleting an export request. * - * @depends testCreateRequest - * #dataProvider deleteRequestProvider + * Creates an export and then deletes it. + * + * @param string $role Role to use during test. + * @param array $params Parameters to create an export request. + * @param int $httpCode Expected HTTP response code. + * @param string $schema Name of JSON schema file that will be used + * to validate returned data. + * @covers ::deleteRequest + * @uses ::createRequest + * @uses ::getRequests + * @dataProvider deleteRequestProvider */ - public function testDeleteRequest() + public function testDeleteRequest($role, array $params, $httpCode, $schema) { + // Get list of requests before deletion. + list($beforeContent) = self::$helpers[$role]->get('rest/warehouse/export/requests'); + $dataBefore = $beforeContent['data']; + + list($createContent) = self::$helpers[$role]->post('rest/warehouse/export/request', null, $params); + $id = $createContent['data'][0]['id']; + list($content, $info, $headers) = self::$helpers[$role]->delete('rest/warehouse/export/request/' . $id); - $this->markTestIncomplete('This test has not been implemented yet.'); + $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code'); + $this->validateAgainstSchema($content, $schema); + $this->assertEquals($id, $content['data'][0]['id'], 'Deleted ID is in response'); + + // Get list of requests after deletion + list($afterContent) = self::$helpers[$role]->get('rest/warehouse/export/requests'); + $dataAfter = $afterContent['data']; + + $this->assertEquals($dataBefore, $dataAfter, 'Data before and after creation/deletion are the same.'); + } + + /** + * Test deleting an export request in cases where it is expected to fail. + * + * @covers ::deleteRequest + */ + public function testDeleteRequestErrors() + { + // Public user can't delete anything. + list($content, $info, $headers) = self::$helpers['pub']->delete('rest/warehouse/export/request/1'); + $this->assertEquals(401, $info['http_code'], 'HTTP response code'); + $this->validateAgainstSchema($content, 'error'); + + // Non-integer ID. + list($content, $info, $headers) = self::$helpers['usr']->delete('rest/warehouse/export/request/abc'); + $this->assertEquals(404, $info['http_code'], 'HTTP response code'); + $this->validateAgainstSchema($content, 'error'); + + // Trying to delete a non-existent request. + list($row) = self::$dbh->query('SELECT MAX(id) + 1 AS id FROM batch_export_requests'); + list($content, $info, $headers) = self::$helpers['usr']->delete('rest/warehouse/export/request/' . $row['id']); + $this->assertEquals(404, $info['http_code'], 'HTTP response code'); + $this->validateAgainstSchema($content, 'error'); + + // Trying to delete another user's request. + list($row) = self::$dbh->query('SELECT id FROM batch_export_requests WHERE user_id = :user_id LIMIT 1', ['user_id' => self::$users['pi']->getUserId()]); + list($content, $info, $headers) = self::$helpers['usr']->delete('rest/warehouse/export/request/' . $row['id']); + $this->assertEquals(404, $info['http_code'], 'HTTP response code'); + $this->validateAgainstSchema($content, 'error'); } /** * Test deleting multiple export requests at a time. + * + * @param string $role Role to use during test. + * @param int $httpCode Expected HTTP response code. + * @param string $schema Name of JSON schema file that will be used + * to validate returned data. + * @covers ::deleteRequests + * @uses ::getRequests + * @dataProvider deleteRequestsProvider */ - public function testDeleteRequests() + public function testDeleteRequests($role, $httpCode, $schema) { - $data = json_encode([]); - //list($content, $info, $headers) = self::$helpers[$role]->delete('rest/warehouse/export/requests', $data); - $this->markTestIncomplete('This test has not been implemented yet.'); + // Get list of requests before deletion. + list($beforeContent) = self::$helpers[$role]->get('rest/warehouse/export/requests'); + + // Gather ID values and also convert to integers for the array + // comparison done below. + $ids = []; + foreach ($beforeContent['data'] as &$datum) { + $datum['id'] = (int)$datum['id']; + $ids[] = $datum['id']; + } + $data = json_encode($ids); + //$this->assertTrue(false, json_encode($beforeContent['data'])); + + // Delete all existing requests. + list($content, $info, $headers) = self::$helpers[$role]->delete('rest/warehouse/export/requests', null, $data); + $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code'); + $this->validateAgainstSchema($content, $schema); + $this->assertArraySubset($content['data'], $beforeContent['data'], 'Deleted IDs are in response'); + + // Get list of requests after deletion + list($afterContent) = self::$helpers[$role]->get('rest/warehouse/export/requests'); + $this->assertEquals([], $afterContent['data'], 'Data after deletion is empty.'); } public function getRealmsProvider() diff --git a/tests/integration/lib/TestHarness/XdmodTestHelper.php b/tests/integration/lib/TestHarness/XdmodTestHelper.php index 24be90a179..bd5802fd4e 100644 --- a/tests/integration/lib/TestHarness/XdmodTestHelper.php +++ b/tests/integration/lib/TestHarness/XdmodTestHelper.php @@ -22,33 +22,49 @@ public function __construct($config = array()) $this->headers = array(); $this->decodeTextAsJson = false; - $this->curl = curl_init(); + $this->cookiefile = tempnam(sys_get_temp_dir(), "xdmodtestcookies."); + + if (isset($config['decodetextasjson'])) { + $this->decodeTextAsJson = true; + } + if (isset($config['verbose'])) { + $this->verbose = true; + } + + $this->resetCurlSession(); + } + + /** + * Reset the cURL session. + * + * This function must be called after any use of CURLOPT_CUSTOMREQUEST to + * reset the request type. + */ + private function resetCurlSession() + { + // Close existing session to write cookies to file. + if (isset($this->curl)) { + curl_close($this->curl); + } + $this->curl = curl_init(); curl_setopt($this->curl, CURLOPT_USERAGENT, "XDMoD REST Test harness"); curl_setopt($this->curl, CURLOPT_RETURNTRANSFER, true); curl_setopt($this->curl, CURLOPT_FOLLOWLOCATION, true); # Enable header information in the response data - curl_setopt($this->curl, CURLOPT_RETURNTRANSFER, 1); - curl_setopt($this->curl, CURLOPT_HEADERFUNCTION, array(&$this, 'processResponseHeader')); + curl_setopt($this->curl, CURLOPT_HEADERFUNCTION, array($this, 'processResponseHeader')); # Disable ssl certificate checks (needed when using self-signed certificates). curl_setopt($this->curl, CURLOPT_SSL_VERIFYHOST, false); curl_setopt($this->curl, CURLOPT_SSL_VERIFYPEER, false); - $this->cookiefile = tempnam(sys_get_temp_dir(), "xdmodtestcookies."); curl_setopt($this->curl, CURLOPT_COOKIEFILE, $this->cookiefile); + curl_setopt($this->curl, CURLOPT_COOKIEJAR, $this->cookiefile); if (isset($this->cookie)) { curl_setopt($this->curl, CURLOPT_COOKIE, $this->cookie); } - - if (isset($config['decodetextasjson'])) { - $this->decodeTextAsJson = true; - } - if (isset($config['verbose'])) { - $this->verbose = true; - } } private function processResponseHeader($curl, $headerline) @@ -292,7 +308,10 @@ public function delete($path, $params = null, $data = null) curl_setopt($this->curl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_setopt($this->curl, CURLOPT_HTTPHEADER, $this->getheaders()); - return $this->docurl(); + $response = $this->docurl(); + $this->resetCurlSession(); + + return $response; } public function get($path, $params = null, $isurl = false) @@ -362,8 +381,8 @@ public function patch($path, $params = null, $data = null) curl_setopt($this->curl, CURLOPT_CUSTOMREQUEST, 'PATCH'); curl_setopt($this->curl, CURLOPT_HTTPHEADER, $this->getheaders()); $response = $this->docurl(); + $this->resetCurlSession(); - curl_setopt($this->curl, CURLOPT_CUSTOMREQUEST, null); return $response; } public function getSiteurl(){ From 70917879c04ac77cb7d720da9153827664371f1a Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 21 Jun 2019 09:21:55 -0400 Subject: [PATCH 107/217] More assertions --- .../lib/Rest/WarehouseExportControllerTest.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/integration/lib/Rest/WarehouseExportControllerTest.php b/tests/integration/lib/Rest/WarehouseExportControllerTest.php index a0870eb4da..529552149c 100644 --- a/tests/integration/lib/Rest/WarehouseExportControllerTest.php +++ b/tests/integration/lib/Rest/WarehouseExportControllerTest.php @@ -209,6 +209,7 @@ function ($error) { public function testGetRealms($role, $httpCode, $schema, array $realms) { list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/realms'); + $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header'); $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code'); $this->validateAgainstSchema($content, $schema); @@ -231,6 +232,7 @@ public function testGetRealms($role, $httpCode, $schema, array $realms) public function testCreateRequest($role, array $params, $httpCode, $schema) { list($content, $info, $headers) = self::$helpers[$role]->post('rest/warehouse/export/request', null, $params); + $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header'); $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code'); $this->validateAgainstSchema($content, $schema); } @@ -254,6 +256,7 @@ public function testGetRequests( array $requests ) { list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/requests'); + $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header'); $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code'); $this->validateAgainstSchema($content, $schema); @@ -274,6 +277,8 @@ public function testGetRequests( public function testGetRequest() { //list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/request/' . $id); + //$this->assertRegExp('#\applcation/zip\b#', $headers['Content-Type'], 'Content type header'); + //$this->assertEquals($httpCode, $info['http_code'], 'HTTP response code'); $this->markTestIncomplete('This test has not been implemented yet.'); } @@ -302,6 +307,7 @@ public function testDeleteRequest($role, array $params, $httpCode, $schema) $id = $createContent['data'][0]['id']; list($content, $info, $headers) = self::$helpers[$role]->delete('rest/warehouse/export/request/' . $id); + $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header'); $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code'); $this->validateAgainstSchema($content, $schema); $this->assertEquals($id, $content['data'][0]['id'], 'Deleted ID is in response'); @@ -322,23 +328,27 @@ public function testDeleteRequestErrors() { // Public user can't delete anything. list($content, $info, $headers) = self::$helpers['pub']->delete('rest/warehouse/export/request/1'); + $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header'); $this->assertEquals(401, $info['http_code'], 'HTTP response code'); $this->validateAgainstSchema($content, 'error'); // Non-integer ID. list($content, $info, $headers) = self::$helpers['usr']->delete('rest/warehouse/export/request/abc'); + $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header'); $this->assertEquals(404, $info['http_code'], 'HTTP response code'); $this->validateAgainstSchema($content, 'error'); // Trying to delete a non-existent request. list($row) = self::$dbh->query('SELECT MAX(id) + 1 AS id FROM batch_export_requests'); list($content, $info, $headers) = self::$helpers['usr']->delete('rest/warehouse/export/request/' . $row['id']); + $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header'); $this->assertEquals(404, $info['http_code'], 'HTTP response code'); $this->validateAgainstSchema($content, 'error'); // Trying to delete another user's request. list($row) = self::$dbh->query('SELECT id FROM batch_export_requests WHERE user_id = :user_id LIMIT 1', ['user_id' => self::$users['pi']->getUserId()]); list($content, $info, $headers) = self::$helpers['usr']->delete('rest/warehouse/export/request/' . $row['id']); + $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header'); $this->assertEquals(404, $info['http_code'], 'HTTP response code'); $this->validateAgainstSchema($content, 'error'); } @@ -371,6 +381,7 @@ public function testDeleteRequests($role, $httpCode, $schema) // Delete all existing requests. list($content, $info, $headers) = self::$helpers[$role]->delete('rest/warehouse/export/requests', null, $data); + $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header'); $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code'); $this->validateAgainstSchema($content, $schema); $this->assertArraySubset($content['data'], $beforeContent['data'], 'Deleted IDs are in response'); From 3cb9c8adef07caf02891f3e1c55608b95a40cfcd Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 24 Jun 2019 08:55:22 -0400 Subject: [PATCH 108/217] Rearrange and comment-out code --- .../WarehouseExportControllerProvider.php | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 1bfe1e6e37..8d3626d42a 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -118,11 +118,13 @@ public function getRequests(Request $request, Application $app) public function createRequest(Request $request, Application $app) { $user = $this->authorize($request); - $userRealms = Realms::getRealmsForUser($user); // XXX Returns data from moddb.realms.display column. - $realm = $this->getStringParam($request, 'realm', true); - // TODO: Get list of exportable realms. + // TODO: Check that realm is in list of exportable realms. + //$userRealms = Realms::getRealmsForUser($user); // XXX Returns data from moddb.realms.display column. + //if (!in_array($realm, $userRealms)) { + // throw new BadRequestHttpException('Invalid realm'); + //} $realms = [ 'jobs', 'supremm', @@ -134,10 +136,6 @@ public function createRequest(Request $request, Application $app) if (!in_array($realm, $realms)) { throw new BadRequestHttpException('Invalid realm'); } - // TODO: Check that realm is in list of exportable realms. - //if (!in_array($realm, $userRealms)) { - // throw new BadRequestHttpException('Invalid realm'); - //} $startDate = $this->getDateFromISO8601Param($request, 'start_date', true); $endDate = $this->getDateFromISO8601Param($request, 'end_date', true); From 3ac94e5680e4ce85d16fe19f51a2f1abbfeeb3ec Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 28 Jun 2019 12:42:49 -0400 Subject: [PATCH 109/217] Add function --- classes/DataWarehouse/Export/QueryHandler.php | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index 936ee43536..6ae55d8e00 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -46,6 +46,8 @@ class QueryHandler // Definition of Submitted state: private $whereSubmitted = "WHERE export_succeeded is NULL and export_created_datetime is NULL and export_expired = FALSE "; + private $whereExpired = "WHERE export_succeeded = TRUE and export_created_datetime is NOT NULL and export_expired = TRUE "; + public function __construct() { // Fetch the database handle @@ -155,6 +157,18 @@ public function listSubmittedRecords() return($result); } + /** + * Return export requests in Expired state. + * + * @return array + */ + public function listExpiredRecords() + { + $sql = 'SELECT id, realm, start_date, end_date, export_file_format, requested_datetime + FROM batch_export_requests ' . $this->whereExpired . ' ORDER BY requested_datetime, id'; + return $this->pdo->query($sql); + } + // Return details of export requests made by specified user. public function listRequestsForUser($user_id) { @@ -185,13 +199,12 @@ public function listUserRequestsByState($user_id) $attributes = "SELECT id, realm, start_date, end_date, export_succeeded, export_expired, export_expires_datetime, export_created_datetime, export_file_format, requested_datetime, "; $fromTable = "FROM batch_export_requests "; $whereAvailable = "WHERE export_succeeded = TRUE and export_created_datetime is NOT NULL and export_expired = FALSE "; - $whereExpired = "WHERE export_succeeded = TRUE and export_created_datetime is NOT NULL and export_expired = TRUE "; $whereFailed = "WHERE export_succeeded = FALSE and export_created_datetime is NULL and export_expired = FALSE "; $userClause = "AND user_id = :user_id "; $sql = $attributes . "'Submitted' as state " . $fromTable . $this->whereSubmitted . $userClause . "UNION " . $attributes . "'Available' as state " . $fromTable . $whereAvailable . $userClause . "UNION " . - $attributes . "'Expired' as state " . $fromTable . $whereExpired . $userClause . "UNION " . + $attributes . "'Expired' as state " . $fromTable . $this->whereExpired . $userClause . "UNION " . $attributes . "'Failed' as state " . $fromTable . $whereFailed . $userClause . "ORDER BY requested_datetime, id"; $params = array('user_id' => $user_id); From 3ce898dfc5316cb993d22aea6ab3ac9b9414db72 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 28 Jun 2019 13:17:21 -0400 Subject: [PATCH 110/217] Update comments and formatting --- classes/DataWarehouse/Export/QueryHandler.php | 263 ++++++++++-------- 1 file changed, 146 insertions(+), 117 deletions(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index 6ae55d8e00..2d36d4ade3 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -1,5 +1,5 @@ Available -> Expired @@ -29,8 +25,6 @@ * Failed * * ...any state can transition to Deleted. - * - * ========================================================================================== */ namespace DataWarehouse\Export; @@ -40,121 +34,149 @@ class QueryHandler { - // database handle, populated in the constructor - private $pdo; + /** + * Database handle. + * @var \CCR\DB\iDatabase + */ + private $dbh; - // Definition of Submitted state: - private $whereSubmitted = "WHERE export_succeeded is NULL and export_created_datetime is NULL and export_expired = FALSE "; + /** + * Definition of Submitted state. + * @var string + */ + private $whereSubmitted = "WHERE export_succeeded IS NULL AND export_created_datetime IS NULL AND export_expired = 0 "; - private $whereExpired = "WHERE export_succeeded = TRUE and export_created_datetime is NOT NULL and export_expired = TRUE "; + /** + * Definition of Expired state. + * @var string + */ + private $whereExpired = "WHERE export_succeeded = TRUE AND export_created_datetime IS NOT NULL AND export_expired = 1 "; + + /** + * Definition of Available state. + * @var string + */ + private $whereAvailable = "WHERE export_succeeded = 1 AND export_created_datetime IS NOT NULL AND export_expired = 0 "; + + /** + * Definition of Failed state. + * @var string + */ + private $whereFailed = "WHERE export_succeeded = 0 AND export_created_datetime IS NULL AND export_expired = 0 "; public function __construct() { - // Fetch the database handle - $this->pdo = DB::factory('database'); + $this->dbh = DB::factory('database'); } - /* ******** Transition between request record states ******** */ - - // Create request record for specified export request. - // Result is a single request in Submitted state. - public function createRequestRecord($userId, $realm, $startDate, $endDate, $format) - { + /** + * Create request record for specified export request. + * + * @param integer $userId + * @param string $realm Realm unique identifier. + * @param string $startDate Start date formatted as YYYY-MM-DD. + * @param string $endDate End date formatted as YYYY-MM-DD. + * @param string $format Export format (CSV or JSON). + * @return integer The id for the inserted record. + */ + public function createRequestRecord( + $userId, + $realm, + $startDate, + $endDate, + $format + ) { $sql = "INSERT INTO batch_export_requests (requested_datetime, user_id, realm, start_date, end_date, export_file_format) VALUES (NOW(), :user_id, :realm, :start_date, :end_date, :export_file_format)"; - $params = array('user_id' => $userId, - 'realm' => $realm, - 'start_date' => $startDate, - 'end_date' => $endDate, - 'export_file_format' => $format - ); + $params = array( + 'user_id' => $userId, + 'realm' => $realm, + 'start_date' => $startDate, + 'end_date' => $endDate, + 'export_file_format' => $format + ); - // return the id for the inserted record - $id = $this->pdo->insert($sql, $params); - return($id); + return $this->dbh->insert($sql, $params); } - // Transition specified export request from Submitted state to Failed state. + /** + * Transition specified export request from Submitted state to Failed state. + * + * @param integer $id Export request primary key. + * @return integer Count of affected rows--should be 1 if successful. + */ public function submittedToFailed($id) { $sql = "UPDATE batch_export_requests - SET export_succeeded=0 " . + SET export_succeeded = 0 " . $this->whereSubmitted . - "AND id=:id"; - - $params = array('id' => $id); - - // Return count of affected rows--should be 1 if successful. - $result = $this->pdo->execute($sql, $params); - return($result); + "AND id = :id"; + return $this->dbh->execute($sql, array('id' => $id)); } - // Transition specified export request from Submitted state to Available state. - // All time is current time as relative to database. + /** + * Transition specified export request from Submitted state to Available state. + * + * @param integer $id Export request primary key. + * @return integer Count of affected rows--should be 1 if successful. + */ public function submittedToAvailable($id) { // read export retention duration from config file. Value is stored in days. $expires_in_days = \xd_utilities\getConfiguration('data_warehouse_export', 'retention_duration_days'); $sql = "UPDATE batch_export_requests - SET export_created_datetime=CAST(NOW() as DATETIME), - export_expires_datetime=DATE_ADD(CAST(NOW() as DATETIME), INTERVAL :expires_in_days DAY), - export_succeeded=1 " . - $this->whereSubmitted . "AND id=:id"; + SET export_created_datetime = NOW(), + export_expires_datetime = DATE_ADD(NOW(), INTERVAL :expires_in_days DAY), + export_succeeded = 1 " . + $this->whereSubmitted . "AND id = :id"; - $params = array('expires_in_days' => $expires_in_days, - 'id' => $id); + $params = array( + 'expires_in_days' => $expires_in_days, + 'id' => $id + ); - // Return count of affected rows--should be 1 if successful. - $result = $this->pdo->execute($sql, $params); - return($result); + return $this->dbh->execute($sql, $params); } - // Transition specified export request from Available state to Expired state. + /** + * Transition specified export request from Available state to Expired state. + * + * @param integer $userId + * @return integer Count of affected rows--should be 1 if successful. + */ public function availableToExpired($id) { - $sql = "UPDATE batch_export_requests - SET export_expired=1 - WHERE id=:id AND - export_succeeded = TRUE AND export_created_datetime IS NOT NULL - AND export_expired = FALSE"; - - $params = array('id' => $id); - - // Return count of affected rows--should be 1 if successful. - $result = $this->pdo->execute($sql, $params); - return($result); + $sql = "UPDATE batch_export_requests SET export_expired = 1 " . + $this->whereAvailable . 'AND id = :id'; + return $this->dbh->execute($sql, array('id' => $id)); } - /* ******** List request records and states ******** */ - - // Return count of all export requests presently in Submitted state. + /** + * Return count of all export requests presently in Submitted state. + * + * @return integer Count of rows. + */ public function countSubmittedRecords() { - $sql = "SELECT COUNT(id) FROM batch_export_requests " . $this->whereSubmitted; - - // Return count of rows. - try { - $result = $this->pdo->query($sql); - return($result[0]['COUNT(id)']); - - } catch (Exception $e) { - return $e; - } + $sql = "SELECT COUNT(id) AS row_count FROM batch_export_requests " . $this->whereSubmitted; + $result = $this->dbh->query($sql); + return $result[0]['row_count']; } - // Return details of all export requests presently in Submitted state. + /** + * Return details of all export requests presently in Submitted state. + * + * @return array + */ public function listSubmittedRecords() { $sql = "SELECT id, realm, start_date, end_date, export_file_format, requested_datetime FROM batch_export_requests " . $this->whereSubmitted . ' ORDER BY requested_datetime, id'; - - // Return query results. - $result = $this->pdo->query($sql); - return($result); + return $this->dbh->query($sql); } /** @@ -166,11 +188,16 @@ public function listExpiredRecords() { $sql = 'SELECT id, realm, start_date, end_date, export_file_format, requested_datetime FROM batch_export_requests ' . $this->whereExpired . ' ORDER BY requested_datetime, id'; - return $this->pdo->query($sql); + return $this->dbh->query($sql); } - // Return details of export requests made by specified user. - public function listRequestsForUser($user_id) + /** + * Return details of export requests made by specified user. + * + * @param integer $userId + * @return array All of the user's export requests. + */ + public function listRequestsForUser($userId) { $sql = "SELECT id, realm, @@ -183,51 +210,53 @@ public function listRequestsForUser($user_id) export_file_format, requested_datetime FROM batch_export_requests - WHERE user_id=:user_id + WHERE user_id = :user_id ORDER BY requested_datetime, id"; - - $params = array('user_id' => $user_id); - - // Return query results. - $result = $this->pdo->query($sql, $params); - return($result); + return $this->dbh->query($sql, array('user_id' => $userId)); } - // Return details (including state) of export requests made by specified user. - public function listUserRequestsByState($user_id) + /** + * Return details (including state) of export requests made by specified user. + * + * @param integer $userId + * @return array All of the user's export requests (including state field). + */ + public function listUserRequestsByState($userId) { - $attributes = "SELECT id, realm, start_date, end_date, export_succeeded, export_expired, export_expires_datetime, export_created_datetime, export_file_format, requested_datetime, "; + $attributes = "SELECT id, + realm, + start_date, + end_date, + export_succeeded, + export_expired, + export_expires_datetime, + export_created_datetime, + export_file_format, + requested_datetime, + "; $fromTable = "FROM batch_export_requests "; - $whereAvailable = "WHERE export_succeeded = TRUE and export_created_datetime is NOT NULL and export_expired = FALSE "; - $whereFailed = "WHERE export_succeeded = FALSE and export_created_datetime is NULL and export_expired = FALSE "; $userClause = "AND user_id = :user_id "; - $sql = $attributes . "'Submitted' as state " . $fromTable . $this->whereSubmitted . $userClause . "UNION " . - $attributes . "'Available' as state " . $fromTable . $whereAvailable . $userClause . "UNION " . - $attributes . "'Expired' as state " . $fromTable . $this->whereExpired . $userClause . "UNION " . - $attributes . "'Failed' as state " . $fromTable . $whereFailed . $userClause . "ORDER BY requested_datetime, id"; - - $params = array('user_id' => $user_id); + $sql = $attributes . "'Submitted' AS state " . $fromTable . $this->whereSubmitted . $userClause . "UNION " . + $attributes . "'Available' AS state " . $fromTable . $this->whereAvailable . $userClause . "UNION " . + $attributes . "'Expired' AS state " . $fromTable . $this->whereExpired . $userClause . "UNION " . + $attributes . "'Failed' AS state " . $fromTable . $this->whereFailed . $userClause . "ORDER BY requested_datetime, id"; - // Return query results. - $result = $this->pdo->query($sql, $params); - return($result); + return $this->dbh->query($sql, array('user_id' => $userId)); } - /* ******** Delete single user-submitted request record ******** */ - - // Delete specified record from the database, regardless of its state. - // Only the user who submitted the request may delete it. - public function deleteRequest($id, $user) + /** + * Delete specified record from the database, regardless of its state. + * + * Only the user who submitted the request may delete it. + * + * @param integer $id Export request primary key. + * @param integer $userId + * @return integer Count of deleted rows--should be 1 if successful. + */ + public function deleteRequest($id, $userId) { - // delete record, providing that requesting user owns specified record - $sql = "DELETE FROM batch_export_requests WHERE id=:request_id AND user_id=:user_id"; - $params = array('request_id' => $id, - 'user_id' => $user - ); - - // Return count of deleted rows--should be 1 if successful. - $result = $this->pdo->execute($sql, $params); - return($result); + $sql = "DELETE FROM batch_export_requests WHERE id = :request_id AND user_id = :user_id"; + return $this->dbh->execute($sql, array('request_id' => $id, 'user_id' => $userId)); } } From f47c6bcc64dc826e334addb5369ca82717cf6a4c Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 28 Jun 2019 14:54:28 -0400 Subject: [PATCH 111/217] Rename and improve function --- classes/DataWarehouse/Export/QueryHandler.php | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index 2d36d4ade3..185a8506d3 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -3,8 +3,6 @@ * * Class governing database access by Data Warehouse Export batch script * - * TODO, possibly: return list of ids for records that need to be marked 'export_expired' - * * timestamp and current datetime: always use the database's value * * Recognized states enforced in this class: @@ -180,14 +178,24 @@ public function listSubmittedRecords() } /** - * Return export requests in Expired state. + * Return export requests in Available state that should expire. * * @return array */ - public function listExpiredRecords() + public function listExpiringRecords() { - $sql = 'SELECT id, realm, start_date, end_date, export_file_format, requested_datetime - FROM batch_export_requests ' . $this->whereExpired . ' ORDER BY requested_datetime, id'; + $sql = 'SELECT id, + realm, + start_date, + end_date, + export_succeeded, + export_expired, + export_expires_datetime, + export_created_datetime, + export_file_format, + requested_datetime + FROM batch_export_requests ' . $this->whereAvailable . ' AND export_expires_datetime > NOW() + ORDER BY requested_datetime, id'; return $this->dbh->query($sql); } From 8d34c895b28693b3712a2ec55c57a86b81a9aa24 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 28 Jun 2019 14:54:51 -0400 Subject: [PATCH 112/217] Add (mostly empty) email templates --- email_templates/batch_export_failure.template | 0 .../batch_export_failure_admin_notice.template | 0 email_templates/batch_export_success.template | 11 +++++++++++ 3 files changed, 11 insertions(+) create mode 100644 email_templates/batch_export_failure.template create mode 100644 email_templates/batch_export_failure_admin_notice.template create mode 100644 email_templates/batch_export_success.template diff --git a/email_templates/batch_export_failure.template b/email_templates/batch_export_failure.template new file mode 100644 index 0000000000..e69de29bb2 diff --git a/email_templates/batch_export_failure_admin_notice.template b/email_templates/batch_export_failure_admin_notice.template new file mode 100644 index 0000000000..e69de29bb2 diff --git a/email_templates/batch_export_success.template b/email_templates/batch_export_success.template new file mode 100644 index 0000000000..d3c6ba78bb --- /dev/null +++ b/email_templates/batch_export_success.template @@ -0,0 +1,11 @@ +Dear [:first_name:], + +Your requested data was generated on [:current_date:] and is available for +download at the following link: + + [:download_url:] + +This data will be available until [:expiration_date:]. + +Sincerely, +[:maintainer_signature:] From f5c27992eab7b41f3c2e920dbd4b4b4e204fa991 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 28 Jun 2019 14:55:54 -0400 Subject: [PATCH 113/217] Add class for realm related export functions --- classes/DataWarehouse/Export/RealmManager.php | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 classes/DataWarehouse/Export/RealmManager.php diff --git a/classes/DataWarehouse/Export/RealmManager.php b/classes/DataWarehouse/Export/RealmManager.php new file mode 100644 index 0000000000..8fca14ee2c --- /dev/null +++ b/classes/DataWarehouse/Export/RealmManager.php @@ -0,0 +1,88 @@ +dbh = DB::factory('database'); + $this->config = XdmodConfiguration::assocArrayFactory( + 'rawstatistics.json', + CONFIG_DIR + ); + } + + /** + * Get an array of all the batch exportable realms. + * + * @return \Models\Realm[] + */ + public function getRealms() + { + // The "display" values from rawstatistics match those in + // moddb.realms.display`, but the "name" values do not. + $exportable = array_map( + function ($realm) { + return $realm['display']; + }, + $this->config['realms'] + ); + + return array_filter( + Realms::getRealms(), + function ($realm) use ($exportable) { + return in_array($realm->getDisplay(), $exportable); + } + ); + } + + /** + * Get an array of all the batch exportable realms for a user. + * + * @param \XDUser $user + * @return \Models\Realm[] + */ + public function getRealmsForUser(XDUser $user) + { + // Returns data from moddb.realms.display column. + $userRealms = Realms::getRealmsForUser($user); + + return array_filter( + $this->getRealms(), + function ($realm) use ($userRealms) { + return in_array($realm->getDisplay(), $userRealms); + } + ); + } + + /** + */ + public function getRawDataQueryClass($realm) + { + } +} From c74cda1ed1bf071b7b823a2dd836817dea26352a Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 28 Jun 2019 14:56:22 -0400 Subject: [PATCH 114/217] Refactor, etc. --- .../WarehouseExportControllerProvider.php | 79 +++++++++---------- 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 8d3626d42a..1eb9e25cda 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -4,9 +4,9 @@ use CCR\DB; use DataWarehouse\Export\QueryHandler; +use DataWarehouse\Export\RealmManager; use DateTime; use Exception; -use Models\Services\Realms; use Silex\Application; use Silex\ControllerCollection; use Symfony\Component\HttpFoundation\Request; @@ -17,6 +17,23 @@ class WarehouseExportControllerProvider extends BaseControllerProvider { + /** + * @var DataWarehouse\Export\QueryHandler + */ + private $queryHandler; + + /** + * @var DataWarehouse\Export\RealmManager + */ + private $realmManager; + + public function __construct(array $params = []) + { + parent::__construct($params); + $this->realmManager = new RealmManager(); + $this->queryHandler = new QueryHandler(); + } + /** * Set up data warehouse export routes. * @@ -56,23 +73,14 @@ public function setupRoutes( public function getRealms(Request $request, Application $app) { $user = $this->authorize($request); - $userRealms = Realms::getRealmsForUser($user); // XXX Returns data from moddb.realms.display column. - - // TODO: Get list of exportable realms. - $realms = [ - ['id' => 'jobs', 'name' => 'Jobs'], - ['id' => 'supremm', 'name' => 'SUPReMM'], - ['id' => 'accounts', 'name' => 'Accounts'], - ['id' => 'allocations', 'name' => 'Allocations'], - ['id' => 'requests', 'name' => 'Requests'], - ['id' => 'resourceallocations', 'name' => 'ResourceAllocations'] - ]; - - $realms = array_filter( - $realms, - function ($realm) use ($userRealms) { - return in_array($realm['name'], $userRealms); - } + $realms = array_map( + function ($realm) { + return [ + 'id' => $realm->getName(), + 'name' => $realm->getDisplay() + ]; + }, + $this->realmManager->getRealmsForUser($user) ); return $app->json( @@ -95,8 +103,7 @@ function ($realm) use ($userRealms) { public function getRequests(Request $request, Application $app) { $user = $this->authorize($request); - $handler = new QueryHandler(); - $results = $handler->listUserRequestsByState($user->getUserId()); + $results = $this->queryHandler->listUserRequestsByState($user->getUserId()); return $app->json( [ 'success' => true, @@ -120,19 +127,12 @@ public function createRequest(Request $request, Application $app) $user = $this->authorize($request); $realm = $this->getStringParam($request, 'realm', true); - // TODO: Check that realm is in list of exportable realms. - //$userRealms = Realms::getRealmsForUser($user); // XXX Returns data from moddb.realms.display column. - //if (!in_array($realm, $userRealms)) { - // throw new BadRequestHttpException('Invalid realm'); - //} - $realms = [ - 'jobs', - 'supremm', - 'accounts', - 'allocations', - 'requests', - 'resourceallocations' - ]; + $realms = array_map( + function ($realm) { + return $this->getName(); + }, + $this->realmManager->getRealmsForUser($user) + ); if (!in_array($realm, $realms)) { throw new BadRequestHttpException('Invalid realm'); } @@ -165,9 +165,7 @@ public function createRequest(Request $request, Application $app) throw new BadRequestHttpException('format must be CSV or JSON'); } - $handler = new QueryHandler(); - - $id = $handler->createRequestRecord( + $id = $this->queryHandler->createRequestRecord( $user->getUserId(), $realm, $startDate->format('Y-m-d'), @@ -198,10 +196,9 @@ public function createRequest(Request $request, Application $app) public function getRequest(Request $request, Application $app, $id) { $user = $this->authorize($request); - $handler = new QueryHandler(); $requests = array_filter( - $handler->listUserRequestsByState($user->getUserId()), + $this->queryHandler->listUserRequestsByState($user->getUserId()), function ($request) use ($id) { return $request['id'] == $id; } @@ -271,8 +268,7 @@ function ($request) use ($id) { public function deleteRequest(Request $request, Application $app, $id) { $user = $this->authorize($request); - $handler = new QueryHandler(); - $count = $handler->deleteRequest($id, $user->getUserId()); + $count = $this->queryHandler->deleteRequest($id, $user->getUserId()); if ($count === 0) { throw new NotFoundHttpException('Export request not found'); @@ -300,7 +296,6 @@ public function deleteRequest(Request $request, Application $app, $id) public function deleteRequests(Request $request, Application $app) { $user = $this->authorize($request); - $handler = new QueryHandler(); $requestIds = []; @@ -331,7 +326,7 @@ public function deleteRequests(Request $request, Application $app) $dbh->beginTransaction(); foreach ($requestIds as $id) { - $count = $handler->deleteRequest($id, $user->getUserId()); + $count = $this->queryHandler->deleteRequest($id, $user->getUserId()); if ($count === 0) { throw new NotFoundHttpException('Export request not found'); } From d1de151639e968faefd1b9398241a444297e9286 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 28 Jun 2019 14:56:38 -0400 Subject: [PATCH 115/217] Add batch script, etc. --- background_scripts/batch_export_manager.php | 109 +++++++ .../DataWarehouse/Export/BatchProcessor.php | 277 ++++++++++++++++++ 2 files changed, 386 insertions(+) create mode 100755 background_scripts/batch_export_manager.php create mode 100644 classes/DataWarehouse/Export/BatchProcessor.php diff --git a/background_scripts/batch_export_manager.php b/background_scripts/batch_export_manager.php new file mode 100755 index 0000000000..3461012383 --- /dev/null +++ b/background_scripts/batch_export_manager.php @@ -0,0 +1,109 @@ +#!/usr/bin/env php + $value) { + if (is_array($value)) { + fwrite(STDERR, "Multiple values not allowed for '$key'\n"); + exit(1); + } + + switch ($key) { + case 'h': + case 'help': + $help = true; + break; + case 'dry-run': + $dryRun = true; + break; + case 'q': + case 'quiet': + $logLevel = max($logLevel, Log::WARNING); + break; + case 'v': + case 'verbose': + $logLevel = max($logLevel, Log::INFO); + break; + case 'd': + case 'debug': + $logLevel = max($logLevel, Log::DEBUG); + break; + default: + fwrite(STDERR, "Unexpected option '$key'\n"); + exit(1); + break; + } + } + + // Set default log level if none was specified. + if ($logLevel === -1) { + $logLevel = Log::NOTICE; + } + + if ($help) { + displayHelpText(); + exit; + } + + $conf = array( + 'file' => false, + 'mail' => false, + 'consoleLogLevel' => $logLevel + ); + $logger = CCR\Log('batch-export', $logConf); + $logger->info('Command: ' . implode(' ', array_map('escapeshellarg', $argv))); + // NOTE: "process_start_time" is needed for the log summary. + $logger->notice(['message' => 'batch_export_manager start', 'process_start_time' => date('Y-m-d H:i:s')]); + $batchProcessor = new BatchProcessor(); + $batchProcessor->setDryRun($dryRun); + $batchProcessor->setLogger($logger); + $batchProcessor->processRequests(); + // NOTE: "process_end_time" is needed for the log summary. + $logger->notice(['message' => 'batch_export_manager end', 'process_end_time' => date('Y-m-d H:i:s')]); + exit; +} catch (Exception $e) { + // Write any unexpected exceptions directly to STDERR since they may not + // have been logged and it may not be able to create a log instance. + fwrite(STDERR, "Data warehouse batch export failed\n"); + do { + fwrite(STDERR, $e->getMessage() . "\n" . $e->getTraceAsString() . "\n"); + } while ($e = $e->getPrevious()); + exit(1); +} + +function displayHelpText() +{ + global $argv; + + echo <<<"EOMSG" +Usage: {$argv[0]} + + -h, --help + Display this message and exit. + + -v, --verbose + Output info level logging. + + -d, --debug + Output debug level logging. + + -q, --quiet + Output warning level logging. + + --dry-run + Perform all the processing steps, but don't 't generate any files, + send any emails change status of any requests. +EOMSG; +} diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php new file mode 100644 index 0000000000..94991831b6 --- /dev/null +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -0,0 +1,277 @@ +dbh = DB::factory('database'); + $this->queryHandler = new QueryHandler(); + } + + /** + * Set whether or not processing the requests will be a dry run. + * + * If this is a dry run then no export files will be generated, no emails + * will be sent and no changes will be made to export requests in the + * database. + * + * @param boolean $dryRun + */ + public function setDryRun($dryRun) + { + $this->dryRun = $dryRun; + } + + /** + * Process all requests. + */ + public function processRequests() + { + $this->processSubmittedRequests(); + $this->processExpiringRequests(); + } + + /** + * Process requests in the "Submitted" state. + * + * Generate the data export and update the request. + */ + private function processSubmittedRequests() + { + $this->logger->info('Processing submitted requests'); + foreach ($this->queryHandler->listSubmittedRecords() as $request) { + $this->processSubmittedRequest($request); + } + } + + /** + */ + private function processSubmittedRequest(array $request) + { + $user = XDUser::getUserByID($request['user_id']); + + if ($user === null) { + $this->logger->err( + sprintf( + 'No user found for id = %d, request_id = %d', + $request['user_id'], + $request['id'] + ) + ); + return; + } + + try { + $this->dbh->beginTransaction(); + + $this->queryHandler->submittedToAvailable($request['id']); + // TODO: update request array with expiration date + //$request = $this->queryHandler->get + $this->dbh->commit(); + $this->sendExportSuccessEmail($user, $request); + } catch (Exception $e) { + $this->dbh->rollback(); + $this->sendExportFailureEmail( + $user, + $request, + $e->getMessage(), + $e->getPrevious() + ); + } + } + + /** + * Process requests in the "Available" state. + * + * Check if the request has expired and, if so, remove expired data and + * update the request. + */ + private function processExpiringdRequests() + { + $this->logger->info('Processing expired requests'); + foreach ($this->queryHandler->listExpiringRecords() as $request) { + $this->processExpiringRequest($request); + } + } + + /** + */ + private function processExpiringRequest(array $request) + { + try { + + } catch (Exception $e) { + } + } + + /** + */ + private function generateExportFile() + { + if ($this->dryRun) { + $this->logger->notice('dry run: Not generating export file'); + return; + } + + $this->logger->info('Generating export file'); + // TODO + } + + /** + */ + private function createZipFile($dataFile, $zipFile) + { + if ($this->dryRun) { + $this->logger->notice('dry run: Not creating zip file'); + return; + } + + $this->logger->info('Creating zip file'); + + $zip = new ZipArchive(); + $zipOpenCode = $zip->open($zipFile, ZipArchive::CREATE); + + if ($zipOpenCode !== true) { + $this->logAndThrowException( + sprintf( + 'Failed to create zip file "%s", error code "%s"', + $zipFile, + $zipOpenCode + ) + ); + } + + if ($zip->addFile($dataFile, basename($dataFile)) === false) { + $this->logAndThrowException( + sprintf( + 'Failed to add file "%s" to zip file "%s"', + $dataFile, + $zipFile + ) + ); + } + + $zip->close(); + } + + /** + */ + private function removeExportFile() + { + if ($this->dryRun) { + $this->logger->notice('dry run: Not removing export file'); + return; + } + + $this->logger->info('Removing export file'); + // TODO + } + + /** + * Send emails indicating a successful export. + * + * @param XDUser $user The user that requested the export. + * @param array $request The batch request data. + */ + private function sendExportSuccessEmail(XDUser $user, array $request) + { + if ($this->dryRun) { + $this->logger->notice('dry run: Not sending success email'); + return; + } + + $this->logger->info('Sending success email'); + + MailWrapper::sendTemplate( + 'batch_export_success', + [ + 'subject' => 'Batch export ready for download', + 'toAddress' => $user->getEmailAddress(), + 'first_name' => $user->getFirstName(), + 'last_name' => $user->getLastName(), + 'current_date' => date('Y-m-d'), + 'expiration_date' => $request[''], // TODO + 'download_url' => '', // TODO + 'maintainer_signature' => MailWrapper::getMaintainerSignature() + ] + ); + } + + /** + */ + private function sendExportFailureEmail( + XDUser $user, + array $request, + $failureReason, + Exception $e + ) { + if ($this->dryRun) { + $this->logger->notice('dry run: Not sending failure email'); + return; + } + + $this->logger->info('Sending failure email'); + + MailWrapper::sendTemplate( + 'batch_export_failure_admin_notice', + [ + 'subject' => 'Batch export failed', + 'toAddress' => xd_utilities\getConfiguration('general', 'tech_support_recipient'), + 'user_email' => $user->getEmailAddress(), + 'user_first_name' => $user->getFirstName(), + 'user_last_name' => $user->getLastName(), + 'current_date' => date('Y-m-d'), + 'failure_reason' => $failureReason, + 'failure_exception' => $e->getMessage(), + 'failure_stack_trace' => $e->getTraceAsString() + ] + ); + + MailWrapper::sendTemplate( + 'batch_export_failure', + [ + 'subject' => 'Batch export failed', + 'toAddress' => $user->getEmailAddress(), + 'first_name' => $user->getFirstName(), + 'last_name' => $user->getLastName(), + 'current_date' => date('Y-m-d'), + 'failure_reason' => $failureReason, + 'maintainer_signature' => MailWrapper::getMaintainerSignature() + ] + ); + } +} From d4aef00dbdad90b2f80d16fab4fea1df71ed70ad Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 28 Jun 2019 15:03:17 -0400 Subject: [PATCH 116/217] Fix typo and add missing code --- background_scripts/batch_export_manager.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/background_scripts/batch_export_manager.php b/background_scripts/batch_export_manager.php index 3461012383..a79ce0546d 100755 --- a/background_scripts/batch_export_manager.php +++ b/background_scripts/batch_export_manager.php @@ -14,6 +14,8 @@ $dryRun = false; $logLevel = -1; + $args = getopt('hqvd', ['help', 'dry-run', 'quiet', 'verbose', 'debug']); + foreach ($args as $key => $value) { if (is_array($value)) { fwrite(STDERR, "Multiple values not allowed for '$key'\n"); @@ -57,7 +59,7 @@ exit; } - $conf = array( + $logConf = array( 'file' => false, 'mail' => false, 'consoleLogLevel' => $logLevel From 6fb3441d9110e58d29d9e4bf7699facf8fc13db4 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Sun, 30 Jun 2019 18:17:53 -0400 Subject: [PATCH 117/217] Mostly documentation changes --- .../DataWarehouse/Export/BatchProcessor.php | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 94991831b6..d32aea433d 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -79,6 +79,9 @@ private function processSubmittedRequests() } /** + * Process a single export request. + * + * @param array $request The export request data. */ private function processSubmittedRequest(array $request) { @@ -97,14 +100,21 @@ private function processSubmittedRequest(array $request) try { $this->dbh->beginTransaction(); - $this->queryHandler->submittedToAvailable($request['id']); + // Get query class + // Execute query + // Write data to file + // Zip file // TODO: update request array with expiration date //$request = $this->queryHandler->get - $this->dbh->commit(); $this->sendExportSuccessEmail($user, $request); + $this->dbh->commit(); } catch (Exception $e) { $this->dbh->rollback(); + $this->logger->err([ + 'message' => 'Failed to export data: ' . $e->getMessage(), + 'stacktrace' => $e->getTraceAsString() + ]); $this->sendExportFailureEmail( $user, $request, @@ -120,7 +130,7 @@ private function processSubmittedRequest(array $request) * Check if the request has expired and, if so, remove expired data and * update the request. */ - private function processExpiringdRequests() + private function processExpiringRequests() { $this->logger->info('Processing expired requests'); foreach ($this->queryHandler->listExpiringRecords() as $request) { @@ -129,12 +139,25 @@ private function processExpiringdRequests() } /** + * Process a single export request that is expiring. + * + * @param array $request The export request data. */ private function processExpiringRequest(array $request) { try { - + $this->dbh->beginTransaction(); + $this->queryHandler->availableToExpired($request['id']); + // TODO + // Delete file + $this->dbh->commit(); } catch (Exception $e) { + $this->dbh->rollback(); + $this->logger->err( + sprintf( + 'Failed to remove expired data' + ) + ); } } @@ -152,6 +175,10 @@ private function generateExportFile() } /** + * Create a zip file containing a single file. + * + * @param string $dataFile Absolute path to file that will be put in zip file. + * @param string $zipFile Absolute path to zip file that will be created. */ private function createZipFile($dataFile, $zipFile) { @@ -202,7 +229,7 @@ private function removeExportFile() } /** - * Send emails indicating a successful export. + * Send email indicating a successful export. * * @param XDUser $user The user that requested the export. * @param array $request The batch request data. @@ -232,6 +259,15 @@ private function sendExportSuccessEmail(XDUser $user, array $request) } /** + * Send email indicating a failed export. + * + * Sends one email to the user that created the request and another email + * to the tech support recipient. + * + * @param XDUser $user The user that created the request. + * @param array $request Export request data. + * @param string $failureReason Friendly text explaining failure. + * @param Exception $e The exception that caused the failure. */ private function sendExportFailureEmail( XDUser $user, From f4ff0e76ecfc333dc478590768648ef7cf57e99a Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 1 Jul 2019 07:02:48 -0400 Subject: [PATCH 118/217] Fix namespace --- classes/DataWarehouse/Export/RealmManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/RealmManager.php b/classes/DataWarehouse/Export/RealmManager.php index 8fca14ee2c..8f9d2529e0 100644 --- a/classes/DataWarehouse/Export/RealmManager.php +++ b/classes/DataWarehouse/Export/RealmManager.php @@ -7,7 +7,7 @@ use CCR\DB; use Configuration\XdmodConfiguration; -use Service\Realms; +use Models\Services\Realms; use XDUser; /** From a12ed843122f7d3b2343acd0403b482b851fb438 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 1 Jul 2019 07:15:55 -0400 Subject: [PATCH 119/217] Fix typo --- classes/Rest/Controllers/WarehouseExportControllerProvider.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 1eb9e25cda..9b43bb8146 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -129,7 +129,7 @@ public function createRequest(Request $request, Application $app) $realms = array_map( function ($realm) { - return $this->getName(); + return $realm->getName(); }, $this->realmManager->getRealmsForUser($user) ); From 4b4b9cb3e6a3dc6b9290220b7fe0cafdac331a13 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 2 Jul 2019 07:57:12 -0400 Subject: [PATCH 120/217] Implement function --- classes/DataWarehouse/Export/RealmManager.php | 48 ++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/RealmManager.php b/classes/DataWarehouse/Export/RealmManager.php index 8f9d2529e0..55ba0e2eb6 100644 --- a/classes/DataWarehouse/Export/RealmManager.php +++ b/classes/DataWarehouse/Export/RealmManager.php @@ -7,6 +7,7 @@ use CCR\DB; use Configuration\XdmodConfiguration; +use Exception; use Models\Services\Realms; use XDUser; @@ -81,8 +82,53 @@ function ($realm) use ($userRealms) { } /** + * Get the raw data query class for the given realm. + * + * @param string $realmName The realm name used in moddb.realms.name. + * @return string The fully qualified name of the query class. */ - public function getRawDataQueryClass($realm) + public function getRawDataQueryClass($realmName) { + // The query classes use the "name" from the rawstatistics + // configuration, but the realm name is taken from moddb.realms.name. + // These use the same "display" name so that is used to find the + // correct class name. + + // Realm model. + $realmObj = null; + + foreach ($this->getRealms() as $realm) { + if ($realm->getName() == $realmName) { + $realmObj = $realm; + break; + } + } + + if ($realmObj === null) { + throw new Exception( + sprintf('Failed to find model for realm "%s"', $realmName) + ); + } + + // Realm rawstatistics configuration. + $realmConfig = null; + + foreach ($this->config['realms'] as $realm) { + if ($realm['display'] == $realmObj->getDisplay()) { + $realmConfig = $realm; + break; + } + } + + if ($realmConfig === null) { + throw new Exception( + sprintf( + 'Failed to find rawstatistics configuration for realm "%s"', + $realmName + ) + ); + } + + return sprintf('\DataWarehouse\Query\%s\JobDataset', $realmConfig['name']); } } From 2e6584871c877cf7bb8e65e0ce144efaed91e1e2 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 2 Jul 2019 07:57:28 -0400 Subject: [PATCH 121/217] Add function --- classes/DataWarehouse/Export/QueryHandler.php | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index 185a8506d3..fdd94fb96e 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -100,6 +100,29 @@ public function createRequestRecord( return $this->dbh->insert($sql, $params); } + /** + * Get a single export request record. + * + * @param integer $id Export request primary key. + * @return array + */ + public function getRequestRecord($id) + { + $sql = 'SELECT id, + realm, + start_date, + end_date, + export_succeeded, + export_expired, + export_expires_datetime, + export_created_datetime, + export_file_format, + requested_datetime + FROM batch_export_requests + WHERE id = :id'; + return $this->dbh->query($sql, ['id' => $id]); + } + /** * Transition specified export request from Submitted state to Failed state. * From da101b12d18ea1c44af62d8b8a87a6c5e0d8dc51 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 3 Jul 2019 08:14:55 -0400 Subject: [PATCH 122/217] Add tests --- .../output/query-class-for-realm.json | 6 + .../realm_manager/output/realms-for-user.json | 11 ++ .../export/realm_manager/output/realms.json | 10 ++ .../component/lib/Export/RealmManagerTest.php | 144 ++++++++++++++++++ 4 files changed, 171 insertions(+) create mode 100644 tests/artifacts/xdmod/component/export/realm_manager/output/query-class-for-realm.json create mode 100644 tests/artifacts/xdmod/component/export/realm_manager/output/realms-for-user.json create mode 100644 tests/artifacts/xdmod/component/export/realm_manager/output/realms.json create mode 100644 tests/component/lib/Export/RealmManagerTest.php diff --git a/tests/artifacts/xdmod/component/export/realm_manager/output/query-class-for-realm.json b/tests/artifacts/xdmod/component/export/realm_manager/output/query-class-for-realm.json new file mode 100644 index 0000000000..73fef7817a --- /dev/null +++ b/tests/artifacts/xdmod/component/export/realm_manager/output/query-class-for-realm.json @@ -0,0 +1,6 @@ +{ + "Jobs realm": [ + "jobs", + "\\DataWarehouse\\Query\\Jobs\\JobDataset" + ] +} diff --git a/tests/artifacts/xdmod/component/export/realm_manager/output/realms-for-user.json b/tests/artifacts/xdmod/component/export/realm_manager/output/realms-for-user.json new file mode 100644 index 0000000000..38f92b3b9b --- /dev/null +++ b/tests/artifacts/xdmod/component/export/realm_manager/output/realms-for-user.json @@ -0,0 +1,11 @@ +{ + "Normal user": [ + "usr", + [ + { + "name": "jobs", + "display": "Jobs" + } + ] + ] +} diff --git a/tests/artifacts/xdmod/component/export/realm_manager/output/realms.json b/tests/artifacts/xdmod/component/export/realm_manager/output/realms.json new file mode 100644 index 0000000000..925dc05cf0 --- /dev/null +++ b/tests/artifacts/xdmod/component/export/realm_manager/output/realms.json @@ -0,0 +1,10 @@ +{ + "Batch exportable realms": [ + [ + { + "name": "jobs", + "display": "Jobs" + } + ] + ] +} diff --git a/tests/component/lib/Export/RealmManagerTest.php b/tests/component/lib/Export/RealmManagerTest.php new file mode 100644 index 0000000000..9308ef41ca --- /dev/null +++ b/tests/component/lib/Export/RealmManagerTest.php @@ -0,0 +1,144 @@ + 'Public User', + 'usr' => 'normaluser', + 'pi' => 'principal', + 'cs' => 'centerstaff', + 'cd' => 'centerdirector', + 'mgr' => 'admin' + ]; + + /** + * User for each role. + * @var XDUser[] + */ + private static $users = []; + + /** + */ + public static function setUpBeforeClass() + { + parent::setUpBeforeClass(); + + self::$realmManager = new RealmManager(); + + foreach (self::$userRoles as $role => $username) { + //if ($role !== 'pub') { + self::$users[$role] = XDUser::getUserByUserName($username); + //} + } + } + + /** + * Convert a realm model object to an array. + * + * Only includes the relevant properties from the realm that are used by + * the realm manager class. + * + * @param \Model\Realm $realm + * @return array + */ + private function convertRealmToArray(Realm $realm) + { + return [ + 'name' => $realm->getName(), + 'display' => $realm->getDisplay() + ]; + } + + /** + * Test which realms may be exported. + * + * @covers ::getRealms + * @dataProvider getRealmsProvider + */ + public function testGetRealms($realms) + { + $this->assertEquals( + $realms, + array_map( + [$this, 'convertRealmToArray'], + self::$realmManager->getRealms() + ), + 'getRealms returns expected realms' + ); + } + + /** + * Test which realms may be exported for a given user. + * + * @covers ::getRealmsForUser + * @dataProvider getRealmsForUserProvider + */ + public function testGetRealmsForUser($role, $realms) + { + $this->assertEquals( + $realms, + array_map( + [$this, 'convertRealmToArray'], + self::$realmManager->getRealmsForUser(self::$users[$role]) + ), + "getRealmsForUser returns expected realms for role $role" + ); + } + + /** + * Test what query class should be used for each realm. + * + * @covers ::getRawDataQueryClass + * @dataProvider getRawDataQueryClassProvider + */ + public function testGetRawDataQueryClassProvider($realmName, $queryClassName) + { + $this->assertEquals( + $queryClassName, + self::$realmManager->getRawDataQueryClass($realmName), + "getRawDataQueryClass returns expected query class for realm $realmName" + ); + } + + public function getRealmsProvider() + { + return $this->getTestFiles()->loadJsonFile(self::TEST_GROUP, 'realms', 'output'); + } + + public function getRealmsForUserProvider() + { + return $this->getTestFiles()->loadJsonFile(self::TEST_GROUP, 'realms-for-user', 'output'); + } + + public function getRawDataQueryClassProvider() + { + return $this->getTestFiles()->loadJsonFile(self::TEST_GROUP, 'query-class-for-realm', 'output'); + } +} From 412be8da38aa0f536df2a9d8df504b28eab1b0f8 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 3 Jul 2019 12:58:01 -0400 Subject: [PATCH 123/217] More batch export changes --- .../DataWarehouse/Export/BatchProcessor.php | 108 ++++++++++++++---- .../DataWarehouse/Query/Jobs/JobDataset.php | 35 +++++- 2 files changed, 118 insertions(+), 25 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index d32aea433d..857109e26f 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -8,6 +8,7 @@ use CCR\DB; use CCR\Loggable; use CCR\MailWrapper; +use DataWarehouse\Data\RawDataset; use Exception; use XDUser; use ZipArchive; @@ -85,28 +86,35 @@ private function processSubmittedRequests() */ private function processSubmittedRequest(array $request) { + $this->logger->debug([ + 'message' => 'Processing request', + 'batch_export_request.id' => $request['id'] + ]); + $user = XDUser::getUserByID($request['user_id']); if ($user === null) { - $this->logger->err( - sprintf( - 'No user found for id = %d, request_id = %d', - $request['user_id'], - $request['id'] - ) - ); + $this->logger->err([ + 'message' => 'User not found', + 'Users.id' => $request['user_id'], + 'batch_export_request.id' => $request['id'] + ]); return; } try { $this->dbh->beginTransaction(); $this->queryHandler->submittedToAvailable($request['id']); - // Get query class - // Execute query - // Write data to file + + $dataSet = $this->getDataSet($request); + + $dataFile = $this->writeDataToFile($dataSet, $request['format']); + // Zip file - // TODO: update request array with expiration date - //$request = $this->queryHandler->get + $this->createZipFile($dataFile, $zipFile) + + // Query for same record to get expiration date. + $request = $this->queryHandler->getRequestRecord($request['id']); $this->sendExportSuccessEmail($user, $request); $this->dbh->commit(); } catch (Exception $e) { @@ -115,17 +123,18 @@ private function processSubmittedRequest(array $request) 'message' => 'Failed to export data: ' . $e->getMessage(), 'stacktrace' => $e->getTraceAsString() ]); + $failureReason = '...'; $this->sendExportFailureEmail( $user, $request, - $e->getMessage(), - $e->getPrevious() + $failureReason, + $e ); } } /** - * Process requests in the "Available" state. + * Process requests in the "Available" state that should be expired. * * Check if the request has expired and, if so, remove expired data and * update the request. @@ -145,19 +154,71 @@ private function processExpiringRequests() */ private function processExpiringRequest(array $request) { + $this->logger->debug([ + 'message' => 'Expiring request', + 'batch_export_request.id' => $request['id'] + ]); + try { $this->dbh->beginTransaction(); - $this->queryHandler->availableToExpired($request['id']); // TODO // Delete file + $this->queryHandler->availableToExpired($request['id']); $this->dbh->commit(); } catch (Exception $e) { $this->dbh->rollback(); - $this->logger->err( - sprintf( - 'Failed to remove expired data' - ) + $this->logger->err([ + 'message' => 'Failed to expire record: ' . $e->getMessage(), + 'stacktrace' => $e->getTraceAsString() + ]); + } + } + + /** + * Get the data set for the given request. + * + * @param array $request + * @param \XDUser $user + * @return \DataWarehouse\Data\RawDataset; + * @throws \Exception + */ + private function getDataSet(array $request, XDUser $user) + { + $this->logger->info([ + 'message' => 'Querying data' + 'Users.id' => $user->getUserID(), + 'user_email' => $user->getEmailAddress(), + 'user_first_name' => $user->getFirstName(), + 'user_last_name' => $user->getLastName(), + 'realm' => $request['realm'], + 'start_date' => $request['start_date'], + 'end_date' => $request['end_date'] + ]); + + try { + $className = $this->realmManager->getRawDataQueryClass($request['realm']); + $this->logger->debug(sprintf('Instantiating query class "%s"', $className)); + $query = new $className( + [ + 'start_date' => $request['start_date'], + 'end_date' => $request['end_date'] + ], + 'accounting' ); + $allRoles = $user->getAllRoles(); + $query->setMultipleRoleParameters($allRoles, $user); + + $dataSet = new RawDataset($query, $user); + + // Data are fetched from the database as a side effect of checking + // for results. + if ($dataSet->hasResults()) { + $this->logger->debug('Data set has results'); + } + + return $dataSet; + } catch (Exception $e) { + throw new Exception('Failed to execute batch export query' 0, $e); } } @@ -179,6 +240,7 @@ private function generateExportFile() * * @param string $dataFile Absolute path to file that will be put in zip file. * @param string $zipFile Absolute path to zip file that will be created. + * @throws \Exception */ private function createZipFile($dataFile, $zipFile) { @@ -231,7 +293,7 @@ private function removeExportFile() /** * Send email indicating a successful export. * - * @param XDUser $user The user that requested the export. + * @param \XDUser $user The user that requested the export. * @param array $request The batch request data. */ private function sendExportSuccessEmail(XDUser $user, array $request) @@ -264,10 +326,10 @@ private function sendExportSuccessEmail(XDUser $user, array $request) * Sends one email to the user that created the request and another email * to the tech support recipient. * - * @param XDUser $user The user that created the request. + * @param \XDUser $user The user that created the request. * @param array $request Export request data. * @param string $failureReason Friendly text explaining failure. - * @param Exception $e The exception that caused the failure. + * @param \Exception $e The exception that caused the failure. */ private function sendExportFailureEmail( XDUser $user, diff --git a/classes/DataWarehouse/Query/Jobs/JobDataset.php b/classes/DataWarehouse/Query/Jobs/JobDataset.php index be3c708b18..08fa4a0bfa 100644 --- a/classes/DataWarehouse/Query/Jobs/JobDataset.php +++ b/classes/DataWarehouse/Query/Jobs/JobDataset.php @@ -42,7 +42,7 @@ public function __construct( if (isset($parameters['primary_key'])) { $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'job_id'), "=", $parameters['primary_key'])); - } else { + } elseif (isset($parameters['job_identifier'])) { $matches = array(); if (preg_match('/^(\d+)(?:[\[_](\d+)\]?)?$/', $parameters['job_identifier'], $matches)) { $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'resource_id'), '=', $parameters['resource_id'])); @@ -53,8 +53,39 @@ public function __construct( $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'local_job_id_raw'), '=', $matches[1])); } } else { - throw new \Exception('invalid query parameters'); + throw new \Exception('invalid "job_identifier" query parameter'); + } + } elseif (isset($parameters['start_date']) && isset($parameters['end_date'])) { + $startDate = date_parse_from_format('Y-m-d', $parameters['start_date']); + $startDateTs = mktime( + $startDate['hour'], + $startDate['minute'], + $startDate['second'], + $startDate['month'], + $startDate['day'], + $startDate['year'] + ); + if ($startDateTs === false) { + throw new \Exception('invalid "start_date" query parameter'); + } + + $endDate = date_parse_from_format('Y-m-d', $parameters['end_date']); + $endDateTs = mktime( + 23, + 59, + 59, + $endDate['month'], + $endDate['day'], + $endDate['year'] + ); + if ($startDateTs === false) { + throw new \Exception('invalid "end_date" query parameter'); } + + $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_date_ts'), ">=", $startDateTs)); + $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_date_ts'), "<=", $endDateTs)); + } else { + throw new \Exception('invalid query parameters'); } if ($stat == "accounting") { From 3f78735d1735def5b6a11fc9d406b6d07de51325 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 3 Jul 2019 15:03:33 -0400 Subject: [PATCH 124/217] Fix syntax, etc. --- classes/DataWarehouse/Export/BatchProcessor.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 857109e26f..0038c077a4 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -110,8 +110,8 @@ private function processSubmittedRequest(array $request) $dataFile = $this->writeDataToFile($dataSet, $request['format']); - // Zip file - $this->createZipFile($dataFile, $zipFile) + $zipFile = 'TODO'; + $this->createZipFile($dataFile, $zipFile); // Query for same record to get expiration date. $request = $this->queryHandler->getRequestRecord($request['id']); @@ -185,7 +185,7 @@ private function processExpiringRequest(array $request) private function getDataSet(array $request, XDUser $user) { $this->logger->info([ - 'message' => 'Querying data' + 'message' => 'Querying data', 'Users.id' => $user->getUserID(), 'user_email' => $user->getEmailAddress(), 'user_first_name' => $user->getFirstName(), @@ -218,7 +218,7 @@ private function getDataSet(array $request, XDUser $user) return $dataSet; } catch (Exception $e) { - throw new Exception('Failed to execute batch export query' 0, $e); + throw new Exception('Failed to execute batch export query', 0, $e); } } From 7b4e3d369e3aba48075781fd9fc77ba2326b83f7 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Sun, 7 Jul 2019 13:48:51 -0400 Subject: [PATCH 125/217] Replace tabs with spaces and improve help text --- background_scripts/batch_export_manager.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/background_scripts/batch_export_manager.php b/background_scripts/batch_export_manager.php index a79ce0546d..f8dbfe79fc 100755 --- a/background_scripts/batch_export_manager.php +++ b/background_scripts/batch_export_manager.php @@ -105,7 +105,7 @@ function displayHelpText() Output warning level logging. --dry-run - Perform all the processing steps, but don't 't generate any files, - send any emails change status of any requests. + Perform all the processing steps, but don't generate or remove any + files, send any emails, or change the status of any export requests. EOMSG; } From 2ab6b16da7ae60c7fc5c787068428aca30572209 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Sun, 7 Jul 2019 17:54:31 -0400 Subject: [PATCH 126/217] More implementation work and tests --- .../DataWarehouse/Export/BatchProcessor.php | 82 +++++++++++--- .../component/lib/Export/BatchProcessTest.php | 105 ++++++++++++++++++ 2 files changed, 173 insertions(+), 14 deletions(-) create mode 100644 tests/component/lib/Export/BatchProcessTest.php diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 0038c077a4..f4d22afa6f 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -34,6 +34,12 @@ class BatchProcessor extends Loggable */ private $queryHandler; + /** + * Path of directory containing export zip files. + * @var string + */ + private $exportDirectory; + /** * Construct a new batch processor. */ @@ -41,6 +47,10 @@ public function __construct() { $this->dbh = DB::factory('database'); $this->queryHandler = new QueryHandler(); + $this->exportDirectory = xd_utilities::getConfiguration( + 'data_warehouse_export', + 'export_directory' + ); } /** @@ -105,12 +115,9 @@ private function processSubmittedRequest(array $request) try { $this->dbh->beginTransaction(); $this->queryHandler->submittedToAvailable($request['id']); - $dataSet = $this->getDataSet($request); - $dataFile = $this->writeDataToFile($dataSet, $request['format']); - - $zipFile = 'TODO'; + $zipFile = $this->getExportZipFilePath($request['id']); $this->createZipFile($dataFile, $zipFile); // Query for same record to get expiration date. @@ -161,9 +168,8 @@ private function processExpiringRequest(array $request) try { $this->dbh->beginTransaction(); - // TODO - // Delete file $this->queryHandler->availableToExpired($request['id']); + $this->removeExportFile($request['id']); $this->dbh->commit(); } catch (Exception $e) { $this->dbh->rollback(); @@ -190,6 +196,7 @@ private function getDataSet(array $request, XDUser $user) 'user_email' => $user->getEmailAddress(), 'user_first_name' => $user->getFirstName(), 'user_last_name' => $user->getLastName(), + 'batch_export_request.id' => $request['id'], 'realm' => $request['realm'], 'start_date' => $request['start_date'], 'end_date' => $request['end_date'] @@ -223,16 +230,36 @@ private function getDataSet(array $request, XDUser $user) } /** + * Write data set to file. + * + * @param \DataWarehouse\Data\RawDataset $dataSet + * @param string $format */ - private function generateExportFile() + private function writeDataToFile(RawDataset $dataSet, $format) { if ($this->dryRun) { - $this->logger->notice('dry run: Not generating export file'); + $this->logger->notice('dry run: Not writing data to file'); return; } - $this->logger->info('Generating export file'); - // TODO + $tmpFile = tempnam(sys_get_temp_dir(), 'batch-export-'); + + $this->logger->info([ + 'message' => 'Writing data to file', + 'tmp_file' => $tmpFile + ]); + + // The `export` function returns the first result along with the + // necessary metadata. + foreach ($dataSet->export() as $datum) { + // TODO + } + + foreach ($dataSet->getResults() as $result) { + // TODO + } + + return $tmpFile; } /** @@ -249,7 +276,11 @@ private function createZipFile($dataFile, $zipFile) return; } - $this->logger->info('Creating zip file'); + $this->logger->info([ + 'message' => 'Creating zip file', + 'data_file' => $dataFile, + 'zip_file' => $zipFile + ]); $zip = new ZipArchive(); $zipOpenCode = $zip->open($zipFile, ZipArchive::CREATE); @@ -278,16 +309,39 @@ private function createZipFile($dataFile, $zipFile) } /** + * Remove an export data file. + * + * @param int $id Export request primary key. */ - private function removeExportFile() + private function removeExportFile($id) { if ($this->dryRun) { $this->logger->notice('dry run: Not removing export file'); return; } - $this->logger->info('Removing export file'); - // TODO + $zipFile = $this->getExportZipFilePath($id); + + $this->logger->info([ + 'message' => 'Removing export file', + 'batch_export_request.id' => $id, + 'zip_file' => $zipFile + ]); + + if (!unlink($zipFile)) { + throw new Exception(sprintf('Failed to delete "%s"', $zipFile)); + } + } + + /** + * Get the full path for the export data file. + * + * @param int $id Export request primary key. + * @return string + */ + private function getExportZipFilePath($id) + { + return $this->exportDirectory . DIRECTORY_SEPARATOR . $id . '.zip'; } /** diff --git a/tests/component/lib/Export/BatchProcessTest.php b/tests/component/lib/Export/BatchProcessTest.php new file mode 100644 index 0000000000..710d0af4b1 --- /dev/null +++ b/tests/component/lib/Export/BatchProcessTest.php @@ -0,0 +1,105 @@ +setDryRun(true); + + $emails = $this->getEmails(); + $files = $this->getExportFiles(); + $submittedRequests = $this->queryHandler->listSubmittedRecords(); + $expiringRequests = $this->queryHandler->listExpiringRecords(); + + $batchProcessor->processRequests(); + + $this->assertEquals($emails, $this->getEmails(), 'No new emails'); + $this->assertEquals($files, $this->getFiles(), 'No new export files'); + $this->assertEquals( + $submittedRequests, + $this->queryHandler->listSubmittedRecords(), + 'No submitted requests changed' + ); + $this->assertEquals( + $expiringRequests, + $this->queryHandler->listExpiringRecords(), + 'No expiring requests changed' + ); + + $this->markTestIncomplete('This test has not been implemented yet.'); + } + + /** + * Test processing batch export requests. + */ + public function testRequestProcessing() + { + $batchProcessor = new BatchProcessor(); + $this->markTestIncomplete('This test has not been implemented yet.'); + } +} From e8d5e7104328f41322581af24c482298e567e58b Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Sun, 7 Jul 2019 17:58:47 -0400 Subject: [PATCH 127/217] Use unused variable --- tests/component/lib/Export/BatchProcessTest.php | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/component/lib/Export/BatchProcessTest.php b/tests/component/lib/Export/BatchProcessTest.php index 710d0af4b1..8a6497d78b 100644 --- a/tests/component/lib/Export/BatchProcessTest.php +++ b/tests/component/lib/Export/BatchProcessTest.php @@ -100,6 +100,7 @@ public function testDryRun() public function testRequestProcessing() { $batchProcessor = new BatchProcessor(); + $batchProcessor->processRequests(); $this->markTestIncomplete('This test has not been implemented yet.'); } } From 374f979f8a97ac2da20ce8ea459be8733aa447b7 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Sun, 7 Jul 2019 18:44:04 -0400 Subject: [PATCH 128/217] Fix namespaces and base class --- tests/component/lib/Export/BatchProcessTest.php | 3 ++- tests/component/lib/Export/RealmManagerTest.php | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/component/lib/Export/BatchProcessTest.php b/tests/component/lib/Export/BatchProcessTest.php index 8a6497d78b..17bb7a8464 100644 --- a/tests/component/lib/Export/BatchProcessTest.php +++ b/tests/component/lib/Export/BatchProcessTest.php @@ -1,7 +1,8 @@ Date: Sun, 7 Jul 2019 19:03:07 -0400 Subject: [PATCH 129/217] Fix use of xd_utilities --- classes/DataWarehouse/Export/BatchProcessor.php | 2 +- tests/component/lib/Export/BatchProcessTest.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index f4d22afa6f..cbdb50a467 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -47,7 +47,7 @@ public function __construct() { $this->dbh = DB::factory('database'); $this->queryHandler = new QueryHandler(); - $this->exportDirectory = xd_utilities::getConfiguration( + $this->exportDirectory = xd_utilities\getConfiguration( 'data_warehouse_export', 'export_directory' ); diff --git a/tests/component/lib/Export/BatchProcessTest.php b/tests/component/lib/Export/BatchProcessTest.php index 17bb7a8464..4659182745 100644 --- a/tests/component/lib/Export/BatchProcessTest.php +++ b/tests/component/lib/Export/BatchProcessTest.php @@ -42,7 +42,7 @@ public static function setUpBeforeClass() { parent::setUpBeforeClass(); self::$queryHandler = new QueryHandler(); - self::$exportDirectory = xd_utilities::getConfiguration( + self::$exportDirectory = xd_utilities\getConfiguration( 'data_warehouse_export', 'export_directory' ); From 44dca683c05b9d92dd236ff84155a1db2905abc6 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 8 Jul 2019 07:21:02 -0400 Subject: [PATCH 130/217] Call parent constructor --- classes/DataWarehouse/Export/BatchProcessor.php | 1 + 1 file changed, 1 insertion(+) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index cbdb50a467..de338058c5 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -45,6 +45,7 @@ class BatchProcessor extends Loggable */ public function __construct() { + parent::__construct(); $this->dbh = DB::factory('database'); $this->queryHandler = new QueryHandler(); $this->exportDirectory = xd_utilities\getConfiguration( From 8a8819bb0f1ace938876fd470175100a66fd3bea Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 8 Jul 2019 08:38:38 -0400 Subject: [PATCH 131/217] Change logging --- classes/DataWarehouse/Export/BatchProcessor.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index de338058c5..87824bb3df 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -97,7 +97,7 @@ private function processSubmittedRequests() */ private function processSubmittedRequest(array $request) { - $this->logger->debug([ + $this->logger->info([ 'message' => 'Processing request', 'batch_export_request.id' => $request['id'] ]); @@ -162,7 +162,7 @@ private function processExpiringRequests() */ private function processExpiringRequest(array $request) { - $this->logger->debug([ + $this->logger->info([ 'message' => 'Expiring request', 'batch_export_request.id' => $request['id'] ]); @@ -397,7 +397,10 @@ private function sendExportFailureEmail( return; } - $this->logger->info('Sending failure email'); + $this->logger->info([ + 'message' => 'Sending failure email', + 'batch_export_request.id' => $request['id'] + ]); MailWrapper::sendTemplate( 'batch_export_failure_admin_notice', From 28fd65080a638b637711d51c6650de22b1fbc13b Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 8 Jul 2019 08:39:09 -0400 Subject: [PATCH 132/217] Change function name --- classes/DataWarehouse/Export/BatchProcessor.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 87824bb3df..3591e1880c 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -117,7 +117,7 @@ private function processSubmittedRequest(array $request) $this->dbh->beginTransaction(); $this->queryHandler->submittedToAvailable($request['id']); $dataSet = $this->getDataSet($request); - $dataFile = $this->writeDataToFile($dataSet, $request['format']); + $dataFile = $this->writeDataSetToFile($dataSet, $request['format']); $zipFile = $this->getExportZipFilePath($request['id']); $this->createZipFile($dataFile, $zipFile); @@ -236,7 +236,7 @@ private function getDataSet(array $request, XDUser $user) * @param \DataWarehouse\Data\RawDataset $dataSet * @param string $format */ - private function writeDataToFile(RawDataset $dataSet, $format) + private function writeDataSetToFile(RawDataset $dataSet, $format) { if ($this->dryRun) { $this->logger->notice('dry run: Not writing data to file'); From d954882c97b51778c66462beebc8e8568f1814b9 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 8 Jul 2019 09:08:27 -0400 Subject: [PATCH 133/217] Change failure reporting emails --- .../DataWarehouse/Export/BatchProcessor.php | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 3591e1880c..6e16cdab45 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -131,13 +131,7 @@ private function processSubmittedRequest(array $request) 'message' => 'Failed to export data: ' . $e->getMessage(), 'stacktrace' => $e->getTraceAsString() ]); - $failureReason = '...'; - $this->sendExportFailureEmail( - $user, - $request, - $failureReason, - $e - ); + $this->sendExportFailureEmail($user, $request, $e); } } @@ -383,13 +377,11 @@ private function sendExportSuccessEmail(XDUser $user, array $request) * * @param \XDUser $user The user that created the request. * @param array $request Export request data. - * @param string $failureReason Friendly text explaining failure. * @param \Exception $e The exception that caused the failure. */ private function sendExportFailureEmail( XDUser $user, array $request, - $failureReason, Exception $e ) { if ($this->dryRun) { @@ -402,6 +394,12 @@ private function sendExportFailureEmail( 'batch_export_request.id' => $request['id'] ]); + $message = $e->getMessage(); + $stackTrace = $e->getTraceAsString(); + while ($e = $e->getPrevious()) { + $stackTrace .= "\n\n" . $e->getTraceAsString(); + } + MailWrapper::sendTemplate( 'batch_export_failure_admin_notice', [ @@ -411,9 +409,8 @@ private function sendExportFailureEmail( 'user_first_name' => $user->getFirstName(), 'user_last_name' => $user->getLastName(), 'current_date' => date('Y-m-d'), - 'failure_reason' => $failureReason, - 'failure_exception' => $e->getMessage(), - 'failure_stack_trace' => $e->getTraceAsString() + 'failure_exception' => $message, + 'failure_stack_trace' => $stackTrace ] ); @@ -425,7 +422,7 @@ private function sendExportFailureEmail( 'first_name' => $user->getFirstName(), 'last_name' => $user->getLastName(), 'current_date' => date('Y-m-d'), - 'failure_reason' => $failureReason, + 'failure_reason' => $message, 'maintainer_signature' => MailWrapper::getMaintainerSignature() ] ); From cff8563a08bc2499247a66ae1ffaa25c5cbfec5a Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 8 Jul 2019 09:11:24 -0400 Subject: [PATCH 134/217] Fix bug and make some functions static --- .../component/lib/Export/BatchProcessTest.php | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/component/lib/Export/BatchProcessTest.php b/tests/component/lib/Export/BatchProcessTest.php index 4659182745..0451465bc1 100644 --- a/tests/component/lib/Export/BatchProcessTest.php +++ b/tests/component/lib/Export/BatchProcessTest.php @@ -50,7 +50,7 @@ public static function setUpBeforeClass() /** */ - private function getEmails() + private static function getEmails() { // TODO return []; @@ -58,7 +58,7 @@ private function getEmails() /** */ - private function getExportFiles() + private static function getExportFiles() { // TODO return []; @@ -72,23 +72,24 @@ public function testDryRun() $batchProcessor = new BatchProcessor(); $batchProcessor->setDryRun(true); - $emails = $this->getEmails(); - $files = $this->getExportFiles(); - $submittedRequests = $this->queryHandler->listSubmittedRecords(); - $expiringRequests = $this->queryHandler->listExpiringRecords(); + // Capture state before processing requests. + $emails = self::getEmails(); + $files = self::getExportFiles(); + $submittedRequests = self::$queryHandler->listSubmittedRecords(); + $expiringRequests = self::$queryHandler->listExpiringRecords(); $batchProcessor->processRequests(); - $this->assertEquals($emails, $this->getEmails(), 'No new emails'); - $this->assertEquals($files, $this->getFiles(), 'No new export files'); + $this->assertEquals($emails, self::getEmails(), 'No new emails'); + $this->assertEquals($files, self::getFiles(), 'No new export files'); $this->assertEquals( $submittedRequests, - $this->queryHandler->listSubmittedRecords(), + self::$queryHandler->listSubmittedRecords(), 'No submitted requests changed' ); $this->assertEquals( $expiringRequests, - $this->queryHandler->listExpiringRecords(), + self::$queryHandler->listExpiringRecords(), 'No expiring requests changed' ); From 5d9f3495c9a05e1f953d38bcffaa612d3f0fc693 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 8 Jul 2019 09:38:34 -0400 Subject: [PATCH 135/217] Remove unnecessary namespace --- background_scripts/batch_export_manager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/background_scripts/batch_export_manager.php b/background_scripts/batch_export_manager.php index f8dbfe79fc..0feee75745 100755 --- a/background_scripts/batch_export_manager.php +++ b/background_scripts/batch_export_manager.php @@ -64,7 +64,7 @@ 'mail' => false, 'consoleLogLevel' => $logLevel ); - $logger = CCR\Log('batch-export', $logConf); + $logger = Log('batch-export', $logConf); $logger->info('Command: ' . implode(' ', array_map('escapeshellarg', $argv))); // NOTE: "process_start_time" is needed for the log summary. $logger->notice(['message' => 'batch_export_manager start', 'process_start_time' => date('Y-m-d H:i:s')]); From 1cfa4f6b926c381215fd994b329db8473698af25 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 8 Jul 2019 09:42:05 -0400 Subject: [PATCH 136/217] Fix typo --- tests/component/lib/Export/BatchProcessTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/component/lib/Export/BatchProcessTest.php b/tests/component/lib/Export/BatchProcessTest.php index 0451465bc1..d49460308a 100644 --- a/tests/component/lib/Export/BatchProcessTest.php +++ b/tests/component/lib/Export/BatchProcessTest.php @@ -81,7 +81,7 @@ public function testDryRun() $batchProcessor->processRequests(); $this->assertEquals($emails, self::getEmails(), 'No new emails'); - $this->assertEquals($files, self::getFiles(), 'No new export files'); + $this->assertEquals($files, self::getExportFiles(), 'No new export files'); $this->assertEquals( $submittedRequests, self::$queryHandler->listSubmittedRecords(), From 1f46815b90ba6d094352352cc91e00fc90e7926d Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 8 Jul 2019 18:15:24 -0400 Subject: [PATCH 137/217] Implement test helper functions --- .../component/lib/Export/BatchProcessTest.php | 76 +++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/tests/component/lib/Export/BatchProcessTest.php b/tests/component/lib/Export/BatchProcessTest.php index d49460308a..ee8ffe2a1c 100644 --- a/tests/component/lib/Export/BatchProcessTest.php +++ b/tests/component/lib/Export/BatchProcessTest.php @@ -28,7 +28,7 @@ class BatchProcessTest extends BaseTest * The file containing email data. * @var string */ - private static $mailSpool = '/var/mail/root'; + private static $mailbox = '/var/mail/root'; /** * The directory containing batch export zip files. @@ -49,19 +49,85 @@ public static function setUpBeforeClass() } /** + * Get emails in root mailbox. + * + * @return array[] */ private static function getEmails() { - // TODO - return []; + if (!file_exists(self::$mailbox)) { + return []; + } + + $emails = []; + $currentEmail = []; + $body = ''; + $lines = file(self::$mailbox); + while (count($lines) > 0) { + $line = array_shift($lines); + // Check for a new message. + if (substr($line, 0, 5) === 'From ') { + // Store previous email. + if (!empty($currentEmail)) { + // Remove trailing newline. + $body = substr($body, 0, -1); + // Undo "From " escaping. + $body = preg_replace("/\n>([>]*From )/", "\n$1", $body); + $currentEmail['body'] = $body; + $emails[] = $currentEmail; + } + $currentEmail = []; + $headers = []; + // Parse headers. + while (count($lines) > 0 && $lines[0] != "\n") { + $line = substr(array_shift($lines), 0, -1); + list($key, $value) = explode(': ', $line, 2); + while ($lines[0][0] === "\t") { + $value .= ' ' . substr(substr(array_shift($lines), 0, -1), 1); + } + $headers[$key] = $value; + } + $currentEmail['headers'] = $headers; + $body = ''; + // Skip blank line before body. + array_shift($lines); + } else { + $body .= $line; + } + } + // Store last email. + if (!empty($currentEmail)) { + // Remove trailing newline. + $body = substr($body, 0, -1); + // Undo "From " escaping. + $body = preg_replace("/\n>([>]*From )/", "\n$1", $body); + $currentEmail['body'] = $body; + $emails[] = $currentEmail; + } + return $emails; } /** + * Get information about all the files in the export directory. + * + * @return array[] */ private static function getExportFiles() { - // TODO - return []; + $dir = self::$exportDirectory; + + return array_map( + function ($file) use ($dir) { + return stat($dir . DIRECTORY_SEPARATOR . $file); + }, + // Filter out files starting with ".". + array_filter( + scandir($dir), + function ($file) { + return $file[0] !== '.'; + } + ) + ); } /** From c560c473799274d1923a00bad8a02df588dee457 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 9 Jul 2019 07:19:26 -0400 Subject: [PATCH 138/217] Add batch export directory --- open_xdmod/modules/xdmod/xdmod.spec.in | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/open_xdmod/modules/xdmod/xdmod.spec.in b/open_xdmod/modules/xdmod/xdmod.spec.in index ceae625bf2..ff8a3914cd 100644 --- a/open_xdmod/modules/xdmod/xdmod.spec.in +++ b/open_xdmod/modules/xdmod/xdmod.spec.in @@ -27,6 +27,8 @@ from resource managers in high-performance computing environments. XDMoD presents resource utilization over set time periods and provides detailed interactive charts, graphs, and tables. +%define xdmod_export_dir %{_var}/spool/%{name}/export + %prep %setup -q -n %{name}-%{version}__PRERELEASE__ @@ -49,6 +51,7 @@ DESTDIR=$RPM_BUILD_ROOT ./install \ --httpdconfdir=%{_sysconfdir}/httpd/conf.d \ --logrotatedconfdir=%{_sysconfdir}/logrotate.d \ --crondconfdir=%{_sysconfdir}/cron.d +mkdir -p $RPM_BUILD_ROOT%{xdmod_export_dir} %post # Create log files that need to be writable by both Apache and Open XDMoD scripts. @@ -87,9 +90,12 @@ rm -rf $RPM_BUILD_ROOT %config(noreplace) %{_sysconfdir}/cron.d/%{name} %config(noreplace) %{_datadir}/%{name}/html/robots.txt +%dir %attr(0570,apache,xdmod) %{xdmod_export_dir} + %changelog * Sun Jul 28 2019 XDMoD - Remove `node_modules` management from `%pre` + - Add data warehouse batch export directory `/var/spool/xdmod/export` * Mon May 06 2019 XDMoD 8.1.2-1.0 - Update `%pre` with steps for `node_modules` on 8.1.2 release * Thu May 02 2019 XDMoD 8.1.1-1.0 From bd8622edba019e4cfd9747cf5ea2fc2fcabc0d2b Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 9 Jul 2019 10:57:44 -0400 Subject: [PATCH 139/217] Add data warehouse config migration --- .../Migration/Version812To850/ConfigFilesMigration.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php index e3e8891fa5..c6dad0cc19 100644 --- a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php +++ b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php @@ -55,6 +55,8 @@ public function execute() ); $this->writePortalSettingsFile(array( + 'data_warehouse_export_export_directory' => '/var/spool/xdmod/export', + 'data_warehouse_export_retention_duration_days' => 30, 'features_novice_user' => $novice_user )); } From 2ab79c9b3f39f7740a99bbaebeec08092b8adef1 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 07:43:20 -0400 Subject: [PATCH 140/217] Add data warehouse export file writing classes --- .../Export/FileWriter/CsvFileWriter.php | 23 +++++ .../Export/FileWriter/JsonFileWriter.php | 84 +++++++++++++++++++ .../Export/FileWriter/aFileWriter.php | 60 +++++++++++++ .../Export/FileWriter/iFileWriter.php | 30 +++++++ 4 files changed, 197 insertions(+) create mode 100644 classes/DataWarehouse/Export/FileWriter/CsvFileWriter.php create mode 100644 classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php create mode 100644 classes/DataWarehouse/Export/FileWriter/aFileWriter.php create mode 100644 classes/DataWarehouse/Export/FileWriter/iFileWriter.php diff --git a/classes/DataWarehouse/Export/FileWriter/CsvFileWriter.php b/classes/DataWarehouse/Export/FileWriter/CsvFileWriter.php new file mode 100644 index 0000000000..ef9a1a5088 --- /dev/null +++ b/classes/DataWarehouse/Export/FileWriter/CsvFileWriter.php @@ -0,0 +1,23 @@ +fh, $record) === false) { + $this->logAndThrowException( + sprintf('Failed to write to file "%s"', $this->file) + ); + } + } +} diff --git a/classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php b/classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php new file mode 100644 index 0000000000..e2821ba08e --- /dev/null +++ b/classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php @@ -0,0 +1,84 @@ +fh, '[') === false) { + $this->logAndThrowException( + sprintf('Failed to write to file "%s"', $this->file) + ); + } + } + + /** + * Write the closing bracket and close the file. + */ + public function close() + { + if (fwrite($this->fh, "\n]") === false) { + $this->logAndThrowException( + sprintf('Failed to write to file "%s"', $this->file) + ); + } + + parent::close(); + } + + /** + * Write a record to file formatted as JSON. + * + * @param array $record + */ + public function writeRecord(array $record) + { + $json = json_encode($record, JSON_UNESCAPED_SLASHES); + + if ($json === false) { + $this->logAndThrowException(sprintf( + 'Failed to encode data as JSON: %s', + Json::getLastErrorMessage() + )); + } + + // If any records have already been written then a comma is needed + // before the next record. + $separator = ($this->recordWritten ? ',' : '') . "\n "; + + if (fwrite($this->fh, $separator . $json) === false) { + $this->logAndThrowException( + sprintf('Failed to write to file "%s"', $this->file) + ); + } + + $recordWritten = true; + } +} diff --git a/classes/DataWarehouse/Export/FileWriter/aFileWriter.php b/classes/DataWarehouse/Export/FileWriter/aFileWriter.php new file mode 100644 index 0000000000..8e88fa04b2 --- /dev/null +++ b/classes/DataWarehouse/Export/FileWriter/aFileWriter.php @@ -0,0 +1,60 @@ +file = $file; + $this->fh = fopen($this->file, 'w'); + + if ($this->fh === false) { + $this->logAndThrowException( + sprintf('Failed to open file "%s"', $this->file) + ); + } + } + + /** + * Close file. + */ + public function close() + { + if (fclose($this->fh) === false) { + $this->logAndThrowException( + sprintf('Failed to close file "%s"', $this->file) + ); + } + } + + /** + * + * @param + */ + //abstract public function writeRecord(array $record); +} diff --git a/classes/DataWarehouse/Export/FileWriter/iFileWriter.php b/classes/DataWarehouse/Export/FileWriter/iFileWriter.php new file mode 100644 index 0000000000..6ce129c770 --- /dev/null +++ b/classes/DataWarehouse/Export/FileWriter/iFileWriter.php @@ -0,0 +1,30 @@ + Date: Wed, 10 Jul 2019 07:43:47 -0400 Subject: [PATCH 141/217] Add expiration date to email --- classes/DataWarehouse/Export/BatchProcessor.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 6e16cdab45..083b9270ee 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -354,6 +354,9 @@ private function sendExportSuccessEmail(XDUser $user, array $request) $this->logger->info('Sending success email'); + // Remove time from expiration date time. + list($expirationDate) = explode(' ', $request['export_expires_datetime']); + MailWrapper::sendTemplate( 'batch_export_success', [ @@ -362,7 +365,7 @@ private function sendExportSuccessEmail(XDUser $user, array $request) 'first_name' => $user->getFirstName(), 'last_name' => $user->getLastName(), 'current_date' => date('Y-m-d'), - 'expiration_date' => $request[''], // TODO + 'expiration_date' => $expirationDate, 'download_url' => '', // TODO 'maintainer_signature' => MailWrapper::getMaintainerSignature() ] From 827ca59cab4eb374785959aba69ccf755be2cf3f Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 07:46:01 -0400 Subject: [PATCH 142/217] Refactor archive error handling --- .../DataWarehouse/Export/BatchProcessor.php | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 083b9270ee..0e86bb4583 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -277,30 +277,30 @@ private function createZipFile($dataFile, $zipFile) 'zip_file' => $zipFile ]); - $zip = new ZipArchive(); - $zipOpenCode = $zip->open($zipFile, ZipArchive::CREATE); + try { + $zip = new ZipArchive(); + $zipOpenCode = $zip->open($zipFile, ZipArchive::CREATE); - if ($zipOpenCode !== true) { - $this->logAndThrowException( - sprintf( - 'Failed to create zip file "%s", error code "%s"', + if ($zipOpenCode !== true) { + $this->logAndThrowException(sprintf( + 'Failed to open zip file "%s", error code "%s"', $zipFile, $zipOpenCode - ) - ); - } + )); + } - if ($zip->addFile($dataFile, basename($dataFile)) === false) { - $this->logAndThrowException( - sprintf( + if ($zip->addFile($dataFile, basename($dataFile)) === false) { + $this->logAndThrowException(sprintf( 'Failed to add file "%s" to zip file "%s"', $dataFile, $zipFile - ) - ); - } + )); + } - $zip->close(); + $zip->close(); + } catch (Exception $e) { + throw new Exception('Failed to create zip file', 0, $e); + } } /** From 92b5b9a324234c29bb371d0ad0b50c57ec11c366 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 08:59:25 -0400 Subject: [PATCH 143/217] More file writing --- .../Export/FileWriter/FileWriterFactory.php | 44 +++++++++++++++++++ .../Export/FileWriter/NullFileWriter.php | 40 +++++++++++++++++ .../Export/FileWriter/aFileWriter.php | 8 +++- 3 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php create mode 100644 classes/DataWarehouse/Export/FileWriter/NullFileWriter.php diff --git a/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php b/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php new file mode 100644 index 0000000000..d7f8eaac93 --- /dev/null +++ b/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php @@ -0,0 +1,44 @@ +logger->debug([ + 'message' => 'Creating new file writer', + 'format' => $format, + 'file' => $file + ]); + + switch ($format) { + case 'csv': + return new CsvFileWriter($file, $this->logger); + break; + case 'json': + return new JsonFileWriter($file, $this->logger); + break; + case 'null': + return new NullFileWriter($file, $this->logger); + break; + default: + $this->logAndThrowException( + sprintf('Unsupported format "%s"', $format) + ); + break; + } + } +} diff --git a/classes/DataWarehouse/Export/FileWriter/NullFileWriter.php b/classes/DataWarehouse/Export/FileWriter/NullFileWriter.php new file mode 100644 index 0000000000..1b54889eff --- /dev/null +++ b/classes/DataWarehouse/Export/FileWriter/NullFileWriter.php @@ -0,0 +1,40 @@ +setLogger($logger); + } + + /** + * Don't close anything. + */ + public function close() + { + } + + /** + * Don't write anything. + * + * @param array $record + */ + public function writeRecord(array $record) + { + } +} diff --git a/classes/DataWarehouse/Export/FileWriter/aFileWriter.php b/classes/DataWarehouse/Export/FileWriter/aFileWriter.php index 8e88fa04b2..4e6207d312 100644 --- a/classes/DataWarehouse/Export/FileWriter/aFileWriter.php +++ b/classes/DataWarehouse/Export/FileWriter/aFileWriter.php @@ -53,8 +53,12 @@ public function close() } /** + * Return string representation of class instance. * - * @param + * @returns string */ - //abstract public function writeRecord(array $record); + public function __toString() + { + return sprintf('%s(file: "%s")', get_class($this), $this->file); + } } From d969c4073c84d171c684cbcd57a52240dcebc6ee Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 09:01:47 -0400 Subject: [PATCH 144/217] Use file writers --- .../DataWarehouse/Export/BatchProcessor.php | 67 +++++++++++++------ 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 0e86bb4583..4d75bd8e2d 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -9,6 +9,8 @@ use CCR\Loggable; use CCR\MailWrapper; use DataWarehouse\Data\RawDataset; +use DataWarehouse\Export\FileWriter\iFileWriter; +use DataWarehouse\Export\FileWriter\FileWriterFactory; use Exception; use XDUser; use ZipArchive; @@ -40,6 +42,11 @@ class BatchProcessor extends Loggable */ private $exportDirectory; + /** + * @var DataWarehouse\Export\FileWriter\FileWriterFactory + */ + private $fileWriterFactory; + /** * Construct a new batch processor. */ @@ -52,6 +59,7 @@ public function __construct() 'data_warehouse_export', 'export_directory' ); + $this->fileWriterFactory = new FileWriterFactory($this->logger); } /** @@ -117,7 +125,12 @@ private function processSubmittedRequest(array $request) $this->dbh->beginTransaction(); $this->queryHandler->submittedToAvailable($request['id']); $dataSet = $this->getDataSet($request); - $dataFile = $this->writeDataSetToFile($dataSet, $request['format']); + $dataFile = tempnam(sys_get_temp_dir(), 'batch-export-'); + $fileWriter = $this->fileWriterFactory->getFileWriter( + ($this->dryRun ? 'null' : $request['format']), + $dataFile + ); + $this->writeDataSetToFile($dataSet, $fileWriter); $zipFile = $this->getExportZipFilePath($request['id']); $this->createZipFile($dataFile, $zipFile); @@ -228,33 +241,43 @@ private function getDataSet(array $request, XDUser $user) * Write data set to file. * * @param \DataWarehouse\Data\RawDataset $dataSet - * @param string $format + * @param \DataWarehouse\Export\FileWriter\iFileWriter $fileWriter */ - private function writeDataSetToFile(RawDataset $dataSet, $format) - { - if ($this->dryRun) { - $this->logger->notice('dry run: Not writing data to file'); - return; - } + private function writeDataSetToFile( + RawDataset $dataSet, + iFileWriter $fileWriter + ) { + try { + $this->logger->info([ + 'message' => 'Writing data to file', + 'file_writer' => $fileWriter + ]); - $tmpFile = tempnam(sys_get_temp_dir(), 'batch-export-'); + $header = []; - $this->logger->info([ - 'message' => 'Writing data to file', - 'tmp_file' => $tmpFile - ]); + // The `export` function returns the first result along with the + // necessary metadata. + foreach ($dataSet->export() as $datum) { + $header[] = $datum['key']; + } - // The `export` function returns the first result along with the - // necessary metadata. - foreach ($dataSet->export() as $datum) { - // TODO - } + $fileWriter->writeRecord($header); + + foreach ($dataSet->getResults() as $result) { + $row = array_map( + function ($datum) { + return $datum['value']; + }, + $result + ); + fputcsv($fh, $row); + $fileWriter->writeRecord($row); + } - foreach ($dataSet->getResults() as $result) { - // TODO + $fileWriter->close(); + } catch (Exception $e) { + throw new Exception('Failed to write data set to file', 0, $e); } - - return $tmpFile; } /** From b3dfc6c648c509de1a57576c3fc80e454e75aa98 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 09:07:19 -0400 Subject: [PATCH 145/217] Add newline to end of help text --- background_scripts/batch_export_manager.php | 1 + 1 file changed, 1 insertion(+) diff --git a/background_scripts/batch_export_manager.php b/background_scripts/batch_export_manager.php index 0feee75745..7eba848e4e 100755 --- a/background_scripts/batch_export_manager.php +++ b/background_scripts/batch_export_manager.php @@ -107,5 +107,6 @@ function displayHelpText() --dry-run Perform all the processing steps, but don't generate or remove any files, send any emails, or change the status of any export requests. + EOMSG; } From 8a81fddbc0f33f03d42fe4eef00acbcf7f3ca7f0 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 09:09:38 -0400 Subject: [PATCH 146/217] Add missing function call --- background_scripts/batch_export_manager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/background_scripts/batch_export_manager.php b/background_scripts/batch_export_manager.php index 7eba848e4e..ad2b387be7 100755 --- a/background_scripts/batch_export_manager.php +++ b/background_scripts/batch_export_manager.php @@ -64,7 +64,7 @@ 'mail' => false, 'consoleLogLevel' => $logLevel ); - $logger = Log('batch-export', $logConf); + $logger = Log::factory('batch-export', $logConf); $logger->info('Command: ' . implode(' ', array_map('escapeshellarg', $argv))); // NOTE: "process_start_time" is needed for the log summary. $logger->notice(['message' => 'batch_export_manager start', 'process_start_time' => date('Y-m-d H:i:s')]); From 188e0589014c56d2f68a85823ad9d78029af6101 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 09:26:07 -0400 Subject: [PATCH 147/217] Order "use" statements --- classes/DataWarehouse/Export/BatchProcessor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 4d75bd8e2d..c90b747487 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -9,8 +9,8 @@ use CCR\Loggable; use CCR\MailWrapper; use DataWarehouse\Data\RawDataset; -use DataWarehouse\Export\FileWriter\iFileWriter; use DataWarehouse\Export\FileWriter\FileWriterFactory; +use DataWarehouse\Export\FileWriter\iFileWriter; use Exception; use XDUser; use ZipArchive; From 68a0f096b4a3a5a93f88cd8cf1565bf415adf2a3 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 09:26:24 -0400 Subject: [PATCH 148/217] Add missing argument --- classes/DataWarehouse/Export/BatchProcessor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index c90b747487..9db8d80bfa 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -124,7 +124,7 @@ private function processSubmittedRequest(array $request) try { $this->dbh->beginTransaction(); $this->queryHandler->submittedToAvailable($request['id']); - $dataSet = $this->getDataSet($request); + $dataSet = $this->getDataSet($request, $user); $dataFile = tempnam(sys_get_temp_dir(), 'batch-export-'); $fileWriter = $this->fileWriterFactory->getFileWriter( ($this->dryRun ? 'null' : $request['format']), From 33ad92cac5d5732838f08afb3dcb76addc50191d Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 09:26:35 -0400 Subject: [PATCH 149/217] Add missing database update for failures --- classes/DataWarehouse/Export/BatchProcessor.php | 1 + 1 file changed, 1 insertion(+) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 9db8d80bfa..addc313a30 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -144,6 +144,7 @@ private function processSubmittedRequest(array $request) 'message' => 'Failed to export data: ' . $e->getMessage(), 'stacktrace' => $e->getTraceAsString() ]); + $this->queryHandler->submittedToFailed($request['id']); $this->sendExportFailureEmail($user, $request, $e); } } From 9d4fff9b3d6725c0955aed590101b7b4a7e4cb0e Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 09:38:42 -0400 Subject: [PATCH 150/217] Add missing class property --- classes/DataWarehouse/Export/BatchProcessor.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index addc313a30..14e5b47822 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -36,6 +36,12 @@ class BatchProcessor extends Loggable */ private $queryHandler; + /** + * Data warehouse export realm manager. + * @var \DataWarehouse\Export\RealmManager + */ + private $realmManager; + /** * Path of directory containing export zip files. * @var string @@ -55,6 +61,7 @@ public function __construct() parent::__construct(); $this->dbh = DB::factory('database'); $this->queryHandler = new QueryHandler(); + $this->realmManager = new RealmManager(); $this->exportDirectory = xd_utilities\getConfiguration( 'data_warehouse_export', 'export_directory' From 05309ce8581c1e0814c5e7cde848d65f58a27301 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 09:41:51 -0400 Subject: [PATCH 151/217] Fix typos --- classes/DataWarehouse/Query/Jobs/JobDataset.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/classes/DataWarehouse/Query/Jobs/JobDataset.php b/classes/DataWarehouse/Query/Jobs/JobDataset.php index 08fa4a0bfa..f07ff33d2e 100644 --- a/classes/DataWarehouse/Query/Jobs/JobDataset.php +++ b/classes/DataWarehouse/Query/Jobs/JobDataset.php @@ -82,8 +82,8 @@ public function __construct( throw new \Exception('invalid "end_date" query parameter'); } - $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_date_ts'), ">=", $startDateTs)); - $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_date_ts'), "<=", $endDateTs)); + $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_time_ts'), ">=", $startDateTs)); + $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_time_ts'), "<=", $endDateTs)); } else { throw new \Exception('invalid query parameters'); } From d3967300aa2af92109009da77e6a0327c2e6c30e Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 10:03:19 -0400 Subject: [PATCH 152/217] Add/change exception logging --- classes/DataWarehouse/Export/BatchProcessor.php | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 14e5b47822..f70e682279 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -241,6 +241,10 @@ private function getDataSet(array $request, XDUser $user) return $dataSet; } catch (Exception $e) { + $this->logger->err([ + 'message' => $e->getMessage(), + 'stacktrace' => $e->getTraceAsString() + ]); throw new Exception('Failed to execute batch export query', 0, $e); } } @@ -284,6 +288,10 @@ function ($datum) { $fileWriter->close(); } catch (Exception $e) { + $this->logger->err([ + 'message' => $e->getMessage(), + 'stacktrace' => $e->getTraceAsString() + ]); throw new Exception('Failed to write data set to file', 0, $e); } } @@ -313,7 +321,7 @@ private function createZipFile($dataFile, $zipFile) $zipOpenCode = $zip->open($zipFile, ZipArchive::CREATE); if ($zipOpenCode !== true) { - $this->logAndThrowException(sprintf( + throw new Exception(sprintf( 'Failed to open zip file "%s", error code "%s"', $zipFile, $zipOpenCode @@ -321,7 +329,7 @@ private function createZipFile($dataFile, $zipFile) } if ($zip->addFile($dataFile, basename($dataFile)) === false) { - $this->logAndThrowException(sprintf( + throw new Exception(sprintf( 'Failed to add file "%s" to zip file "%s"', $dataFile, $zipFile @@ -330,6 +338,10 @@ private function createZipFile($dataFile, $zipFile) $zip->close(); } catch (Exception $e) { + $this->logger->err([ + 'message' => $e->getMessage(), + 'stacktrace' => $e->getTraceAsString() + ]); throw new Exception('Failed to create zip file', 0, $e); } } From 307c5c6a219e386f15ae91e3eae0eec56f8f5b31 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 10:03:57 -0400 Subject: [PATCH 153/217] Change fields selected in some queries --- classes/DataWarehouse/Export/QueryHandler.php | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index fdd94fb96e..b98f91ab4c 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -109,15 +109,19 @@ public function createRequestRecord( public function getRequestRecord($id) { $sql = 'SELECT id, + user_id realm, start_date, end_date, + export_file_format, + requested_datetime export_succeeded, + export_created_datetime, + downloaded_datetime, export_expired, export_expires_datetime, - export_created_datetime, - export_file_format, - requested_datetime + export_expired, + last_modified FROM batch_export_requests WHERE id = :id'; return $this->dbh->query($sql, ['id' => $id]); @@ -195,7 +199,7 @@ public function countSubmittedRecords() */ public function listSubmittedRecords() { - $sql = "SELECT id, realm, start_date, end_date, export_file_format, requested_datetime + $sql = "SELECT id, user_id, realm, start_date, end_date, export_file_format, requested_datetime FROM batch_export_requests " . $this->whereSubmitted . ' ORDER BY requested_datetime, id'; return $this->dbh->query($sql); } @@ -208,16 +212,17 @@ public function listSubmittedRecords() public function listExpiringRecords() { $sql = 'SELECT id, + user_id, realm, start_date, end_date, + export_file_format, + requested_datetime, export_succeeded, - export_expired, - export_expires_datetime, export_created_datetime, - export_file_format, - requested_datetime - FROM batch_export_requests ' . $this->whereAvailable . ' AND export_expires_datetime > NOW() + export_expires_datetime + FROM batch_export_requests + ' . $this->whereAvailable . ' AND export_expires_datetime > NOW() ORDER BY requested_datetime, id'; return $this->dbh->query($sql); } From b4674c8bed853d6299f932b0c1c15ffb22d2aa26 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 10:07:32 -0400 Subject: [PATCH 154/217] Remove old code --- classes/DataWarehouse/Export/BatchProcessor.php | 1 - 1 file changed, 1 deletion(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index f70e682279..f0fcd8d41c 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -282,7 +282,6 @@ function ($datum) { }, $result ); - fputcsv($fh, $row); $fileWriter->writeRecord($row); } From b89e0f0d4d48cb7236f2277492df6b172f417ecd Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 10:26:26 -0400 Subject: [PATCH 155/217] Fix record format --- classes/DataWarehouse/Export/BatchProcessor.php | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index f0fcd8d41c..fbd7f7e7c6 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -276,13 +276,7 @@ private function writeDataSetToFile( $fileWriter->writeRecord($header); foreach ($dataSet->getResults() as $result) { - $row = array_map( - function ($datum) { - return $datum['value']; - }, - $result - ); - $fileWriter->writeRecord($row); + $fileWriter->writeRecord(array_values($result)); } $fileWriter->close(); From f79783cf84f829cf13a72142b4d8e95ea9826a4b Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 10:26:41 -0400 Subject: [PATCH 156/217] Refactor loggers --- background_scripts/batch_export_manager.php | 3 +-- .../DataWarehouse/Export/BatchProcessor.php | 21 ++++++++++++++++--- .../Export/FileWriter/JsonFileWriter.php | 4 ++-- .../Export/FileWriter/NullFileWriter.php | 4 ++-- .../Export/FileWriter/aFileWriter.php | 4 ++-- .../Export/FileWriter/iFileWriter.php | 5 ++++- 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/background_scripts/batch_export_manager.php b/background_scripts/batch_export_manager.php index ad2b387be7..5313744e22 100755 --- a/background_scripts/batch_export_manager.php +++ b/background_scripts/batch_export_manager.php @@ -68,9 +68,8 @@ $logger->info('Command: ' . implode(' ', array_map('escapeshellarg', $argv))); // NOTE: "process_start_time" is needed for the log summary. $logger->notice(['message' => 'batch_export_manager start', 'process_start_time' => date('Y-m-d H:i:s')]); - $batchProcessor = new BatchProcessor(); + $batchProcessor = new BatchProcessor($logger); $batchProcessor->setDryRun($dryRun); - $batchProcessor->setLogger($logger); $batchProcessor->processRequests(); // NOTE: "process_end_time" is needed for the log summary. $logger->notice(['message' => 'batch_export_manager end', 'process_end_time' => date('Y-m-d H:i:s')]); diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index fbd7f7e7c6..2888422c76 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -12,6 +12,7 @@ use DataWarehouse\Export\FileWriter\FileWriterFactory; use DataWarehouse\Export\FileWriter\iFileWriter; use Exception; +use Log; use XDUser; use ZipArchive; use xd_utilities; @@ -56,9 +57,10 @@ class BatchProcessor extends Loggable /** * Construct a new batch processor. */ - public function __construct() + public function __construct(Log $logger) { - parent::__construct(); + $this->fileWriterFactory = new FileWriterFactory($this->logger); + parent::__construct($logger); $this->dbh = DB::factory('database'); $this->queryHandler = new QueryHandler(); $this->realmManager = new RealmManager(); @@ -66,7 +68,20 @@ public function __construct() 'data_warehouse_export', 'export_directory' ); - $this->fileWriterFactory = new FileWriterFactory($this->logger); + } + + /** + * Set the logger for this object. + * + * @see \CCR\Loggable::setLogger() + * @param \Log $logger A logger class or NULL to use the null logger + * @return \DataWarehouse\Export\BatchProcessor This object for method chaining. + */ + public function setLogger(Log $logger = null) + { + parent::setLogger($logger); + $this->fileWriterFactory->setLogger($logger); + return $this; } /** diff --git a/classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php b/classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php index e2821ba08e..6e0500a85e 100644 --- a/classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php +++ b/classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php @@ -3,7 +3,7 @@ namespace DataWarehouse\Export\FileWriter; use CCR\Json; -use CCR\Log; +use Log; /** * Write data warehouse batch export to file in JSON format. @@ -26,7 +26,7 @@ class JsonFileWriter extends aFileWriter * Open the file and write the opening bracket. * * @param string $file - * @param \CCR\Log $logger + * @param \Log $logger */ public function __construct($file, Log $logger) { diff --git a/classes/DataWarehouse/Export/FileWriter/NullFileWriter.php b/classes/DataWarehouse/Export/FileWriter/NullFileWriter.php index 1b54889eff..6326e64170 100644 --- a/classes/DataWarehouse/Export/FileWriter/NullFileWriter.php +++ b/classes/DataWarehouse/Export/FileWriter/NullFileWriter.php @@ -2,7 +2,7 @@ namespace DataWarehouse\Export\FileWriter; -use CCR\Log; +use Log; /** * File writer that doesn't actually write to any files. @@ -15,7 +15,7 @@ class NullFileWriter extends aFileWriter * Don't open the file or do anything. * * @param string $file - * @param \CCR\Log $logger + * @param \Log $logger */ public function __construct($file, Log $logger) { diff --git a/classes/DataWarehouse/Export/FileWriter/aFileWriter.php b/classes/DataWarehouse/Export/FileWriter/aFileWriter.php index 4e6207d312..cce8e52021 100644 --- a/classes/DataWarehouse/Export/FileWriter/aFileWriter.php +++ b/classes/DataWarehouse/Export/FileWriter/aFileWriter.php @@ -3,7 +3,7 @@ namespace DataWarehouse\Export\FileWriter; use CCR\Loggable; -use CCR\Log; +use Log; /** * Abstract class for writing data warehouse batch export data. @@ -24,7 +24,7 @@ abstract class aFileWriter extends Loggable implements iFileWriter * Open the file for writing. * * @param string $file - * @param \CCR\Log $logger + * @param \Log $logger */ public function __construct($file, Log $logger) { diff --git a/classes/DataWarehouse/Export/FileWriter/iFileWriter.php b/classes/DataWarehouse/Export/FileWriter/iFileWriter.php index 6ce129c770..1e385c9176 100644 --- a/classes/DataWarehouse/Export/FileWriter/iFileWriter.php +++ b/classes/DataWarehouse/Export/FileWriter/iFileWriter.php @@ -2,6 +2,8 @@ namespace DataWarehouse\Export\FileWriter; +use Log; + /** * Interface for writing data warehouse batch export data to a file. */ @@ -11,8 +13,9 @@ interface iFileWriter * Open the file for writing. * * @param string $file + * @param \Log $logger */ - public function __construct($file); + public function __construct($file, Log $logger); /** * Close the file being written to. From fcf8dbe531a2ef06c941c7b1f88ba13bfa2d0b68 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 10:29:31 -0400 Subject: [PATCH 157/217] Don't update database records during dry run --- classes/DataWarehouse/Export/BatchProcessor.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 2888422c76..af44cf544c 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -145,7 +145,9 @@ private function processSubmittedRequest(array $request) try { $this->dbh->beginTransaction(); - $this->queryHandler->submittedToAvailable($request['id']); + if (!$this->dryRun) { + $this->queryHandler->submittedToAvailable($request['id']); + } $dataSet = $this->getDataSet($request, $user); $dataFile = tempnam(sys_get_temp_dir(), 'batch-export-'); $fileWriter = $this->fileWriterFactory->getFileWriter( @@ -166,7 +168,9 @@ private function processSubmittedRequest(array $request) 'message' => 'Failed to export data: ' . $e->getMessage(), 'stacktrace' => $e->getTraceAsString() ]); - $this->queryHandler->submittedToFailed($request['id']); + if (!$this->dryRun) { + $this->queryHandler->submittedToFailed($request['id']); + } $this->sendExportFailureEmail($user, $request, $e); } } @@ -199,7 +203,9 @@ private function processExpiringRequest(array $request) try { $this->dbh->beginTransaction(); - $this->queryHandler->availableToExpired($request['id']); + if (!$this->dryRun) { + $this->queryHandler->availableToExpired($request['id']); + } $this->removeExportFile($request['id']); $this->dbh->commit(); } catch (Exception $e) { From eb6631cf9d76089fa6669be2e6eca939aef2cdf0 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 10:31:06 -0400 Subject: [PATCH 158/217] Fix typo --- classes/DataWarehouse/Export/BatchProcessor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index af44cf544c..4faab6b3d3 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -151,7 +151,7 @@ private function processSubmittedRequest(array $request) $dataSet = $this->getDataSet($request, $user); $dataFile = tempnam(sys_get_temp_dir(), 'batch-export-'); $fileWriter = $this->fileWriterFactory->getFileWriter( - ($this->dryRun ? 'null' : $request['format']), + ($this->dryRun ? 'null' : $request['export_file_format']), $dataFile ); $this->writeDataSetToFile($dataSet, $fileWriter); From 2dc186987f22da60de61c38c8eb302be84e9193c Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 10:38:09 -0400 Subject: [PATCH 159/217] Make format check case insensitive --- classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php b/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php index d7f8eaac93..4e8fde9d57 100644 --- a/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php +++ b/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php @@ -24,7 +24,7 @@ public function getFileWriter($format, $file) 'file' => $file ]); - switch ($format) { + switch (strtolower($format)) { case 'csv': return new CsvFileWriter($file, $this->logger); break; From 14da63a6b632d1e53e63acde1549d3bcb1af9664 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 10:52:59 -0400 Subject: [PATCH 160/217] Change log message --- classes/DataWarehouse/Export/BatchProcessor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 4faab6b3d3..82d8f4ac47 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -183,7 +183,7 @@ private function processSubmittedRequest(array $request) */ private function processExpiringRequests() { - $this->logger->info('Processing expired requests'); + $this->logger->info('Processing expiring requests'); foreach ($this->queryHandler->listExpiringRecords() as $request) { $this->processExpiringRequest($request); } From 76bf8bd82f50ea8dedba33b22a7887281806a44a Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 10:53:12 -0400 Subject: [PATCH 161/217] Fix querying --- classes/DataWarehouse/Export/QueryHandler.php | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/classes/DataWarehouse/Export/QueryHandler.php b/classes/DataWarehouse/Export/QueryHandler.php index b98f91ab4c..1f4dc3f32e 100644 --- a/classes/DataWarehouse/Export/QueryHandler.php +++ b/classes/DataWarehouse/Export/QueryHandler.php @@ -45,16 +45,16 @@ class QueryHandler private $whereSubmitted = "WHERE export_succeeded IS NULL AND export_created_datetime IS NULL AND export_expired = 0 "; /** - * Definition of Expired state. + * Definition of Available state. * @var string */ - private $whereExpired = "WHERE export_succeeded = TRUE AND export_created_datetime IS NOT NULL AND export_expired = 1 "; + private $whereAvailable = "WHERE export_succeeded = 1 AND export_created_datetime IS NOT NULL AND export_expired = 0 "; /** - * Definition of Available state. + * Definition of Expired state. * @var string */ - private $whereAvailable = "WHERE export_succeeded = 1 AND export_created_datetime IS NOT NULL AND export_expired = 0 "; + private $whereExpired = "WHERE export_succeeded = 1 AND export_created_datetime IS NOT NULL AND export_expired = 1 "; /** * Definition of Failed state. @@ -118,13 +118,13 @@ public function getRequestRecord($id) export_succeeded, export_created_datetime, downloaded_datetime, - export_expired, export_expires_datetime, export_expired, last_modified FROM batch_export_requests WHERE id = :id'; - return $this->dbh->query($sql, ['id' => $id]); + list($record) = $this->dbh->query($sql, ['id' => $id]); + return $record; } /** @@ -222,7 +222,9 @@ public function listExpiringRecords() export_created_datetime, export_expires_datetime FROM batch_export_requests - ' . $this->whereAvailable . ' AND export_expires_datetime > NOW() + ' . $this->whereAvailable . ' + AND export_expires_datetime IS NOT NULL + AND export_expires_datetime < NOW() ORDER BY requested_datetime, id'; return $this->dbh->query($sql); } From 9a05cb1abbd39356d41a679d866d781b62542433 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 10:54:48 -0400 Subject: [PATCH 162/217] Change file naming --- .../Rest/Controllers/WarehouseExportControllerProvider.php | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 9b43bb8146..29d5fd548a 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -222,12 +222,7 @@ function ($request) use ($id) { throw new NotFoundHttpException('Export directory is not configured'); } - $file = sprintf( - '%s/%s.%s.zip', - $exportDir, - $id, - strtolower($request['export_file_format']) - ); + $file = sprintf('%s/%s.zip', $exportDir, $id); if (!is_file($file)) { throw new NotFoundHttpException('Exported data not found'); From 5bce86af049cbaf71a5c2e0b857475e902a91991 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 12:07:20 -0400 Subject: [PATCH 163/217] Fix typo --- classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php b/classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php index 6e0500a85e..60cbaa09eb 100644 --- a/classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php +++ b/classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php @@ -79,6 +79,6 @@ public function writeRecord(array $record) ); } - $recordWritten = true; + $this->recordWritten = true; } } From 3ccb2353df68529529da73e4aa091e12f8f44148 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 13:35:54 -0400 Subject: [PATCH 164/217] Add default constructor argument --- classes/DataWarehouse/Export/BatchProcessor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 82d8f4ac47..c43a5751ed 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -57,7 +57,7 @@ class BatchProcessor extends Loggable /** * Construct a new batch processor. */ - public function __construct(Log $logger) + public function __construct(Log $logger = null) { $this->fileWriterFactory = new FileWriterFactory($this->logger); parent::__construct($logger); From 662b980f2db2e69c4c352252acbbc0086d5fd2ab Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 13:37:34 -0400 Subject: [PATCH 165/217] Update expected keys --- tests/component/lib/Export/ExportDBTest.php | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/component/lib/Export/ExportDBTest.php b/tests/component/lib/Export/ExportDBTest.php index fb4f51358d..8e34c55df7 100644 --- a/tests/component/lib/Export/ExportDBTest.php +++ b/tests/component/lib/Export/ExportDBTest.php @@ -154,6 +154,7 @@ public function testSubmittedRecordFieldList() // Expect these keys from the associative array $expectedKeys = array( 'id', + 'user_id', 'realm', 'start_date', 'end_date', From 80764e70d5e353f56a533b4dc9efc981f8a25dcf Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 10 Jul 2019 14:04:43 -0400 Subject: [PATCH 166/217] Update assertions, move comments into messages --- tests/component/lib/Export/ExportDBTest.php | 130 ++++++-------------- 1 file changed, 40 insertions(+), 90 deletions(-) diff --git a/tests/component/lib/Export/ExportDBTest.php b/tests/component/lib/Export/ExportDBTest.php index 8e34c55df7..775e87e3dc 100644 --- a/tests/component/lib/Export/ExportDBTest.php +++ b/tests/component/lib/Export/ExportDBTest.php @@ -104,21 +104,18 @@ public function testNewRecordCreation() // Add another new record and verify $requestId2 = $query->createRequestRecord($userId, 'Accounts', '2016-12-01', '2017-01-01', 'JSON'); - $this->assertNotNull($requestId2 ); + $this->assertNotNull($requestId2); // Add another new record and verify $requestId3 = $query->createRequestRecord($userId, 'Jobs', '2014-01-05', '2014-01-26', 'CSV'); - $this->assertNotNull($requestId3 ); + $this->assertNotNull($requestId3); // Determine final count $finalCount = $query->countSubmittedRecords(); $finalCountTest = $this->countSubmittedRecords(); - // Verify final Submitted count. Should have added 3 records. - $this->assertTrue($finalCount-$initialCount==3); - - // Verify test and class methods return same Submitted counts - $this->assertEquals($finalCount, $finalCountTest); + $this->assertEquals(3, $finalCount - $initialCount, 'Verify final Submitted count. Should have added 3 records'); + $this->assertEquals($finalCount, $finalCountTest, 'Verify test and class methods return same Submitted counts'); // debug if (self::$debug) @@ -137,7 +134,7 @@ public function testCountSubmitted() $this->assertEquals($submittedCount, $submittedCountTest); $this->assertNotNull($submittedCount); - $this->assertTrue($submittedCount>=0); + $this->assertGreaterThanOrEqual(0, $submittedCount); // debug if (self::$debug) @@ -165,11 +162,8 @@ public function testSubmittedRecordFieldList() // List all records in Submitted state: $actual = $query->listSubmittedRecords(); - // assert that the expected fields are returned from the query - $this->assertEquals($expectedKeys, array_keys($actual[0])); - - // assert that the expected number of records is returned from the query - $this->assertEquals($this->countSubmittedRecords(), count($actual)); + $this->assertEquals($expectedKeys, array_keys($actual[0]), 'the expected fields are returned from the query'); + $this->assertEquals($this->countSubmittedRecords(), count($actual), 'the expected number of records is returned from the query'); } public function testSubmittedToFailed() @@ -188,13 +182,9 @@ public function testSubmittedToFailed() $submittedCountFinal = $this->countSubmittedRecords(); $failedCountFinal = $this->countFailedRecords(); - // Assert that: - // Exactly one record was transitioned - $this->assertTrue($result==1); - - // There is one fewer Submitted record, one more Failed. - $this->assertTrue($submittedCountFinal + 1 == $submittedCountInitial); - $this->assertTrue($failedCountFinal - 1 == $failedCountInitial); + $this->assertEquals(1, $result, 'Exactly one record was transitioned'); + $this->assertEquals($submittedCountInitial - 1, $submittedCountFinal, 'There is one fewer Submitted record'); + $this->assertEquals($failedCountInitial + 1, $failedCountFinal, 'There is one more Failed record'); // debug if (self::$debug) @@ -219,13 +209,9 @@ public function testSubmittedToExpired() $submittedCountFinal = $this->countSubmittedRecords(); $expiredCountFinal = $this->countExpiredRecords(); - // Assert that: - // Exactly zero records transitioned - $this->assertTrue($result==0); - - // No change in Submitted and Expired state counts occurred - $this->assertTrue($submittedCountFinal == $submittedCountInitial); - $this->assertTrue($expiredCountFinal == $expiredCountInitial); + $this->assertEquals(0, $result, 'Exactly zero records transitioned'); + $this->assertEquals($submittedCountInitial, $submittedCountFinal, 'No change in Submitted counts occurred'); + $this->assertEquals($expiredCountInitial, $expiredCountFinal, 'No change in Expired state counts occurred'); // debug if (self::$debug) @@ -250,13 +236,9 @@ public function testSubmittedToAvailable() $submittedCountFinal = $this->countSubmittedRecords(); $availCountFinal = $this->countAvailableRecords(); - // Assert that: - // Exactly one record was transitioned - $this->assertTrue($result==1); - - // There is one fewer Submitted record, one more Available. - $this->assertTrue($submittedCountFinal + 1 == $submittedCountInitial); - $this->assertTrue($availCountFinal - 1 == $availCountInitial); + $this->assertEquals(1, $result, 'Exactly one record was transitioned'); + $this->assertEquals($submittedCountInitial - 1, $submittedCountFinal, 'There is one fewer Submitted record'); + $this->assertEquals($availCountInitial + 1, $availCountFinal, 'There is one more Available record'); // debug if (self::$debug) @@ -281,13 +263,9 @@ public function testAvailableToFailed() $availCountFinal = $this->countAvailableRecords(); $failCountFinal = $this->countFailedRecords(); - // Assert that: - // Exactly zero records were transitioned - $this->assertTrue($result==0); - - // No change in Available and Failed state counts occurred - $this->assertTrue($availCountFinal == $availCountInitial); - $this->assertTrue($failCountFinal == $failCountInitial); + $this->assertEquals(0, $result, 'Exactly zero records were transitioned'); + $this->assertEquals($availCountInitial, $availCountFinal, 'No change in Available state counts occurred'); + $this->assertEquals($failCountInitial, $failCountFinal, 'No change in Failed state counts occurred'); // debug if (self::$debug) @@ -312,13 +290,9 @@ public function testAvailableToExpired() $availCountFinal = $this->countAvailableRecords(); $expiredCountFinal = $this->countExpiredRecords(); - // Assert that: - // Exactly one record was transitioned - $this->assertTrue($result==1); - - // There is one fewer Available record, one more Expired. - $this->assertTrue($availCountFinal + 1 == $availCountInitial); - $this->assertTrue($expiredCountFinal - 1 == $expiredCountInitial); + $this->assertEquals(1, $result, 'Exactly one record was transitioned'); + $this->assertEquals($availCountInitial - 1, $availCountFinal, 'There is one fewer Available record'); + $this->assertEquals($expiredCountInitial + 1, $expiredCountFinal, 'There is one more Expired record'); // debug if (self::$debug) @@ -343,13 +317,9 @@ public function testExpiredToFailed() $expiredCountFinal = $this->countExpiredRecords(); $failCountFinal = $this->countFailedRecords(); - // Assert that: - // Exactly zero records were transitioned - $this->assertTrue($result==0); - - // No change in Expired and Failed state counts occurred - $this->assertTrue($expiredCountFinal == $expiredCountInitial); - $this->assertTrue($failCountFinal == $failCountInitial); + $this->assertEquals(0, $result, 'Exactly zero records were transitioned'); + $this->assertEquals($expiredCountInitial, $expiredCountFinal, 'No change in Expired state counts occurred'); + $this->assertEquals($failCountInitial, $failCountFinal, 'No change in Failed state counts occurred'); // debug if (self::$debug) @@ -374,13 +344,9 @@ public function testFailedToExpired() $expiredCountFinal = $this->countExpiredRecords(); $failCountFinal = $this->countFailedRecords(); - // Assert that: - // Exactly zero records transitioned - $this->assertTrue($result==0); - - // No change in Expired and Failed state counts occurred - $this->assertTrue($expiredCountFinal == $expiredCountInitial); - $this->assertTrue($failCountFinal == $failCountInitial); + $this->assertEquals(0, $result, 'Exactly zero records transitioned'); + $this->assertEquals($expiredCountInitial, $expiredCountFinal, 'No change in Expired state counts occurred'); + $this->assertEquals($failCountInitial, $failCountFinal, 'No change in Failed state counts occurred'); // debug if (self::$debug) @@ -405,13 +371,9 @@ public function testExpiredToAvailable() $expiredCountFinal = $this->countExpiredRecords(); $availCountFinal = $this->countAvailableRecords(); - // Assert that: - // Exactly zero records transitioned - $this->assertTrue($result==0); - - // No change in Expired and Available state counts occurred - $this->assertTrue($expiredCountFinal == $expiredCountInitial); - $this->assertTrue($availCountFinal == $availCountInitial); + $this->assertEquals(0, $result, 'Exactly zero records transitioned'); + $this->assertEquals($expiredCountInitial, $expiredCountFinal, 'No change in Expired state counts occurred'); + $this->assertEquals($availCountInitial, $availCountFinal, 'No change in Available state counts occurred'); // debug if (self::$debug) @@ -436,13 +398,9 @@ public function testFailedToAvailable() $availCountFinal = $this->countAvailableRecords(); $failCountFinal = $this->countFailedRecords(); - // Assert that: - // Exactly zero records transitioned - $this->assertTrue($result==0); - - // No change in Available and Failed state counts occurred - $this->assertTrue($availCountFinal == $availCountInitial); - $this->assertTrue($failCountFinal == $failCountInitial); + $this->assertEquals(0, $result, 'Exactly zero records transitioned'); + $this->assertEquals($availCountInitial, $availCountFinal, 'No change in Available state counts occurred'); + $this->assertEquals($failCountInitial, $failCountFinal, 'No change in Failed state counts occurred'); // debug if (self::$debug) @@ -474,11 +432,8 @@ public function testUserRecordFieldList() // Requests via this user have been created as part of these tests $actual = $query->listRequestsForUser($userId); - // assert that the expected fields are returned from the query - $this->assertEquals($expectedKeys, array_keys($actual[0])); - - // assert that the expected number of records is returned from the query - $this->assertEquals($this->countUserRequests(), count($actual)); + $this->assertEquals($expectedKeys, array_keys($actual[0]), 'the expected fields are returned from the query'); + $this->assertEquals($this->countUserRequests(), count($actual), 'the expected number of records is returned from the query'); } // Verify field list returned from listUserRequestsByState() @@ -505,11 +460,8 @@ public function testUserRecordReportStates() // Requests via this user have been created as part of these tests $actual = $query->listUserRequestsByState($userId); - // assert that the expected fields are returned from the query - $this->assertEquals($expectedKeys, array_keys($actual[0])); - - // assert that the expected number of records is returned from the query - $this->assertEquals($this->countUserRequests(), count($actual)); + $this->assertEquals($expectedKeys, array_keys($actual[0]), 'the expected fields are returned from the query'); + $this->assertEquals($this->countUserRequests(), count($actual), 'the expected number of records is returned from the query'); } // Verify that user that did not create request cannot delete it @@ -528,8 +480,7 @@ public function testRecordDeleteIncorrectUser() // try to delete the request: $actual = $query->deleteRequest($maxSubmitted, $wrongUserId); - // assert that the delete attempt affected 0 rows - $this->assertEquals($actual, 0); + $this->assertEquals(0, $actual, 'the delete attempt affected 0 rows'); if (self::$debug) { @@ -550,8 +501,7 @@ public function testRecordDeleteCorrectUser() // try to delete the request: $actual = $query->deleteRequest($maxSubmitted, $userId); - // assert that the delete affected 1 row - $this->assertEquals($actual, 1); + $this->assertEquals(1, $actual, 'the delete affected 1 row'); if (self::$debug) { From 7524388adcef7abd782ea00afe92c45e38368203 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 11 Jul 2019 08:24:02 -0400 Subject: [PATCH 167/217] Refactor export file related functions --- .../DataWarehouse/Export/BatchProcessor.php | 178 ++---------- classes/DataWarehouse/Export/FileManager.php | 254 ++++++++++++++++++ .../Export/FileWriter/FileWriterFactory.php | 2 +- .../WarehouseExportControllerProvider.php | 24 +- 4 files changed, 283 insertions(+), 175 deletions(-) create mode 100644 classes/DataWarehouse/Export/FileManager.php diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index c43a5751ed..5192799e20 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -9,12 +9,9 @@ use CCR\Loggable; use CCR\MailWrapper; use DataWarehouse\Data\RawDataset; -use DataWarehouse\Export\FileWriter\FileWriterFactory; -use DataWarehouse\Export\FileWriter\iFileWriter; use Exception; use Log; use XDUser; -use ZipArchive; use xd_utilities; class BatchProcessor extends Loggable @@ -44,43 +41,36 @@ class BatchProcessor extends Loggable private $realmManager; /** - * Path of directory containing export zip files. - * @var string + * Data warehouse export file manager. + * @var \DataWarehouse\Export\FileManager */ - private $exportDirectory; - - /** - * @var DataWarehouse\Export\FileWriter\FileWriterFactory - */ - private $fileWriterFactory; + private $fileManager; /** * Construct a new batch processor. */ public function __construct(Log $logger = null) { - $this->fileWriterFactory = new FileWriterFactory($this->logger); + // Must set properties that are used in `setLogger` before calling the + // parent constructor. + $this->fileManager = new FileManager($logger); parent::__construct($logger); $this->dbh = DB::factory('database'); $this->queryHandler = new QueryHandler(); $this->realmManager = new RealmManager(); - $this->exportDirectory = xd_utilities\getConfiguration( - 'data_warehouse_export', - 'export_directory' - ); } /** * Set the logger for this object. * * @see \CCR\Loggable::setLogger() - * @param \Log $logger A logger class or NULL to use the null logger - * @return \DataWarehouse\Export\BatchProcessor This object for method chaining. + * @param \Log $logger A logger instance or null to use the null logger. + * @return self This object for method chaining. */ public function setLogger(Log $logger = null) { parent::setLogger($logger); - $this->fileWriterFactory->setLogger($logger); + $this->fileManager->setLogger($logger); return $this; } @@ -149,14 +139,9 @@ private function processSubmittedRequest(array $request) $this->queryHandler->submittedToAvailable($request['id']); } $dataSet = $this->getDataSet($request, $user); - $dataFile = tempnam(sys_get_temp_dir(), 'batch-export-'); - $fileWriter = $this->fileWriterFactory->getFileWriter( - ($this->dryRun ? 'null' : $request['export_file_format']), - $dataFile - ); - $this->writeDataSetToFile($dataSet, $fileWriter); - $zipFile = $this->getExportZipFilePath($request['id']); - $this->createZipFile($dataFile, $zipFile); + $format = $this->dryRun ? 'null' : $request['export_file_format']; + $dataFile = $this->fileManager->writeDataSetToFile($dataSet, $format); + $zipFile = $this->fileManager->createZipFile($dataFile, $request); // Query for same record to get expiration date. $request = $this->queryHandler->getRequestRecord($request['id']); @@ -201,12 +186,15 @@ private function processExpiringRequest(array $request) 'batch_export_request.id' => $request['id'] ]); + if ($this->dryRun) { + $this->logger->notice('dry run: Not expiring export file'); + return; + } + try { $this->dbh->beginTransaction(); - if (!$this->dryRun) { - $this->queryHandler->availableToExpired($request['id']); - } - $this->removeExportFile($request['id']); + $this->queryHandler->availableToExpired($request['id']); + $this->fileManager->removeExportFile($request['id']); $this->dbh->commit(); } catch (Exception $e) { $this->dbh->rollback(); @@ -254,6 +242,8 @@ private function getDataSet(array $request, XDUser $user) $dataSet = new RawDataset($query, $user); + $this->logger->debug('Executing query'); + // Data are fetched from the database as a side effect of checking // for results. if ($dataSet->hasResults()) { @@ -270,132 +260,6 @@ private function getDataSet(array $request, XDUser $user) } } - /** - * Write data set to file. - * - * @param \DataWarehouse\Data\RawDataset $dataSet - * @param \DataWarehouse\Export\FileWriter\iFileWriter $fileWriter - */ - private function writeDataSetToFile( - RawDataset $dataSet, - iFileWriter $fileWriter - ) { - try { - $this->logger->info([ - 'message' => 'Writing data to file', - 'file_writer' => $fileWriter - ]); - - $header = []; - - // The `export` function returns the first result along with the - // necessary metadata. - foreach ($dataSet->export() as $datum) { - $header[] = $datum['key']; - } - - $fileWriter->writeRecord($header); - - foreach ($dataSet->getResults() as $result) { - $fileWriter->writeRecord(array_values($result)); - } - - $fileWriter->close(); - } catch (Exception $e) { - $this->logger->err([ - 'message' => $e->getMessage(), - 'stacktrace' => $e->getTraceAsString() - ]); - throw new Exception('Failed to write data set to file', 0, $e); - } - } - - /** - * Create a zip file containing a single file. - * - * @param string $dataFile Absolute path to file that will be put in zip file. - * @param string $zipFile Absolute path to zip file that will be created. - * @throws \Exception - */ - private function createZipFile($dataFile, $zipFile) - { - if ($this->dryRun) { - $this->logger->notice('dry run: Not creating zip file'); - return; - } - - $this->logger->info([ - 'message' => 'Creating zip file', - 'data_file' => $dataFile, - 'zip_file' => $zipFile - ]); - - try { - $zip = new ZipArchive(); - $zipOpenCode = $zip->open($zipFile, ZipArchive::CREATE); - - if ($zipOpenCode !== true) { - throw new Exception(sprintf( - 'Failed to open zip file "%s", error code "%s"', - $zipFile, - $zipOpenCode - )); - } - - if ($zip->addFile($dataFile, basename($dataFile)) === false) { - throw new Exception(sprintf( - 'Failed to add file "%s" to zip file "%s"', - $dataFile, - $zipFile - )); - } - - $zip->close(); - } catch (Exception $e) { - $this->logger->err([ - 'message' => $e->getMessage(), - 'stacktrace' => $e->getTraceAsString() - ]); - throw new Exception('Failed to create zip file', 0, $e); - } - } - - /** - * Remove an export data file. - * - * @param int $id Export request primary key. - */ - private function removeExportFile($id) - { - if ($this->dryRun) { - $this->logger->notice('dry run: Not removing export file'); - return; - } - - $zipFile = $this->getExportZipFilePath($id); - - $this->logger->info([ - 'message' => 'Removing export file', - 'batch_export_request.id' => $id, - 'zip_file' => $zipFile - ]); - - if (!unlink($zipFile)) { - throw new Exception(sprintf('Failed to delete "%s"', $zipFile)); - } - } - - /** - * Get the full path for the export data file. - * - * @param int $id Export request primary key. - * @return string - */ - private function getExportZipFilePath($id) - { - return $this->exportDirectory . DIRECTORY_SEPARATOR . $id . '.zip'; - } - /** * Send email indicating a successful export. * diff --git a/classes/DataWarehouse/Export/FileManager.php b/classes/DataWarehouse/Export/FileManager.php new file mode 100644 index 0000000000..b2d4bda80d --- /dev/null +++ b/classes/DataWarehouse/Export/FileManager.php @@ -0,0 +1,254 @@ +fileWriterFactory = new FileWriterFactory($logger); + parent::__construct($logger); + + try { + $this->exportDir = xd_utilities\getConfiguration( + 'data_warehouse_export', + 'export_directory' + ); + } catch (Exception $e) { + $this->logger->err([ + 'message' => $e->getMessage(), + 'stacktrace' => $e->getTraceAsString() + ]); + throw new Exception('Export directory is not configured', 0, $e); + } + } + + /** + * Set the logger for this object. + * + * @see \CCR\Loggable::setLogger() + * @param \Log $logger A logger instance or null to use the null logger. + * @return self This object for method chaining. + */ + public function setLogger(Log $logger = null) + { + parent::setLogger($logger); + $this->fileWriterFactory->setLogger($logger); + return $this; + } + + /** + * Get the batch export data file path. + * + * This is the path of the file that will be used to store the export data on + * the Open XDMoD server after it has been generated. + * + * @param intger $id Batch export request primary key. + * @return string + */ + public function getExportDataFilePath($id) + { + return sprintf('%s/%s.zip', $this->exportDir, $id); + } + + /** + * Get the batch export data file name. + * + * This is the name that will be used by the file that contains the + * exported data. + * + * @param array $request Batch export request data. + * @return string + */ + public function getDataFileName(array $request) + { + return sprintf( + '%s--%s-%s.%s', + $request['realm'], + $request['start_date'], + $request['end_date'], + strtolower($request['export_file_format']) + ); + } + + /** + * Get the batch export zip file name. + * + * This is the name that will be used when the file is downloaded by a + * user. + * + * @param array $request Batch export request data. + * @return string + */ + public function getZipFileName(array $request) + { + return sprintf( + '%s--%s-%s.zip', + $request['realm'], + $request['start_date'], + $request['end_date'] + ); + } + + /** + * Write a data set to a temporary file. + * + * @param \DataWarehouse\Data\RawDataset $dataSet + * @param string $format + * @return string Path to file that was written to. + * @throws \Exception If writing the data fails. + */ + public function writeDataSetToFile(RawDataset $dataSet, $format) + { + $this->logger->info([ + 'message' => 'Writing data to file', + 'format' => $format + ]); + + try { + $dataFile = tempnam(sys_get_temp_dir(), 'batch-export-'); + $fileWriter = $this->fileWriterFactory->createFileWriter( + $format, + $dataFile + ); + + $this->logger->debug([ + 'message' => 'Created file writer', + 'file_writer' => $fileWriter + ]); + + $header = []; + + // The `export` function returns the first result along with the + // necessary metadata. + foreach ($dataSet->export() as $datum) { + $header[] = $datum['key']; + } + + $fileWriter->writeRecord($header); + + foreach ($dataSet->getResults() as $result) { + $fileWriter->writeRecord(array_values($result)); + } + + $fileWriter->close(); + + return $dataFile; + } catch (Exception $e) { + $this->logger->err([ + 'message' => $e->getMessage(), + 'stacktrace' => $e->getTraceAsString() + ]); + throw new Exception('Failed to write data set to file', 0, $e); + } + } + + /** + * Create a zip file containing a single file. + * + * @param string $dataFile Path to file that will be put in the zip file. + * @param array $request Batch export request data. + * @throws \Exception If creating the zip file fails. + */ + public function createZipFile($dataFile, array $request) + { + $zipFile = $this->getExportDataFilePath($request['id']); + + $this->logger->info([ + 'message' => 'Creating zip file', + 'batch_export_request.id' => $request['id'], + 'data_file' => $dataFile, + 'zip_file' => $zipFile + ]); + + try { + $zip = new ZipArchive(); + $zipOpenCode = $zip->open($zipFile, ZipArchive::CREATE); + + if ($zipOpenCode !== true) { + throw new Exception(sprintf( + 'Failed to open zip file "%s", error code "%s"', + $zipFile, + $zipOpenCode + )); + } + + // Override the name of the temporary data file with the proper + // name that will, be used in the archive file. + $localName = $this->getDataFileName($request); + + if ($zip->addFile($dataFile, $localName) === false) { + throw new Exception(sprintf( + 'Failed to add file "%s" to zip file "%s"', + $dataFile, + $zipFile + )); + } + + $zip->close(); + + return $zipFile; + } catch (Exception $e) { + $this->logger->err([ + 'message' => $e->getMessage(), + 'stacktrace' => $e->getTraceAsString() + ]); + throw new Exception('Failed to create zip file', 0, $e); + } + } + + /** + * Remove a batch export data file. + * + * @param int $id Export request primary key. + * @throws \Exception If removing the file fails. + */ + public function removeExportFile($id) + { + $zipFile = $this->getExportZipFilePath($id); + + $this->logger->info([ + 'message' => 'Removing export file', + 'batch_export_request.id' => $id, + 'zip_file' => $zipFile + ]); + + if (!unlink($zipFile)) { + throw new Exception(sprintf('Failed to delete "%s"', $zipFile)); + } + } +} diff --git a/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php b/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php index 4e8fde9d57..ed54fd1392 100644 --- a/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php +++ b/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php @@ -16,7 +16,7 @@ class FileWriterFactory extends Loggable * @param string $file File path. * @return \DataWarehouse\Export\FileWriter\iFileWriter */ - public function getFileWriter($format, $file) + public function createFileWriter($format, $file) { $this->logger->debug([ 'message' => 'Creating new file writer', diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 29d5fd548a..8f40ef66ce 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -3,6 +3,7 @@ namespace Rest\Controllers; use CCR\DB; +use DataWarehouse\Export\FileManager; use DataWarehouse\Export\QueryHandler; use DataWarehouse\Export\RealmManager; use DateTime; @@ -13,7 +14,6 @@ use Symfony\Component\HttpKernel\Exception\AccessDeniedHttpException; use Symfony\Component\HttpKernel\Exception\BadRequestHttpException; use Symfony\Component\HttpKernel\Exception\NotFoundHttpException; -use xd_utilities; class WarehouseExportControllerProvider extends BaseControllerProvider { @@ -30,6 +30,7 @@ class WarehouseExportControllerProvider extends BaseControllerProvider public function __construct(array $params = []) { parent::__construct($params); + $this->fileManager = new FileManager(); $this->realmManager = new RealmManager(); $this->queryHandler = new QueryHandler(); } @@ -216,13 +217,7 @@ function ($request) use ($id) { throw new BadRequestHttpException('Requested data is not available'); } - try { - $exportDir = xd_utilities\getConfiguration('data_warehouse_export', 'export_directory'); - } catch (Exception $e) { - throw new NotFoundHttpException('Export directory is not configured'); - } - - $file = sprintf('%s/%s.zip', $exportDir, $id); + $file = $this->fileManager->getExportDataFilePath($id); if (!is_file($file)) { throw new NotFoundHttpException('Exported data not found'); @@ -232,20 +227,15 @@ function ($request) use ($id) { throw new AccessDeniedHttpException('Exported data is not readable'); } - $fileName = sprintf( - '%s--%s-%s.%s.zip', - $request['realm'], - $request['start_date'], - $request['end_date'], - strtolower($request['export_file_format']) - ); - return $app->sendFile( $file, 200, [ 'Content-type' => 'application/zip', - 'Content-Disposition' => sprintf('attachment; filename="%s"', $fileName) + 'Content-Disposition' => sprintf( + 'attachment; filename="%s"', + $this->fileManager->getZipFileName($request) + ) ] ); } From 1e53430c1772b4259fe54e441a2238d94d3febc7 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 11 Jul 2019 11:16:17 -0400 Subject: [PATCH 168/217] Remove unused variable --- classes/DataWarehouse/Export/BatchProcessor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 5192799e20..76007bded1 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -141,7 +141,7 @@ private function processSubmittedRequest(array $request) $dataSet = $this->getDataSet($request, $user); $format = $this->dryRun ? 'null' : $request['export_file_format']; $dataFile = $this->fileManager->writeDataSetToFile($dataSet, $format); - $zipFile = $this->fileManager->createZipFile($dataFile, $request); + $this->fileManager->createZipFile($dataFile, $request); // Query for same record to get expiration date. $request = $this->queryHandler->getRequestRecord($request['id']); From f646b93f407a5a3f0610a415ac787511f93d955e Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 11 Jul 2019 14:13:50 -0400 Subject: [PATCH 169/217] Remove role parameters from query --- classes/DataWarehouse/Export/BatchProcessor.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 76007bded1..0cf3296828 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -237,9 +237,6 @@ private function getDataSet(array $request, XDUser $user) ], 'accounting' ); - $allRoles = $user->getAllRoles(); - $query->setMultipleRoleParameters($allRoles, $user); - $dataSet = new RawDataset($query, $user); $this->logger->debug('Executing query'); From 318a2fe5bdfb75b794abccd95ad572c53c7d7cde Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 12 Jul 2019 14:11:45 -0400 Subject: [PATCH 170/217] Fix typo --- classes/DataWarehouse/Export/FileManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/FileManager.php b/classes/DataWarehouse/Export/FileManager.php index b2d4bda80d..0b71028bed 100644 --- a/classes/DataWarehouse/Export/FileManager.php +++ b/classes/DataWarehouse/Export/FileManager.php @@ -239,7 +239,7 @@ public function createZipFile($dataFile, array $request) */ public function removeExportFile($id) { - $zipFile = $this->getExportZipFilePath($id); + $zipFile = $this->getExportDataFilePath($id); $this->logger->info([ 'message' => 'Removing export file', From 8616a8271b89e529bc5b663985cfc9fe99173ee0 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 12 Jul 2019 14:19:14 -0400 Subject: [PATCH 171/217] Change download URL and implement test --- .../WarehouseExportControllerProvider.php | 4 +-- html/gui/js/modules/DataExport.js | 2 +- .../Rest/WarehouseExportControllerTest.php | 30 +++++++++++++------ 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 8f40ef66ce..4135bfa0ae 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -54,7 +54,7 @@ public function setupRoutes( $controller->get("$root/requests", "$current::getRequests"); $controller->delete("$root/requests", "$current::deleteRequests"); - $controller->get("$root/request/{id}", "$current::getRequest") + $controller->get("$root/download/{id}", "$current::getExportedDataFile") ->assert('id', '\d+') ->convert('id', "$conversions::toInt"); @@ -194,7 +194,7 @@ function ($realm) { * @throws NotFoundHttpException * @throws BadRequestHttpException */ - public function getRequest(Request $request, Application $app, $id) + public function getExportedDataFile(Request $request, Application $app, $id) { $user = $this->authorize($request); diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index d93a250c2a..2367d37d3a 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -495,7 +495,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { }, downloadRequest: function (record) { - window.open('rest/v1/warehouse/export/request/' + record.get('id')); + window.open('rest/v1/warehouse/export/download/' + record.get('id')); }, resubmitRequest: function (record) { diff --git a/tests/integration/lib/Rest/WarehouseExportControllerTest.php b/tests/integration/lib/Rest/WarehouseExportControllerTest.php index 529552149c..e6c7a03127 100644 --- a/tests/integration/lib/Rest/WarehouseExportControllerTest.php +++ b/tests/integration/lib/Rest/WarehouseExportControllerTest.php @@ -3,6 +3,7 @@ namespace IntegrationTests\Rest; use CCR\DB; +use DataWarehouse\Export\FileManager; use DataWarehouse\Export\QueryHandler; use Exception; use JsonSchema\Constraints\Constraint; @@ -61,6 +62,12 @@ class WarehouseExportControllerTest extends PHPUnit_Framework_TestCase */ private static $queryHandler; + /** + * Data warehouse export file manager. + * @var FileManager + */ + private static $fileManager; + /** * JSON schema validator. * @var Validator @@ -113,6 +120,7 @@ public static function setUpBeforeClass() self::$schemaValidator = new Validator(); self::$queryHandler = new QueryHandler(); + self::$fileManager = new FileManager(); } /** @@ -269,17 +277,21 @@ public function testGetRequests( /** * Test getting the exported data. * - * @covers ::getRequest - * @depends testCreateRequest - * #dataProvider getRequestProvider + * @covers ::getExportedDataFile */ - //public function testGetRequest($role, $params, $httpCode) - public function testGetRequest() + public function testDownloadExportedDataFile() { - //list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/request/' . $id); - //$this->assertRegExp('#\applcation/zip\b#', $headers['Content-Type'], 'Content type header'); - //$this->assertEquals($httpCode, $info['http_code'], 'HTTP response code'); - $this->markTestIncomplete('This test has not been implemented yet.'); + $role = 'usr'; + $zipContent = 'Mock Zip File'; + $id = self::$queryHandler->createRequestRecord(self::$users[$role]->getUserID(), 'jobs', '2019-01-01', '2019-01-31', 'CSV'); + self::$queryHandler->submittedToAvailable($id); + @file_put_contents(self::$fileManager->getExportDataFilePath($id), $zipContent); + list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/download/' . $id); + $this->assertRegExp('#\bapplication/zip\b#', $headers['Content-Type'], 'Content type header'); + $this->assertEquals(200, $info['http_code'], 'HTTP response code'); + $this->assertEquals($zipContent, $content, 'Download content'); + self::$fileManager->removeExportFile($id); + self::$queryHandler->deleteRequest($id); } /** From b35343c62a70a6e2d9f08616b0ef8b4baaaf2847 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 12 Jul 2019 14:21:43 -0400 Subject: [PATCH 172/217] Add email details --- classes/DataWarehouse/Export/BatchProcessor.php | 14 +++++++++----- email_templates/batch_export_failure.template | 10 ++++++++++ .../batch_export_failure_admin_notice.template | 14 ++++++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 0cf3296828..7694beb3ff 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -284,7 +284,11 @@ private function sendExportSuccessEmail(XDUser $user, array $request) 'last_name' => $user->getLastName(), 'current_date' => date('Y-m-d'), 'expiration_date' => $expirationDate, - 'download_url' => '', // TODO + 'download_url' => sprintf( + '%srest/v1/warehouse/export/download/%d', + xd_utilities\getConfigurationUrlBase('general', 'site_address'), + $request['id'] + ), 'maintainer_signature' => MailWrapper::getMaintainerSignature() ] ); @@ -327,11 +331,11 @@ private function sendExportFailureEmail( 'subject' => 'Batch export failed', 'toAddress' => xd_utilities\getConfiguration('general', 'tech_support_recipient'), 'user_email' => $user->getEmailAddress(), - 'user_first_name' => $user->getFirstName(), - 'user_last_name' => $user->getLastName(), + 'user_username' => $user->getUsername(), + 'user_formal_name' => $user->getFormalName(true), 'current_date' => date('Y-m-d'), - 'failure_exception' => $message, - 'failure_stack_trace' => $stackTrace + 'exception_message' => $message, + 'exception_stack_trace' => $stackTrace ] ); diff --git a/email_templates/batch_export_failure.template b/email_templates/batch_export_failure.template index e69de29bb2..d5d646fe9b 100644 --- a/email_templates/batch_export_failure.template +++ b/email_templates/batch_export_failure.template @@ -0,0 +1,10 @@ +Dear [:first_name:], + +Your batch export request failed on [:current_date:] for the following reason: + + [:failure_reason:] + +The technical staff has been notified and will investigate the issue. + +Sincerely, +[:maintainer_signature:] diff --git a/email_templates/batch_export_failure_admin_notice.template b/email_templates/batch_export_failure_admin_notice.template index e69de29bb2..cddea0f996 100644 --- a/email_templates/batch_export_failure_admin_notice.template +++ b/email_templates/batch_export_failure_admin_notice.template @@ -0,0 +1,14 @@ +Dear Tech Support, + +There was an error while processing a data warehouse batch export on [:current_date:]. + +Person Details ----------------------------------- +Name: [:user_formal_name:] +Username: [:user_username:] +E-Mail: [:user_email:] + +Exception Details -------------------------------- +Message: [:exception_message:] + +Stack Trace: +[:exception_stack_trace:] From 792146bd47edf1c35fdb3dde013055001275b639 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 12 Jul 2019 14:39:36 -0400 Subject: [PATCH 173/217] Fix test --- tests/integration/lib/Rest/WarehouseExportControllerTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/lib/Rest/WarehouseExportControllerTest.php b/tests/integration/lib/Rest/WarehouseExportControllerTest.php index e6c7a03127..ea2e7a39dc 100644 --- a/tests/integration/lib/Rest/WarehouseExportControllerTest.php +++ b/tests/integration/lib/Rest/WarehouseExportControllerTest.php @@ -291,7 +291,7 @@ public function testDownloadExportedDataFile() $this->assertEquals(200, $info['http_code'], 'HTTP response code'); $this->assertEquals($zipContent, $content, 'Download content'); self::$fileManager->removeExportFile($id); - self::$queryHandler->deleteRequest($id); + self::$queryHandler->deleteRequest($id, self::$users[$role]->getUserID()); } /** From 988d6162b8bae1d8719bffae3e1c461e22e2f1ed Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 15 Jul 2019 07:35:17 -0400 Subject: [PATCH 174/217] Ensure realms are returned in numeric array --- classes/Rest/Controllers/WarehouseExportControllerProvider.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php index 4135bfa0ae..7a49d51430 100644 --- a/classes/Rest/Controllers/WarehouseExportControllerProvider.php +++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php @@ -87,7 +87,7 @@ function ($realm) { return $app->json( [ 'success' => true, - 'data' => $realms, + 'data' => array_values($realms), 'total' => count($realms) ] ); From 3fe3f2a066bca633310d7a737c77a4063fa1f3c9 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 15 Jul 2019 09:23:04 -0400 Subject: [PATCH 175/217] Include messages for previous stack traces --- classes/DataWarehouse/Export/BatchProcessor.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 7694beb3ff..5ef06b2a7c 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -322,7 +322,11 @@ private function sendExportFailureEmail( $message = $e->getMessage(); $stackTrace = $e->getTraceAsString(); while ($e = $e->getPrevious()) { - $stackTrace .= "\n\n" . $e->getTraceAsString(); + $stackTrace .= sprintf( + "\n\n%s\n%s", + $e->getMessage(), + $e->getTraceAsString() + ); } MailWrapper::sendTemplate( From dfdc4f51c54198fd84a2833a66b3114cebed318b Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 15 Jul 2019 10:56:20 -0400 Subject: [PATCH 176/217] Disable memory and time limits --- background_scripts/batch_export_manager.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/background_scripts/batch_export_manager.php b/background_scripts/batch_export_manager.php index 5313744e22..fe5d2720a8 100755 --- a/background_scripts/batch_export_manager.php +++ b/background_scripts/batch_export_manager.php @@ -9,6 +9,12 @@ use CCR\Log; use DataWarehouse\Export\BatchProcessor; +// Disable memory limit. +ini_set('memory_limit', -1); + +// Disable maximum execution time. +set_time_limit(0); + try { $help = false; $dryRun = false; From 320bcc2088ea09e28438a0faa2dc4b92a8fc9add Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 16 Jul 2019 13:49:05 -0400 Subject: [PATCH 177/217] Add debug logging statement --- classes/DataWarehouse/Export/FileManager.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/classes/DataWarehouse/Export/FileManager.php b/classes/DataWarehouse/Export/FileManager.php index 0b71028bed..9b86f475a0 100644 --- a/classes/DataWarehouse/Export/FileManager.php +++ b/classes/DataWarehouse/Export/FileManager.php @@ -159,6 +159,10 @@ public function writeDataSetToFile(RawDataset $dataSet, $format) $header[] = $datum['key']; } + $this->logger->debug([ + 'message' => 'Writing header', + 'header' => json_encode($header) + ]); $fileWriter->writeRecord($header); foreach ($dataSet->getResults() as $result) { From 3d10ce756b5ef1dd5881e07ad0da441e5cce805b Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 19 Jul 2019 15:01:04 -0400 Subject: [PATCH 178/217] WIP --- classes/DataWarehouse/Data/BatchDataset.php | 177 ++++++++++++++++++ .../DataWarehouse/Export/BatchProcessor.php | 18 +- classes/DataWarehouse/Export/FileManager.php | 24 +-- configuration/rawstatistics.d/20_jobs.json | 72 ++++--- 4 files changed, 236 insertions(+), 55 deletions(-) create mode 100644 classes/DataWarehouse/Data/BatchDataset.php diff --git a/classes/DataWarehouse/Data/BatchDataset.php b/classes/DataWarehouse/Data/BatchDataset.php new file mode 100644 index 0000000000..774054c271 --- /dev/null +++ b/classes/DataWarehouse/Data/BatchDataset.php @@ -0,0 +1,177 @@ +query = $query; + $this->docs = $query->getColumnDocumentation(); + $this->dbh = DB::factory($query->_db_profile); + + foreach ($this->docs as $doc) { + switch ($doc['batch_export']) { + case true: + $this->header[] = $doc['name']; + break; + case 'anonymize': + $this->anonymousFields[] = $doc['name']; + $this->header[] = $doc['name']; + break; + case false: + default: + // Don't include fields by default. + break; + } + } + + } + + /** + * Get the header row. + * + * @return string[] + */ + public function getHeader() + { + return $this->header; + } + + /** + * Get the current row from the data set. + * + * @return mixed[] + */ + public current() + { + return $this->currentRow; + } + + /** + * Get the current row index. + * + * @return int + */ + public key() + { + return $this->currentRowIndex; + } + + /** + * Advance iterator to the next row. + * + * Fetches the next row. + */ + public next() + { + $this->currentRowIndex++; + $this->currentRow = $this->getNextRow(); + } + + /** + * Rewind the iterator to the beginning. + * + * Executes the underlying raw query. + */ + public rewind() + { + $this->logger->debug('Executing query'); + $this->sth = $this->query->getRawStatement(); + // Set query to be unbuffered so results are not all loaded into memory. + // @see ETL\Ingestor\pdoIngestor::multiDatabaseIngest() + /* + $this->sth->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); + $result = $this->dbh->query( + "SHOW SESSION VARIABLES WHERE Variable_name = 'net_write_timeout'" + ); + + $currentTimeout = 0; + if ( 0 != count($result) ) { + $currentTimeout = $result[0]['Value']; + $this->logger->debug("Current net_write_timeout = $currentTimeout"); + } + + $newTimeout = $numDestinationTables * $this->netWriteTimeoutSecondsPerFileChunk; + + if ( $newTimeout > $currentTimeout ) { + $sql = sprintf('SET SESSION net_write_timeout = %d', $newTimeout); + $this->executeSqlList(array($sql), $this->sourceEndpoint); + } + */ + + $this->currentRowIndex = 1; + $this->currentRow = $this->getNextRow(); + } + + /** + * @return bool + */ + public valid() + { + return $this->currentRow !== false; + } + + private function getNextRow() + { + $rawRow = $this->sth->fetch(PDO::FETCH_ASSOC); + } +} diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 5ef06b2a7c..8004a1dffa 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -8,7 +8,7 @@ use CCR\DB; use CCR\Loggable; use CCR\MailWrapper; -use DataWarehouse\Data\RawDataset; +use DataWarehouse\Data\BatchDataset; use Exception; use Log; use XDUser; @@ -210,7 +210,7 @@ private function processExpiringRequest(array $request) * * @param array $request * @param \XDUser $user - * @return \DataWarehouse\Data\RawDataset; + * @return \DataWarehouse\Data\BatchDataset; * @throws \Exception */ private function getDataSet(array $request, XDUser $user) @@ -237,23 +237,15 @@ private function getDataSet(array $request, XDUser $user) ], 'accounting' ); - $dataSet = new RawDataset($query, $user); - - $this->logger->debug('Executing query'); - - // Data are fetched from the database as a side effect of checking - // for results. - if ($dataSet->hasResults()) { - $this->logger->debug('Data set has results'); - } - + $dataSet = new BatchDataset($query, $user); + $dataSet->setLogger($this->logger); return $dataSet; } catch (Exception $e) { $this->logger->err([ 'message' => $e->getMessage(), 'stacktrace' => $e->getTraceAsString() ]); - throw new Exception('Failed to execute batch export query', 0, $e); + throw new Exception('Failed to create batch export query', 0, $e); } } diff --git a/classes/DataWarehouse/Export/FileManager.php b/classes/DataWarehouse/Export/FileManager.php index 9b86f475a0..0e766f57b9 100644 --- a/classes/DataWarehouse/Export/FileManager.php +++ b/classes/DataWarehouse/Export/FileManager.php @@ -3,7 +3,7 @@ namespace DataWarehouse\Export; use CCR\Loggable; -use DataWarehouse\Data\RawDataset; +use DataWarehouse\Data\BatchDataset; use DataWarehouse\Export\FileWriter\FileWriterFactory; use Exception; use Log; @@ -127,12 +127,12 @@ public function getZipFileName(array $request) /** * Write a data set to a temporary file. * - * @param \DataWarehouse\Data\RawDataset $dataSet + * @param \DataWarehouse\Data\BatchDataset $dataSet * @param string $format * @return string Path to file that was written to. * @throws \Exception If writing the data fails. */ - public function writeDataSetToFile(RawDataset $dataSet, $format) + public function writeDataSetToFile(BatchDataset $dataSet, $format) { $this->logger->info([ 'message' => 'Writing data to file', @@ -151,22 +151,10 @@ public function writeDataSetToFile(RawDataset $dataSet, $format) 'file_writer' => $fileWriter ]); - $header = []; + $fileWriter->writeRecord($dataSet->getHeader()) - // The `export` function returns the first result along with the - // necessary metadata. - foreach ($dataSet->export() as $datum) { - $header[] = $datum['key']; - } - - $this->logger->debug([ - 'message' => 'Writing header', - 'header' => json_encode($header) - ]); - $fileWriter->writeRecord($header); - - foreach ($dataSet->getResults() as $result) { - $fileWriter->writeRecord(array_values($result)); + foreach ($dataSet as $record) { + $fileWriter->writeRecord($record); } $fileWriter->close(); diff --git a/configuration/rawstatistics.d/20_jobs.json b/configuration/rawstatistics.d/20_jobs.json index 5481ec84c3..0d6258d433 100644 --- a/configuration/rawstatistics.d/20_jobs.json +++ b/configuration/rawstatistics.d/20_jobs.json @@ -11,7 +11,8 @@ "name": "Local Job Id", "dtype": "accounting", "group": "Administration", - "documentation": "The unique identifier assigned to the job by the job scheduler." + "documentation": "The unique identifier assigned to the job by the job scheduler.", + "batch_export": true }, { "key": "resource_id", @@ -22,7 +23,8 @@ "schema": "modw", "table": "resourcefact" }, - "documentation": "The resource that ran the job." + "documentation": "The resource that ran the job.", + "batch_export": true }, { "key": "systemaccount_id", @@ -34,7 +36,8 @@ "table": "systemaccount", "column": "username" }, - "documentation": "The username on the resource of the user that ran the job. May be a UID or string username depending on the resource." + "documentation": "The username on the resource of the user that ran the job. May be a UID or string username depending on the resource.", + "batch_export": false }, { "key": "person_id", @@ -46,7 +49,8 @@ "table": "person", "column": "long_name" }, - "documentation": "The name of the job owner." + "documentation": "The name of the job owner.", + "batch_export": "anonymize" }, { "key": "person_organization_id", @@ -57,14 +61,16 @@ "schema": "modw", "table": "organization" }, - "documentation": "The organization of the person who ran the task" + "documentation": "The organization of the person who ran the task", + "batch_export": true }, { "key": "name", "name": "Name", "documentation": "The name of the job as reported by the job scheduler.", "dtype": "accounting", - "group": "Executable" + "group": "Executable", + "batch_export": false }, { "key": "submit_time_ts", @@ -72,7 +78,8 @@ "dtype": "accounting", "group": "Timing", "units": "ts", - "documentation": "Task submission time" + "documentation": "Task submission time", + "batch_export": true }, { "key": "start_time_ts", @@ -80,7 +87,8 @@ "dtype": "accounting", "group": "Timing", "units": "ts", - "documentation": "The time that the job started running." + "documentation": "The time that the job started running.", + "batch_export": true }, { "key": "end_time_ts", @@ -88,7 +96,8 @@ "units": "ts", "dtype": "accounting", "group": "Timing", - "documentation": "The time that the job ended." + "documentation": "The time that the job ended.", + "batch_export": true }, { "key": "eligible_time_ts", @@ -96,7 +105,8 @@ "units": "ts", "dtype": "accounting", "group": "Timing", - "documentation": "The time that the job was eligible for scheduling by the resource manager." + "documentation": "The time that the job was eligible for scheduling by the resource manager.", + "batch_export": true }, { "key": "node_count", @@ -108,14 +118,16 @@ "table": "nodecount", "column": "nodes" }, - "documentation": "The number of nodes that were assigned to the job." + "documentation": "The number of nodes that were assigned to the job.", + "batch_export": true }, { "key": "processor_count", "name": "Cores", "dtype": "accounting", "group": "Allocated Resource", - "documentation": "The number of cores that were assigned to the job." + "documentation": "The number of cores that were assigned to the job.", + "batch_export": true }, { "key": "memory_kb", @@ -123,7 +135,8 @@ "dtype": "accounting", "group": "Allocated resource", "units": "kilobyte", - "documentation": "Memory consumed as reported by the resource manager." + "documentation": "Memory consumed as reported by the resource manager.", + "batch_export": true }, { "key": "wallduration", @@ -131,7 +144,8 @@ "dtype": "accounting", "group": "Timing", "units": "seconds", - "documentation": "Overall job duration." + "documentation": "Overall job duration.", + "batch_export": true }, { "key": "waitduration", @@ -139,7 +153,8 @@ "dtype": "accounting", "group": "Timing", "units": "seconds", - "documentation": "Time the job waited in the queue" + "documentation": "Time the job waited in the queue", + "batch_export": true }, { "key": "cpu_time", @@ -147,49 +162,56 @@ "dtype": "accounting", "group": "Allocated resource", "units": "seconds", - "documentation": "The amount of CPU core time (Core Count * Wall Time)" + "documentation": "The amount of CPU core time (Core Count * Wall Time)", + "batch_export": true }, { "key": "group_name", "name": "UNIX group name", "dtype": "accounting", "group": "Administration", - "documentation": "The name of the group that ran the job." + "documentation": "The name of the group that ran the job.", + "batch_export": false }, { "key": "gid_number", "name": "UNIX group GID", "dtype": "accounting", "group": "Administration", - "documentation": "The GID of the group that ran the job." + "documentation": "The GID of the group that ran the job.", + "batch_export": false }, { "key": "uid_number", "name": "UNIX UID", "dtype": "accounting", "group": "Administration", - "documentation": "The UID of the user that ran the job." + "documentation": "The UID of the user that ran the job.", + "batch_export": false }, { "key": "exit_code", "name": "Exit Code", "dtype": "accounting", "group": "Executable", - "documentation": "The code that the job exited with." + "documentation": "The code that the job exited with.", + "batch_export": true }, { "key": "exit_state", "name": "Exit State", "dtype": "accounting", "group": "Executable", - "documentation": "The state of the job when it completed." + "documentation": "The state of the job when it completed.", + "batch_export": true }, { "key": "cpu_req", "name": "Requested Cores", "dtype": "accounting", "group": "Requested resource", - "documentation": "The number of CPUs required by the job." + "documentation": "The number of CPUs required by the job.", + "batch_export": true }, { "key": "mem_req", @@ -197,7 +219,8 @@ "dtype": "accounting", "group": "Requested resource", "units": "bytes", - "documentation": "The amount of memory required by the job." + "documentation": "The amount of memory required by the job.", + "batch_export": true }, { "key": "timelimit", @@ -205,7 +228,8 @@ "dtype": "accounting", "group": "Requested resource", "units": "seconds", - "documentation": "The time limit of the job." + "documentation": "The time limit of the job.", + "batch_export": true } ] } From 1ea58a32b536d40f38be0e7817aa701fd9771246 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 23 Jul 2019 08:33:17 -0400 Subject: [PATCH 179/217] Add missing semicolon --- classes/DataWarehouse/Export/FileManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/FileManager.php b/classes/DataWarehouse/Export/FileManager.php index 0e766f57b9..e8559150ff 100644 --- a/classes/DataWarehouse/Export/FileManager.php +++ b/classes/DataWarehouse/Export/FileManager.php @@ -151,7 +151,7 @@ public function writeDataSetToFile(BatchDataset $dataSet, $format) 'file_writer' => $fileWriter ]); - $fileWriter->writeRecord($dataSet->getHeader()) + $fileWriter->writeRecord($dataSet->getHeader()); foreach ($dataSet as $record) { $fileWriter->writeRecord($record); From b95ecdd5e6f13f4a89131faf2fa08c005d4d971d Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 23 Jul 2019 08:36:11 -0400 Subject: [PATCH 180/217] Comment-out paging until implemented --- html/gui/js/modules/DataExport.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 2367d37d3a..4f0c567b38 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -400,7 +400,9 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { scope: this, handler: this.deleteExpiredRequests }, - '->', + '->' + /* + , { xtype: 'paging', store: this.store, @@ -409,6 +411,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { displayMsg: 'Displaying export requests {0} - {1} of {2}', emptyMsg: 'No export requests to display' } + */ ] }); From 5b93ceb9c2350eae9725179ce6b8bb97227e6882 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 24 Jul 2019 11:33:19 -0400 Subject: [PATCH 181/217] Refactor batch data set --- classes/DataWarehouse/Data/BatchDataset.php | 77 +++++++++++++++---- .../DataWarehouse/Export/BatchProcessor.php | 3 +- 2 files changed, 63 insertions(+), 17 deletions(-) diff --git a/classes/DataWarehouse/Data/BatchDataset.php b/classes/DataWarehouse/Data/BatchDataset.php index 774054c271..f1f6b23560 100644 --- a/classes/DataWarehouse/Data/BatchDataset.php +++ b/classes/DataWarehouse/Data/BatchDataset.php @@ -3,11 +3,13 @@ namespace DataWarehouse\Data; use CCR\DB; +use CCR\Loggable; use DataWarehouse\Query\RawQuery; +use Exception; use Iterator; use PDO; use XDUser; -use CCR\Loggable +use Log; /** * Data set for batch export queries. @@ -56,34 +58,56 @@ class BatchDataset extends Loggable implements Iterator private $header = []; /** - * @var string[] + * Fields that need to be anonymized. + * + * Keys correspond to the field name. + * + * @var array */ private $anonymousFields = []; /** + * @param mixed $name Description. + * @param \XDUser $user + * @param \Log $logger */ - public function __construct(RawQuery $query, XDUser $user) + public function __construct(RawQuery $query, XDUser $user, Log $logger = null) { + parent::__construct($logger); + $this->query = $query; $this->docs = $query->getColumnDocumentation(); $this->dbh = DB::factory($query->_db_profile); - foreach ($this->docs as $doc) { - switch ($doc['batch_export']) { + foreach ($this->docs as $key => $doc) { + $export = isset($doc['batch_export']) ? $doc['batch_export'] : false; + $name = $doc['name']; + + if (isset($doc['units']) && $doc['units'] === 'ts') { + $name .= ' (Timestamp)'; + } + + switch ($export) { case true: - $this->header[] = $doc['name']; + $this->header[$key] = $name; break; case 'anonymize': - $this->anonymousFields[] = $doc['name']; - $this->header[] = $doc['name']; + $this->header[$key] = $name . ' (Deidentified)'; + $this->anonymousFields[$key] = true; break; case false: + break; default: - // Don't include fields by default. + throw new Exception(sprintf( + 'Unknown "batch_export" option %s', + var_export($export, true) + )); break; } } + $this->logger->debug(sprintf('Header: %s', json_encode($this->header))); + $this->logger->debug(sprintf('Anonymous fields: %s', json_encode($this->anonymousFields))); } /** @@ -93,7 +117,7 @@ public function __construct(RawQuery $query, XDUser $user) */ public function getHeader() { - return $this->header; + return array_values($this->header); } /** @@ -101,7 +125,7 @@ public function getHeader() * * @return mixed[] */ - public current() + public function current() { return $this->currentRow; } @@ -111,7 +135,7 @@ public function getHeader() * * @return int */ - public key() + public function key() { return $this->currentRowIndex; } @@ -121,7 +145,7 @@ public function getHeader() * * Fetches the next row. */ - public next() + public function next() { $this->currentRowIndex++; $this->currentRow = $this->getNextRow(); @@ -132,7 +156,7 @@ public function getHeader() * * Executes the underlying raw query. */ - public rewind() + public function rewind() { $this->logger->debug('Executing query'); $this->sth = $this->query->getRawStatement(); @@ -165,13 +189,36 @@ public function getHeader() /** * @return bool */ - public valid() + public function valid() { return $this->currentRow !== false; } + /** + */ + private function anonymizeField($field) + { + return md5($field); + } + + /** + */ private function getNextRow() { $rawRow = $this->sth->fetch(PDO::FETCH_ASSOC); + + if ($rawRow === false) { + return false; + } + + $row = []; + + foreach (array_keys($this->header) as $key) { + $row[] = isset($this->anonymousFields[$key]) + ? $this->anonymizeField($rawRow[$key]) + : $rawRow[$key]; + } + + return $row; } } diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index 8004a1dffa..c0eaad2981 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -237,8 +237,7 @@ private function getDataSet(array $request, XDUser $user) ], 'accounting' ); - $dataSet = new BatchDataset($query, $user); - $dataSet->setLogger($this->logger); + $dataSet = new BatchDataset($query, $user, $this->logger); return $dataSet; } catch (Exception $e) { $this->logger->err([ From dc0f7c0563e5c3b7cf8af2b74f7f297d349e3e37 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 26 Jul 2019 07:17:02 -0400 Subject: [PATCH 182/217] Add message for successful request submission --- html/gui/js/modules/DataExport.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 4f0c567b38..42e5f290bf 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -55,6 +55,17 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { // Reload the requests every time a new request is submitted. this.requestForm.on('actioncomplete', this.requestsStore.reload, this.requestsStore); + // Display alert every time a new request is submitted. + this.requestForm.on( + 'actioncomplete', + function () { + Ext.Msg.alert( + 'Request Submitted', + XDMoD.Module.DataExport.requestSubmittedText + ); + } + ); + this.items = [ { xtype: 'panel', @@ -457,6 +468,10 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { scope: this, success: function () { this.store.reload(); + Ext.Msg.alert( + 'Request Submitted', + XDMoD.Module.DataExport.requestSubmittedText + ); }, failure: function (response) { Ext.Msg.alert( @@ -540,6 +555,11 @@ XDMoD.Module.DataExport.exportStatusHelpText = 'Failed: Data export failed. Submit a support request for more ' + 'information.'; +XDMoD.Module.DataExport.requestSubmittedText = +'Your bulk data export request has been successfully submitted. Requests ' + +'are typically fulfilled in 24 hours. You will recieve an email notifying ' + +'you when your data is available.'; + /** * Data warehouse export batch requests data store. */ From a5b1098f9788122d909f1faa677b6fb6a7515fac Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 26 Jul 2019 07:31:58 -0400 Subject: [PATCH 183/217] Change batch export option --- configuration/rawstatistics.d/20_jobs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/rawstatistics.d/20_jobs.json b/configuration/rawstatistics.d/20_jobs.json index 0d6258d433..67ae488b91 100644 --- a/configuration/rawstatistics.d/20_jobs.json +++ b/configuration/rawstatistics.d/20_jobs.json @@ -37,7 +37,7 @@ "column": "username" }, "documentation": "The username on the resource of the user that ran the job. May be a UID or string username depending on the resource.", - "batch_export": false + "batch_export": "anonymize" }, { "key": "person_id", From a3b2a5d2d80fe1f6a67127cad513ea07ec30dae3 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 26 Jul 2019 08:17:06 -0400 Subject: [PATCH 184/217] Replace switch statement with if statements --- classes/DataWarehouse/Data/BatchDataset.php | 28 +++++++++------------ 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/classes/DataWarehouse/Data/BatchDataset.php b/classes/DataWarehouse/Data/BatchDataset.php index f1f6b23560..89660284fe 100644 --- a/classes/DataWarehouse/Data/BatchDataset.php +++ b/classes/DataWarehouse/Data/BatchDataset.php @@ -87,22 +87,18 @@ public function __construct(RawQuery $query, XDUser $user, Log $logger = null) $name .= ' (Timestamp)'; } - switch ($export) { - case true: - $this->header[$key] = $name; - break; - case 'anonymize': - $this->header[$key] = $name . ' (Deidentified)'; - $this->anonymousFields[$key] = true; - break; - case false: - break; - default: - throw new Exception(sprintf( - 'Unknown "batch_export" option %s', - var_export($export, true) - )); - break; + if ($export === true) { + $this->header[$key] = $name; + } elseif ($export === 'anonymize') { + $this->header[$key] = $name . ' (Deidentified)'; + $this->anonymousFields[$key] = true; + } elseif ($export === false) { + // Skip field. + } else { + throw new Exception(sprintf( + 'Unknown "batch_export" option %s', + var_export($export, true) + )); } } From f87e41bca8b8497996eb2852a28f24922fbbc4a4 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 26 Jul 2019 08:18:01 -0400 Subject: [PATCH 185/217] Update batch export configuration --- configuration/rawstatistics.d/20_jobs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/rawstatistics.d/20_jobs.json b/configuration/rawstatistics.d/20_jobs.json index 67ae488b91..b3772fe1df 100644 --- a/configuration/rawstatistics.d/20_jobs.json +++ b/configuration/rawstatistics.d/20_jobs.json @@ -50,7 +50,7 @@ "column": "long_name" }, "documentation": "The name of the job owner.", - "batch_export": "anonymize" + "batch_export": true }, { "key": "person_organization_id", From 4ca708d577e7d035c2fd7ccd56c4b0aaf0b9d070 Mon Sep 17 00:00:00 2001 From: Joseph White Date: Thu, 25 Jul 2019 22:07:52 -0400 Subject: [PATCH 186/217] Update ChartPortlet for UI consistency. As per the UI meeting update the title so that it explicitly states the date range. The auto ellipses code was updated to use the libary function. --- html/gui/js/modules/summary/ChartPortlet.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/html/gui/js/modules/summary/ChartPortlet.js b/html/gui/js/modules/summary/ChartPortlet.js index fd9d0a6d0c..7b4c783bca 100644 --- a/html/gui/js/modules/summary/ChartPortlet.js +++ b/html/gui/js/modules/summary/ChartPortlet.js @@ -23,10 +23,7 @@ XDMoD.Modules.SummaryPortlets.ChartPortlet = Ext.extend(Ext.ux.Portlet, { initComponent: function () { var self = this; - this.title = this.config.chart.title; - if (this.title.length > 60) { - this.title = this.title.substring(0, 57) + '...'; - } + this.title = Ext.util.Format.ellipsis(this.config.chart.title, 60, true); // Sync date ranges var dateRanges = CCR.xdmod.ui.DurationToolbar.getDateRanges(); @@ -40,6 +37,8 @@ XDMoD.Modules.SummaryPortlets.ChartPortlet = Ext.extend(Ext.ux.Portlet, { this.config.chart.end_date = date.end.format('Y-m-d'); } + this.title += ' - ' + this.config.chart.start_date + ' to ' + this.config.chart.end_date; + var highchartConfig = {}; jQuery.extend(true, highchartConfig, this.config.chart); highchartConfig.title = ''; From 2c508bb53ec5a491ea111f46b78772d69f356da6 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 6 Aug 2019 07:56:08 -0400 Subject: [PATCH 187/217] Change data export download URL This URL now directs the user to the data export tab where the download is then initiated. This allows logged-out users to use the URL, log in, then be directed to the download. --- .../DataWarehouse/Export/BatchProcessor.php | 2 +- html/gui/js/modules/DataExport.js | 31 ++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index c0eaad2981..c46d4dc71b 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -276,7 +276,7 @@ private function sendExportSuccessEmail(XDUser $user, array $request) 'current_date' => date('Y-m-d'), 'expiration_date' => $expirationDate, 'download_url' => sprintf( - '%srest/v1/warehouse/export/download/%d', + '%s#data_export?action=download&id=%d', xd_utilities\getConfigurationUrlBase('general', 'site_address'), $request['id'] ), diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index 42e5f290bf..ffcea2772d 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -47,6 +47,28 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { // Defer loading of realms so they are not loaded immediately. this.on('beforerender', this.realmsStore.load, this.realmsStore, { single: true }); + // Open the download window if this is a download URL. + this.on('activate', function () { + var token = CCR.tokenize(document.location.hash); + var params = Ext.urlDecode(token.params); + + if (params.action === 'download') { + // Update history so the download URL is no longer present. + Ext.History.add(this.id); + + // A confirmation message is used because the download cannot + // be initiated automatically as it would be blocked as a + // pop-up. + Ext.Msg.confirm( + 'Data Export', + 'Download exported data now?', + function () { + XDMoD.Module.DataExport.openDownloadWindow(params.id); + } + ); + } + }, this, { single: true }); + // Load the requests after the realms have loaded. This is necessary so // that the realm name can be determined from its ID when displayed in // the grid. @@ -95,6 +117,13 @@ XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, { } }); +/** + * Open a new window to download the requested data. + */ +XDMoD.Module.DataExport.openDownloadWindow = function (requestId) { + window.open('rest/v1/warehouse/export/download/' + requestId); +}; + /** * Data export request form. */ @@ -513,7 +542,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { }, downloadRequest: function (record) { - window.open('rest/v1/warehouse/export/download/' + record.get('id')); + XDMoD.Module.DataExport.openDownloadWindow(record.get('id')); }, resubmitRequest: function (record) { From c682a57f57e8e76ac952805b7e35d983f142db1a Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 6 Aug 2019 08:37:31 -0400 Subject: [PATCH 188/217] Add hash salt --- classes/DataWarehouse/Data/BatchDataset.php | 44 ++++++++++++++++++- .../Version812To850/ConfigFilesMigration.php | 1 + classes/OpenXdmod/Setup/GeneralSetup.php | 4 ++ configuration/portal_settings.ini | 2 + templates/portal_settings.template | 2 + 5 files changed, 51 insertions(+), 2 deletions(-) diff --git a/classes/DataWarehouse/Data/BatchDataset.php b/classes/DataWarehouse/Data/BatchDataset.php index 89660284fe..f4122170d5 100644 --- a/classes/DataWarehouse/Data/BatchDataset.php +++ b/classes/DataWarehouse/Data/BatchDataset.php @@ -7,9 +7,10 @@ use DataWarehouse\Query\RawQuery; use Exception; use Iterator; +use Log; use PDO; use XDUser; -use Log; +use xd_utilities; /** * Data set for batch export queries. @@ -66,6 +67,20 @@ class BatchDataset extends Loggable implements Iterator */ private $anonymousFields = []; + /** + * Salt used during deidentification. + * + * @var string + */ + private $hashSalt = ''; + + /** + * Cache for hashed values. + * + * @var array + */ + private $hashCache = []; + /** * @param mixed $name Description. * @param \XDUser $user @@ -79,6 +94,15 @@ public function __construct(RawQuery $query, XDUser $user, Log $logger = null) $this->docs = $query->getColumnDocumentation(); $this->dbh = DB::factory($query->_db_profile); + try { + $this->hashSalt = xd_utilities\getConfiguration( + 'data_warehouse_export', + 'hash_salt' + ); + } catch (Exception $e) { + $this->logger->warn('data_warehouse_export hash_salt is not set'); + } + foreach ($this->docs as $key => $doc) { $export = isset($doc['batch_export']) ? $doc['batch_export'] : false; $name = $doc['name']; @@ -183,6 +207,8 @@ public function rewind() } /** + * Is this iterator valid? + * * @return bool */ public function valid() @@ -191,13 +217,27 @@ public function valid() } /** + * Anonymize a field. + * + * @param string $field + * @return string */ private function anonymizeField($field) { - return md5($field); + if (array_key_exists($field, $this->hashCache)) { + return $this->hashCache[$field]; + } + + $hash = sha1($field . $this->hashSalt); + $this->hashCache[$field] = $hash; + + return $hash; } /** + * Get the next row of data. + * + * @return array */ private function getNextRow() { diff --git a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php index c6dad0cc19..dee346ccce 100644 --- a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php +++ b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php @@ -57,6 +57,7 @@ public function execute() $this->writePortalSettingsFile(array( 'data_warehouse_export_export_directory' => '/var/spool/xdmod/export', 'data_warehouse_export_retention_duration_days' => 30, + 'data_warehouse_export_hash_salt' => bin2hex(random_bytes(32)), 'features_novice_user' => $novice_user )); } diff --git a/classes/OpenXdmod/Setup/GeneralSetup.php b/classes/OpenXdmod/Setup/GeneralSetup.php index e9bb909358..1df443abf0 100644 --- a/classes/OpenXdmod/Setup/GeneralSetup.php +++ b/classes/OpenXdmod/Setup/GeneralSetup.php @@ -149,6 +149,10 @@ public function handle() array('on', 'off') ); + if (empty($settings['data_warehouse_export_hash_salt'])) { + $settings['data_warehouse_export_hash_salt'] = bin2hex(random_bytes(32)); + } + $this->saveIniConfig($settings, 'portal_settings'); } } diff --git a/configuration/portal_settings.ini b/configuration/portal_settings.ini index ae0480e2d1..79c950985b 100644 --- a/configuration/portal_settings.ini +++ b/configuration/portal_settings.ini @@ -157,3 +157,5 @@ sacct = "sacct" export_directory = "/var/spool/xdmod/export" ; Length of time in days that files will be retained before automatic deletion. retention_duration_days = 30 +; Salt used during deidentification. +hash_salt = "" diff --git a/templates/portal_settings.template b/templates/portal_settings.template index ef07c822b5..162c852020 100644 --- a/templates/portal_settings.template +++ b/templates/portal_settings.template @@ -157,3 +157,5 @@ sacct = "[:slurm_sacct:]" export_directory = "[:data_warehouse_export_export_directory:]" ; Length of time in days that files will be retained before automatic deletion. retention_duration_days = [:data_warehouse_export_retention_duration_days:] +; Salt used during deidentification. +hash_salt = "[:data_warehouse_export_hash_salt:]" From 3dfc000e64ef08caa43b5c0b72b72c1aeb814393 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 7 Aug 2019 11:55:21 -0400 Subject: [PATCH 189/217] Change debugging output --- classes/DataWarehouse/Data/BatchDataset.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/classes/DataWarehouse/Data/BatchDataset.php b/classes/DataWarehouse/Data/BatchDataset.php index f4122170d5..c66d19017d 100644 --- a/classes/DataWarehouse/Data/BatchDataset.php +++ b/classes/DataWarehouse/Data/BatchDataset.php @@ -126,8 +126,14 @@ public function __construct(RawQuery $query, XDUser $user, Log $logger = null) } } - $this->logger->debug(sprintf('Header: %s', json_encode($this->header))); - $this->logger->debug(sprintf('Anonymous fields: %s', json_encode($this->anonymousFields))); + $this->logger->debug(sprintf( + 'Header: %s', + json_encode(array_values($this->header)) + )); + $this->logger->debug(sprintf( + 'Anonymous fields: %s', + json_encode(array_keys($this->anonymousFields)) + )); } /** From a491691afdd50b8fc80489c7225b8f833aef72ef Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 9 Aug 2019 12:36:36 -0400 Subject: [PATCH 190/217] Quote field alias --- classes/DataWarehouse/Query/Model/Field.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Query/Model/Field.php b/classes/DataWarehouse/Query/Model/Field.php index 28c2b2fdd4..91f2495607 100644 --- a/classes/DataWarehouse/Query/Model/Field.php +++ b/classes/DataWarehouse/Query/Model/Field.php @@ -67,7 +67,7 @@ public function getQualifiedName($show_alias = false) $ret = $this->_def; if ($show_alias == true && $this->getAlias() != '') { - $ret .= ' as ' . $this->getAlias(); + $ret .= " as '" . $this->getAlias() . "'"; } return $ret; From 26a32bd9ed1e0898f1c22dd8187ac3a8b42f8b38 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Mon, 12 Aug 2019 07:09:30 -0400 Subject: [PATCH 191/217] Refactor configuration file format --- classes/DataWarehouse/Data/BatchDataset.php | 4 +- .../DataWarehouse/Query/Jobs/JobDataset.php | 109 ++-- configuration/rawstatistics.d/20_jobs.json | 545 +++++++++++------- 3 files changed, 387 insertions(+), 271 deletions(-) diff --git a/classes/DataWarehouse/Data/BatchDataset.php b/classes/DataWarehouse/Data/BatchDataset.php index c66d19017d..934e50ace3 100644 --- a/classes/DataWarehouse/Data/BatchDataset.php +++ b/classes/DataWarehouse/Data/BatchDataset.php @@ -104,7 +104,7 @@ public function __construct(RawQuery $query, XDUser $user, Log $logger = null) } foreach ($this->docs as $key => $doc) { - $export = isset($doc['batch_export']) ? $doc['batch_export'] : false; + $export = isset($doc['batchExport']) ? $doc['batchExport'] : false; $name = $doc['name']; if (isset($doc['units']) && $doc['units'] === 'ts') { @@ -120,7 +120,7 @@ public function __construct(RawQuery $query, XDUser $user, Log $logger = null) // Skip field. } else { throw new Exception(sprintf( - 'Unknown "batch_export" option %s', + 'Unknown "batchExport" option %s', var_export($export, true) )); } diff --git a/classes/DataWarehouse/Query/Jobs/JobDataset.php b/classes/DataWarehouse/Query/Jobs/JobDataset.php index f07ff33d2e..682c6b1126 100644 --- a/classes/DataWarehouse/Query/Jobs/JobDataset.php +++ b/classes/DataWarehouse/Query/Jobs/JobDataset.php @@ -1,11 +1,13 @@ getDataTable(); - $joblistTable = new Table($dataTable->getSchema(), $dataTable->getName() . "_joblist", "jl"); - $factTable = new Table(new Schema('modw'), 'job_tasks', 'jt'); + // The data table is always aliased to "jf". + $tables = ['jf' => $this->getDataTable()]; - $this->addTable($joblistTable); - $this->addTable($factTable); + foreach ($config['tables'] as $tableDef) { + $alias = $tableDef['alias']; + $table = new Table( + new Schema($tableDef['schema']), + $tableDef['name'], + $alias + ); + $tables[$alias] = $table; + $this->addTable($table); + + $join = $tableDef['join']; + $this->addWhereCondition(new WhereCondition( + new TableField($table, $join['primaryKey']), + '=', + new TableField($tables[$join['foreignTableAlias']], $join['foreignKey']) + )); + } - $this->addWhereCondition(new WhereCondition( - new TableField($joblistTable, "agg_id"), - "=", - new TableField($dataTable, "id") - )); - $this->addWhereCondition(new WhereCondition( - new TableField($joblistTable, "jobid"), - "=", - new TableField($factTable, "job_id") - )); + // This table is defined in the configuration file, but used in the section below. + $factTable = $tables['jt']; if (isset($parameters['primary_key'])) { $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'job_id'), "=", $parameters['primary_key'])); @@ -53,7 +61,7 @@ public function __construct( $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'local_job_id_raw'), '=', $matches[1])); } } else { - throw new \Exception('invalid "job_identifier" query parameter'); + throw new Exception('invalid "job_identifier" query parameter'); } } elseif (isset($parameters['start_date']) && isset($parameters['end_date'])) { $startDate = date_parse_from_format('Y-m-d', $parameters['start_date']); @@ -66,7 +74,7 @@ public function __construct( $startDate['year'] ); if ($startDateTs === false) { - throw new \Exception('invalid "start_date" query parameter'); + throw new Exception('invalid "start_date" query parameter'); } $endDate = date_parse_from_format('Y-m-d', $parameters['end_date']); @@ -79,46 +87,43 @@ public function __construct( $endDate['year'] ); if ($startDateTs === false) { - throw new \Exception('invalid "end_date" query parameter'); + throw new Exception('invalid "end_date" query parameter'); } $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_time_ts'), ">=", $startDateTs)); $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_time_ts'), "<=", $endDateTs)); } else { - throw new \Exception('invalid query parameters'); + throw new Exception('invalid query parameters'); } if ($stat == "accounting") { - $i = 0; - foreach ($config['modw.job_tasks'] as $sdata) { - $sfield = $sdata['key']; - if ($sdata['dtype'] == 'accounting') { - $this->addField(new TableField($factTable, $sfield)); - $this->documentation[$sfield] = $sdata; - } elseif ($sdata['dtype'] == 'foreignkey') { - if (isset($sdata['join'])) { - $info = $sdata['join']; - $i += 1; - $tmptable = new Table(new Schema($info['schema']), $info['table'], "ft$i"); - $this->addTable($tmptable); - $this->addWhereCondition(new WhereCondition(new TableField($factTable, $sfield), '=', new TableField($tmptable, "id"))); - $fcol = isset($info['column']) ? $info['column'] : 'name'; - $this->addField(new TableField($tmptable, $fcol, $sdata['name'])); - - $this->documentation[ $sdata['name'] ] = $sdata; + foreach ($config['fields'] as $field) { + // Replace hierarchy constants. + foreach (['name', 'documentation'] as $key) { + $value = $field[$key]; + if (strpos($value, 'HIERARCHY_') === 0 && defined($value)) { + $field[$key] = constant($value); } } + + $alias = $field['name']; + if (isset($field['tableAlias']) && isset($field['column'])) { + $this->addField(new TableField( + $tables[$field['tableAlias']], + $field['column'], + $alias + )); + } else if (isset($field['formula'])) { + $this->addField(new FormulaField($field['formula'], $alias)); + } else { + throw new Exception(sprintf( + 'Missing tableAlias and column or formula for "%s", definition: %s', + $alias, + json_encode($field) + )); + } + $this->documentation[$alias] = $field; } - $rf = new Table(new Schema('modw'), 'resourcefact', 'rf'); - $this->addTable($rf); - $this->addWhereCondition(new WhereCondition(new TableField($factTable, 'resource_id'), '=', new TableField($rf, 'id'))); - $this->addField(new TableField($rf, 'timezone')); - $this->documentation['timezone'] = array( - "name" => "Timezone", - "documentation" => "The timezone of the resource.", - "group" => "Administration", - 'visibility' => 'public', - "per" => "resource"); } else { $this->addField(new TableField($factTable, "job_id", "jobid")); $this->addField(new TableField($factTable, "local_jobid", "local_job_id")); diff --git a/configuration/rawstatistics.d/20_jobs.json b/configuration/rawstatistics.d/20_jobs.json index b3772fe1df..3362dde0e8 100644 --- a/configuration/rawstatistics.d/20_jobs.json +++ b/configuration/rawstatistics.d/20_jobs.json @@ -5,231 +5,342 @@ "display": "Jobs" } ], - "modw.job_tasks": [ - { - "key": "local_jobid", - "name": "Local Job Id", - "dtype": "accounting", - "group": "Administration", - "documentation": "The unique identifier assigned to the job by the job scheduler.", - "batch_export": true - }, - { - "key": "resource_id", - "name": "Resource", - "group": "Administration", - "dtype": "foreignkey", - "join": { + "Jobs": { + "tables": [ + { + "schema": "modw_aggregates", + "name": "jobfact_by_day_joblist", + "alias": "jl", + "join": { + "primaryKey": "agg_id", + "foreignTableAlias": "jf", + "foreignKey": "id" + } + }, + { "schema": "modw", - "table": "resourcefact" + "name": "job_tasks", + "alias": "jt", + "join": { + "primaryKey": "job_id", + "foreignTableAlias": "jl", + "foreignKey": "jobid" + } }, - "documentation": "The resource that ran the job.", - "batch_export": true - }, - { - "key": "systemaccount_id", - "name": "System Username", - "group": "Administration", - "dtype": "foreignkey", - "join": { + { "schema": "modw", - "table": "systemaccount", - "column": "username" + "name": "job_records", + "alias": "jr", + "join": { + "primaryKey": "job_record_id", + "foreignTableAlias": "jt", + "foreignKey": "job_record_id" + } }, - "documentation": "The username on the resource of the user that ran the job. May be a UID or string username depending on the resource.", - "batch_export": "anonymize" - }, - { - "key": "person_id", - "name": "User", - "group": "Administration", - "dtype": "foreignkey", - "join": { + { "schema": "modw", - "table": "person", - "column": "long_name" + "name": "resourcefact", + "alias": "rf", + "join": { + "primaryKey": "id", + "foreignTableAlias": "jt", + "foreignKey": "resource_id" + } }, - "documentation": "The name of the job owner.", - "batch_export": true - }, - { - "key": "person_organization_id", - "name": "Organization", - "group": "Administration", - "dtype": "foreignkey", - "join": { + { "schema": "modw", - "table": "organization" + "name": "systemaccount", + "alias": "sa", + "join": { + "primaryKey": "id", + "foreignTableAlias": "jt", + "foreignKey": "systemaccount_id" + } }, - "documentation": "The organization of the person who ran the task", - "batch_export": true - }, - { - "key": "name", - "name": "Name", - "documentation": "The name of the job as reported by the job scheduler.", - "dtype": "accounting", - "group": "Executable", - "batch_export": false - }, - { - "key": "submit_time_ts", - "name": "Submit Time", - "dtype": "accounting", - "group": "Timing", - "units": "ts", - "documentation": "Task submission time", - "batch_export": true - }, - { - "key": "start_time_ts", - "name": "Start Time", - "dtype": "accounting", - "group": "Timing", - "units": "ts", - "documentation": "The time that the job started running.", - "batch_export": true - }, - { - "key": "end_time_ts", - "name": "End Time", - "units": "ts", - "dtype": "accounting", - "group": "Timing", - "documentation": "The time that the job ended.", - "batch_export": true - }, - { - "key": "eligible_time_ts", - "name": "Eligible Time", - "units": "ts", - "dtype": "accounting", - "group": "Timing", - "documentation": "The time that the job was eligible for scheduling by the resource manager.", - "batch_export": true - }, - { - "key": "node_count", - "name": "Nodes", - "dtype": "foreignkey", - "group": "Allocated Resource", - "join": { + { "schema": "modw", - "table": "nodecount", - "column": "nodes" + "name": "person", + "alias": "p", + "join": { + "primaryKey": "id", + "foreignTableAlias": "jt", + "foreignKey": "person_id" + } }, - "documentation": "The number of nodes that were assigned to the job.", - "batch_export": true - }, - { - "key": "processor_count", - "name": "Cores", - "dtype": "accounting", - "group": "Allocated Resource", - "documentation": "The number of cores that were assigned to the job.", - "batch_export": true - }, - { - "key": "memory_kb", - "name": "Memory Used", - "dtype": "accounting", - "group": "Allocated resource", - "units": "kilobyte", - "documentation": "Memory consumed as reported by the resource manager.", - "batch_export": true - }, - { - "key": "wallduration", - "name": "Wall Time", - "dtype": "accounting", - "group": "Timing", - "units": "seconds", - "documentation": "Overall job duration.", - "batch_export": true - }, - { - "key": "waitduration", - "name": "Wait Time", - "dtype": "accounting", - "group": "Timing", - "units": "seconds", - "documentation": "Time the job waited in the queue", - "batch_export": true - }, - { - "key": "cpu_time", - "name": "Core Time", - "dtype": "accounting", - "group": "Allocated resource", - "units": "seconds", - "documentation": "The amount of CPU core time (Core Count * Wall Time)", - "batch_export": true - }, - { - "key": "group_name", - "name": "UNIX group name", - "dtype": "accounting", - "group": "Administration", - "documentation": "The name of the group that ran the job.", - "batch_export": false - }, - { - "key": "gid_number", - "name": "UNIX group GID", - "dtype": "accounting", - "group": "Administration", - "documentation": "The GID of the group that ran the job.", - "batch_export": false - }, - { - "key": "uid_number", - "name": "UNIX UID", - "dtype": "accounting", - "group": "Administration", - "documentation": "The UID of the user that ran the job.", - "batch_export": false - }, - { - "key": "exit_code", - "name": "Exit Code", - "dtype": "accounting", - "group": "Executable", - "documentation": "The code that the job exited with.", - "batch_export": true - }, - { - "key": "exit_state", - "name": "Exit State", - "dtype": "accounting", - "group": "Executable", - "documentation": "The state of the job when it completed.", - "batch_export": true - }, - { - "key": "cpu_req", - "name": "Requested Cores", - "dtype": "accounting", - "group": "Requested resource", - "documentation": "The number of CPUs required by the job.", - "batch_export": true - }, - { - "key": "mem_req", - "name": "Requested memory", - "dtype": "accounting", - "group": "Requested resource", - "units": "bytes", - "documentation": "The amount of memory required by the job.", - "batch_export": true - }, - { - "key": "timelimit", - "name": "Requested Wall Time", - "dtype": "accounting", - "group": "Requested resource", - "units": "seconds", - "documentation": "The time limit of the job.", - "batch_export": true - } - ] + { + "schema": "modw", + "name": "organization", + "alias": "o", + "join": { + "primaryKey": "id", + "foreignTableAlias": "jt", + "foreignKey": "person_organization_id" + } + }, + { + "schema": "modw", + "name": "nodecount", + "alias": "nc", + "join": { + "primaryKey": "id", + "foreignTableAlias": "jt", + "foreignKey": "node_count" + } + }, + { + "schema": "modw", + "name": "fieldofscience_hierarchy", + "alias": "fos", + "join": { + "primaryKey": "id", + "foreignTableAlias": "jr", + "foreignKey": "fos_id" + } + } + ], + "fields": [ + { + "name": "Local Job Id", + "formula": "IF(jt.local_job_array_index = -1, jt.local_jobid, CONCAT(jt.local_jobid, '[', jt.local_job_array_index, ']'))", + "group": "Administration", + "documentation": "The unique identifier assigned to the job by the job scheduler.", + "batchExport": true + }, + { + "name": "Resource", + "tableAlias": "rf", + "column": "name", + "group": "Administration", + "documentation": "The resource that ran the job.", + "batchExport": true + }, + { + "name": "Timezone", + "tableAlias": "rf", + "column": "timezone", + "group": "Administration", + "documentation": "The timezone of the resource.", + "batchExport": true + }, + { + "name": "System Username", + "tableAlias": "sa", + "column": "username", + "group": "Administration", + "visibility": "non-public", + "documentation": "The username on the resource of the user that ran the job. May be a UID or string username depending on the resource.", + "batchExport": "anonymize" + }, + { + "name": "User", + "tableAlias": "p", + "column": "long_name", + "group": "Administration", + "documentation": "The name of the job owner.", + "batchExport": true + }, + { + "name": "Organization", + "tableAlias": "o", + "column": "name", + "group": "Administration", + "documentation": "The organization of the person who ran the task", + "batchExport": true + }, + { + "name": "Name", + "tableAlias": "jt", + "column": "name", + "documentation": "The name of the job as reported by the job scheduler.", + "group": "Executable", + "batchExport": false + }, + { + "name": "Submit Time", + "tableAlias": "jt", + "column": "submit_time_ts", + "group": "Timing", + "units": "ts", + "documentation": "Task submission time", + "batchExport": true + }, + { + "name": "Start Time", + "tableAlias": "jt", + "column": "start_time_ts", + "group": "Timing", + "units": "ts", + "documentation": "The time that the job started running.", + "batchExport": true + }, + { + "name": "End Time", + "tableAlias": "jt", + "column": "end_time_ts", + "units": "ts", + "group": "Timing", + "documentation": "The time that the job ended.", + "batchExport": true + }, + { + "name": "Eligible Time", + "tableAlias": "jt", + "column": "eligible_time_ts", + "units": "ts", + "group": "Timing", + "documentation": "The time that the job was eligible for scheduling by the resource manager.", + "batchExport": true + }, + { + "name": "Nodes", + "tableAlias": "nc", + "column": "nodes", + "group": "Allocated Resource", + "documentation": "The number of nodes that were assigned to the job.", + "batchExport": true + }, + { + "name": "Cores", + "tableAlias": "jt", + "column": "processor_count", + "group": "Allocated Resource", + "documentation": "The number of cores that were assigned to the job.", + "batchExport": true + }, + { + "name": "Memory Used", + "tableAlias": "jt", + "column": "memory_kb", + "group": "Allocated Resource", + "units": "kilobyte", + "documentation": "Memory consumed as reported by the resource manager.", + "batchExport": true + }, + { + "name": "Wall Time", + "tableAlias": "jt", + "column": "wallduration", + "group": "Timing", + "units": "seconds", + "documentation": "Overall job duration.", + "batchExport": true + }, + { + "name": "Wait Time", + "tableAlias": "jt", + "column": "waitduration", + "group": "Timing", + "units": "seconds", + "documentation": "Time the job waited in the queue", + "batchExport": true + }, + { + "name": "Core Time", + "tableAlias": "jt", + "column": "cpu_time", + "group": "Allocated Resource", + "units": "seconds", + "documentation": "The amount of CPU core time (Core Count * Wall Time)", + "batchExport": true + }, + { + "name": "UNIX group name", + "tableAlias": "jt", + "column": "group_name", + "group": "Administration", + "documentation": "The name of the group that ran the job.", + "batchExport": false + }, + { + "name": "UNIX group GID", + "tableAlias": "jt", + "column": "gid_number", + "group": "Administration", + "documentation": "The GID of the group that ran the job.", + "batchExport": false + }, + { + "name": "UNIX UID", + "tableAlias": "jt", + "column": "uid_number", + "group": "Administration", + "documentation": "The UID of the user that ran the job.", + "batchExport": false + }, + { + "name": "Exit Code", + "tableAlias": "jt", + "column": "exit_code", + "group": "Executable", + "documentation": "The code that the job exited with.", + "batchExport": true + }, + { + "name": "Exit State", + "tableAlias": "jt", + "column": "exit_state", + "group": "Executable", + "documentation": "The state of the job when it completed.", + "batchExport": true + }, + { + "name": "Requested Cores", + "tableAlias": "jt", + "column": "cpu_req", + "group": "Requested Resource", + "documentation": "The number of CPUs required by the job.", + "batchExport": true + }, + { + "name": "Requested memory", + "tableAlias": "jt", + "column": "mem_req", + "group": "Requested Resource", + "units": "bytes", + "documentation": "The amount of memory required by the job.", + "batchExport": true + }, + { + "name": "Requested Wall Time", + "tableAlias": "jt", + "column": "timelimit", + "group": "Requested Resource", + "units": "seconds", + "documentation": "The time limit of the job.", + "batchExport": true + }, + { + "name": "Queue", + "tableAlias": "jr", + "column": "queue", + "group": "Requested Resource", + "documentation": "The name of the queue to which the job was submitted.", + "batchExport": true + }, + { + "name": "HIERARCHY_TOP_LEVEL_LABEL", + "tableAlias": "fos", + "column": "directorate_description", + "group": "Requested Resource", + "documentation": "HIERARCHY_TOP_LEVEL_INFO", + "batchExport": true + }, + { + "name": "HIERARCHY_MIDDLE_LEVEL_LABEL", + "tableAlias": "fos", + "column": "parent_description", + "group": "Requested Resource", + "documentation": "HIERARCHY_MIDDLE_LEVEL_INFO", + "batchExport": true + }, + { + "name": "HIERARCHY_BOTTOM_LEVEL_LABEL", + "tableAlias": "fos", + "column": "description", + "group": "Requested Resource", + "documentation": "HIERARCHY_BOTTOM_LEVEL_INFO", + "batchExport": true + } + ] + } } From a52bca28203d10052ab9e26392bc51006c7ed923 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 13 Aug 2019 14:03:22 -0400 Subject: [PATCH 192/217] Make function protected --- classes/DataWarehouse/Query/Query.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Query/Query.php b/classes/DataWarehouse/Query/Query.php index 97e4cb39a8..03d0a2b8d8 100644 --- a/classes/DataWarehouse/Query/Query.php +++ b/classes/DataWarehouse/Query/Query.php @@ -676,7 +676,7 @@ public function cloneParameters(Query $other) $this->roleParameterDescriptions = $other->roleParameterDescriptions; } - private function getLeftJoinSql() + protected function getLeftJoinSql() { $stmt = ''; foreach ($this->leftJoins as $joincond) { From 251a9a1ae5928e51b8977375d8c9b51aa26d1201 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 13 Aug 2019 14:25:28 -0400 Subject: [PATCH 193/217] Fix style --- classes/DataWarehouse/Query/Jobs/JobDataset.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Query/Jobs/JobDataset.php b/classes/DataWarehouse/Query/Jobs/JobDataset.php index 682c6b1126..bfcd774918 100644 --- a/classes/DataWarehouse/Query/Jobs/JobDataset.php +++ b/classes/DataWarehouse/Query/Jobs/JobDataset.php @@ -113,7 +113,7 @@ public function __construct( $field['column'], $alias )); - } else if (isset($field['formula'])) { + } elseif (isset($field['formula'])) { $this->addField(new FormulaField($field['formula'], $alias)); } else { throw new Exception(sprintf( From 01b08c99de8ab9692a73cdc2ab54e90159482f86 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 15 Aug 2019 08:29:21 -0400 Subject: [PATCH 194/217] Fix logging --- classes/DataWarehouse/Data/BatchDataset.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Data/BatchDataset.php b/classes/DataWarehouse/Data/BatchDataset.php index 934e50ace3..e6bfa48c7e 100644 --- a/classes/DataWarehouse/Data/BatchDataset.php +++ b/classes/DataWarehouse/Data/BatchDataset.php @@ -100,7 +100,7 @@ public function __construct(RawQuery $query, XDUser $user, Log $logger = null) 'hash_salt' ); } catch (Exception $e) { - $this->logger->warn('data_warehouse_export hash_salt is not set'); + $this->logger->warning('data_warehouse_export hash_salt is not set'); } foreach ($this->docs as $key => $doc) { From c395aa4422a2f70fb269180e7bf70fcaf6c889e9 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 15 Aug 2019 10:55:23 -0400 Subject: [PATCH 195/217] Change realm name field --- classes/DataWarehouse/Export/RealmManager.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/classes/DataWarehouse/Export/RealmManager.php b/classes/DataWarehouse/Export/RealmManager.php index 55ba0e2eb6..a9507b3d85 100644 --- a/classes/DataWarehouse/Export/RealmManager.php +++ b/classes/DataWarehouse/Export/RealmManager.php @@ -45,11 +45,11 @@ public function __construct() */ public function getRealms() { - // The "display" values from rawstatistics match those in - // moddb.realms.display`, but the "name" values do not. + // The "name" values from rawstatistics match those in + // moddb.realms.display`. $exportable = array_map( function ($realm) { - return $realm['display']; + return $realm['name']; }, $this->config['realms'] ); From 0b7fcb0c9da83fe7a92e17bfc37ba24482916461 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Fri, 16 Aug 2019 07:01:48 -0400 Subject: [PATCH 196/217] Add batch export option to ETL profile --- etl/js/lib/etl_profile.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etl/js/lib/etl_profile.js b/etl/js/lib/etl_profile.js index f121ac4dd3..4e00f77f7d 100644 --- a/etl/js/lib/etl_profile.js +++ b/etl/js/lib/etl_profile.js @@ -713,6 +713,7 @@ ETLProfile.prototype.integrateWithXDMoD = function () { var dtype = columns[c].dtype ? columns[c].dtype : (columns[c].queries ? "foreignkey" : "statistic" ); var group = columns[c].group ? columns[c].group : "misc"; var visibility = columns[c].visibility ? columns[c].visibility : 'public'; + var batchExport = columns[c].batchExport ? columns[c].batchExport : false; var name = extractandsubst(columns[c], "name"); if(!name) { @@ -727,6 +728,7 @@ ETLProfile.prototype.integrateWithXDMoD = function () { documentation: columns[c].comments, dtype: dtype, visibility: visibility, + batchExport: batchExport, group: group }); } From 0acc8054e23cbf8a3602db27df758498f394a7f1 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Sun, 18 Aug 2019 14:01:30 -0400 Subject: [PATCH 197/217] Add data warehouse export configuration --- .../Version812To850/ConfigFilesMigration.php | 22 +- classes/OpenXdmod/Setup/GeneralSetup.php | 4 - .../OpenXdmod/Setup/WarehouseExportSetup.php | 225 ++++++++++++++++++ configuration/setup.json | 5 + tests/ci/scripts/xdmod-setup.tcl | 6 + tests/ci/scripts/xdmod-upgrade.tcl | 2 + 6 files changed, 255 insertions(+), 9 deletions(-) create mode 100644 classes/OpenXdmod/Setup/WarehouseExportSetup.php diff --git a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php index e3495041d9..da5f58b056 100644 --- a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php +++ b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php @@ -9,6 +9,8 @@ use CCR\Json; use OpenXdmod\Migration\ConfigFilesMigration as AbstractConfigFilesMigration; use OpenXdmod\Setup\Console; +use OpenXdmod\Setup\WarehouseExportSetup; +use xd_utilities; class ConfigFilesMigration extends AbstractConfigFilesMigration { @@ -92,11 +94,21 @@ public function execute() array('on', 'off') ); - $this->writePortalSettingsFile(array( - 'data_warehouse_export_export_directory' => '/var/spool/xdmod/export', - 'data_warehouse_export_retention_duration_days' => 30, - 'data_warehouse_export_hash_salt' => bin2hex(random_bytes(32)), - 'features_novice_user' => $novice_user + $console->displayMessage(<<<"EOT" +This release of XDMoD includes support for batch exporting of data from the +data warehouse. +EOT + ); + $console->displayBlankLine(); + $exportSetup = new OpenXdmod\Setup\WarehouseExportSetup($console); + $exportSettings = $exportSetup->promptForSettings([ + 'data_warehouse_export_export_directory' => xd_utilities\getConfiguration('data_warehouse_export', 'export_directory'), + 'data_warehouse_export_retention_duration_days' => xd_utilities\getConfiguration('data_warehouse_export', 'retention_duration_days') + ]); + + $this->writePortalSettingsFile(array_merge( + ['features_novice_user' => $novice_user], + $exportSettings )); } diff --git a/classes/OpenXdmod/Setup/GeneralSetup.php b/classes/OpenXdmod/Setup/GeneralSetup.php index 1df443abf0..e9bb909358 100644 --- a/classes/OpenXdmod/Setup/GeneralSetup.php +++ b/classes/OpenXdmod/Setup/GeneralSetup.php @@ -149,10 +149,6 @@ public function handle() array('on', 'off') ); - if (empty($settings['data_warehouse_export_hash_salt'])) { - $settings['data_warehouse_export_hash_salt'] = bin2hex(random_bytes(32)); - } - $this->saveIniConfig($settings, 'portal_settings'); } } diff --git a/classes/OpenXdmod/Setup/WarehouseExportSetup.php b/classes/OpenXdmod/Setup/WarehouseExportSetup.php new file mode 100644 index 0000000000..61f8717c63 --- /dev/null +++ b/classes/OpenXdmod/Setup/WarehouseExportSetup.php @@ -0,0 +1,225 @@ + + */ + +namespace OpenXdmod\Setup; + +use Exception; + +/** + * Data warehouse batch export setup. + */ +class WarehouseExportSetup extends SetupItem +{ + /** + * Configure data warehouse export. + * + * @see \OpenXdmod\Setup\SetupItem::handle() + */ + public function handle() + { + $this->console->displaySectionHeader('Data Warehouse Batch Export'); + $newSettings = $this->promptForSettings($this->loadIniConfig('portal_settings')); + $this->saveIniConfig($newSettings, 'portal_settings'); + } + + /** + * Prompt user for settings. + * + * This function is public so that it may also be used during the upgrade + * process. + * + * @param array $settings Current settings. + * @return array New settings. + */ + public function promptForSettings(array $settings) + { + $this->console->displayMessage(<<<'MSG' +The data warehouse batch export feature allows users to create requests to +export data which is then generated by a cron job and stored on the server. +The directory where this data is stored and the duration that the data will be +retained are configurable. +MSG + ); + $this->console->displayBlankLine(); + $exportDir = $this->console->prompt( + 'Export Directory:', + $settings['data_warehouse_export_export_directory'] + ); + + try { + $this->checkExportDirectory($exportDir); + } catch (Exception $e) { + $this->console->displayMessage('There was an error while updating the export directory: ' . $e->getMessage()); + $this->console->displayMessage('You must manually create the directory and set permissions'); + } + $settings['data_warehouse_export_export_directory'] = $exportDir; + + $settings['data_warehouse_export_retention_duration_days'] = $this->promptForRetentionDuration($settings['data_warehouse_export_retention_duration_days']); + + if (empty($settings['data_warehouse_export_hash_salt'])) { + $settings['data_warehouse_export_hash_salt'] = bin2hex(random_bytes(32)); + } + + return $settings; + } + + /** + * Prompt the user for the export file retention duration. + * + * @param int $defaultDuration The default retention duration. + * @return int The user's response. + */ + private function promptForRetentionDuration($defaultDuration) + { + $haveResponse = false; + $retentionDuration = $defaultDuration; + + while (!$haveResponse) { + $retentionDuration = $this->console->prompt( + 'Export File Retention Duration (Days):', + $defaultDuration + ); + + if (filter_var($retentionDuration, FILTER_VALIDATE_INT) !== false + && $retentionDuration > 0) { + $haveResponse = true; + } else { + $this->console->displayMessage('The export file retention duration must be a positive integer.'); + } + } + + return $retentionDuration; + } + + /** + * Check the export directory. + * + * Checks that the export directory exists, has the correct ownership and + * permissions. Prompts the user if the directory needs to be created or + * changed. + * + * @param string $dir Path of the export directory. + */ + private function checkExportDirectory($dir) + { + // Desired attributes. + $desiredPerms = 0570; + $desiredUser = 'apache'; + $desiredGroup = 'xdmod'; + + $this->console->displayMessage(<<<"MSG" +If the export directory does not exist, it must be created and assigned the +correct permissions and ownership. It must be readable by the web server and +both readable and writable by the user that is used to generate the export +files. By default, the web server user is expected to be {$desiredUser} and +the group is expected to be {$desiredGroup}. If your system uses a different +user and group then the automatic process will fail and you must set the +permissions manually. +MSG + ); + $this->console->displayBlankLine(); + + if (!is_dir($dir)) { + $response = $this->console->prompt( + 'Export directory does not exist. Create and set permissions?', + 'yes', + ['yes', 'no'] + ); + if ($response !== 'yes') { + return; + } + if (!mkdir($dir, $desiredPerms, true)) { + throw new Exception(sprintf( + 'Failed to create directory "%s"', + $dir + )); + } + if (!chmod($dir, $desiredPerms)) { + throw new Exception(sprintf( + 'Failed to change permissions of "%s"', + $dir + )); + } + if (!chown($dir, $desiredUser)) { + throw new Exception(sprintf( + 'Failed to change owner of "%s"', + $dir + )); + } + if (!chgrp($dir, $desiredGroup)) { + throw new Exception(sprintf( + 'Failed to change group of "%s"', + $dir + )); + } + } + + $perms = fileperms($dir) & 0777; + if ($perms != $desiredPerms) { + $this->console->displayMessage(sprintf( + 'Directory permissions are "%o", expected "%o".', + $perms, + $desiredPerms + )); + $response = $this->console->prompt( + 'Update permissions?', + 'yes', + ['yes', 'no'] + ); + if ($response != 'no') { + if (!chmod($dir, $desiredPerms)) { + throw new Exception(sprintf( + 'Failed to change permissions of "%s"', + $dir + )); + } + } + } + + $user = posix_getpwuid(fileowner($dir))['name']; + if ($user != $desiredUser) { + $this->console->displayMessage(sprintf( + 'Directory owner is "%s", expected "%s".', + $user, + $desiredUser + )); + $response = $this->console->prompt( + 'Update owner?', + 'yes', + ['yes', 'no'] + ); + if ($response != 'no') { + if (!chown($dir, $desiredUser)) { + throw new Exception(sprintf( + 'Failed to change owner of "%s"', + $dir + )); + } + } + } + + $group = posix_getgrgid(filegroup($dir))['name']; + if ($group != $desiredGroup) { + $this->console->displayMessage(sprintf( + 'Directory group is "%s", expected "%s".', + $group, + $desiredGroup + )); + $response = $this->console->prompt( + 'Update group?', + 'yes', + ['yes', 'no'] + ); + if ($response != 'no') { + if (!chgrp($dir, $desiredGroup)) { + throw new Exception(sprintf( + 'Failed to change group of "%s"', + $dir + )); + } + } + } + } +} diff --git a/configuration/setup.json b/configuration/setup.json index 0a1b7711e7..cbd81b5d04 100644 --- a/configuration/setup.json +++ b/configuration/setup.json @@ -30,6 +30,11 @@ "label": "Hierarchy", "handler": "HierarchySetup" }, + { + "position": 65, + "label": "Data Warehouse Batch Export", + "handler": "WarehouseExportSetup" + }, { "position": 70, "label": "Automatically Check for Updates", diff --git a/tests/ci/scripts/xdmod-setup.tcl b/tests/ci/scripts/xdmod-setup.tcl index 4bb0ec86df..a6296f7f05 100644 --- a/tests/ci/scripts/xdmod-setup.tcl +++ b/tests/ci/scripts/xdmod-setup.tcl @@ -87,6 +87,12 @@ provideInput {Bottom Level Description:} {PI Group} confirmFileWrite yes enterToContinue +selectMenuOption 7 +provideInput {Export Directory:} {} +provideInput {Export File Retention Duration (Days):} {31} +confirmFileWrite yes +enterToContinue + selectMenuOption q lassign [wait] pid spawnid os_error_flag value diff --git a/tests/ci/scripts/xdmod-upgrade.tcl b/tests/ci/scripts/xdmod-upgrade.tcl index 1ccb40b84a..81cdbdde43 100644 --- a/tests/ci/scripts/xdmod-upgrade.tcl +++ b/tests/ci/scripts/xdmod-upgrade.tcl @@ -22,6 +22,8 @@ set timeout 180 spawn "xdmod-upgrade" confirmUpgrade provideInput {Enable Novice User Tab*} {off} +provideInput {Export Directory:} {} +provideInput {Export File Retention Duration (Days):} {31} expect { -re "\nDo you want to run aggregation now.*\\\]" { send yes\n From 3cd72e7f58e98c915056931a37c5022a9b4513a6 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Sun, 18 Aug 2019 14:27:33 -0400 Subject: [PATCH 198/217] Replace tabs with spaces --- .../Migration/Version812To850/ConfigFilesMigration.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php index da5f58b056..76aa1dcf24 100644 --- a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php +++ b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php @@ -100,7 +100,7 @@ public function execute() EOT ); $console->displayBlankLine(); - $exportSetup = new OpenXdmod\Setup\WarehouseExportSetup($console); + $exportSetup = new OpenXdmod\Setup\WarehouseExportSetup($console); $exportSettings = $exportSetup->promptForSettings([ 'data_warehouse_export_export_directory' => xd_utilities\getConfiguration('data_warehouse_export', 'export_directory'), 'data_warehouse_export_retention_duration_days' => xd_utilities\getConfiguration('data_warehouse_export', 'retention_duration_days') From a6ec73a3cf4151ab9f8ff6f89dd69fb1bb7cae8d Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Tue, 20 Aug 2019 08:08:06 -0400 Subject: [PATCH 199/217] Update role configuration --- configuration/roles.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/configuration/roles.json b/configuration/roles.json index e417934519..57b7528f6e 100644 --- a/configuration/roles.json +++ b/configuration/roles.json @@ -30,6 +30,15 @@ "userManualSectionName": "Metric Explorer", "tooltip": "" }, + { + "name": "data_export", + "title": "Data Export", + "position": 500, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "userManualSectionName": "Data Export", + "tooltip": "Data Warehouse Batch Export" + }, { "name": "report_generator", "title": "Report Generator", @@ -57,15 +66,6 @@ "javascriptReference": "CCR.xdmod.ui.aboutXD", "userManualSectionName": "About", "tooltip": "" - }, - { - "name": "data_export", - "title": "Data Export", - "position": 400, - "javascriptClass": "XDMoD.Module.DataExport", - "javascriptReference": "CCR.xdmod.ui.dataExport", - "userManualSectionName": "Data Export", - "tooltip": "" } ], "query_descripters": [], From 8353e039dea1c1c46451a66227bd361ce4d72020 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 21 Aug 2019 07:17:46 -0400 Subject: [PATCH 200/217] Add error checking --- classes/DataWarehouse/Export/FileManager.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/classes/DataWarehouse/Export/FileManager.php b/classes/DataWarehouse/Export/FileManager.php index e8559150ff..6d9180bf9c 100644 --- a/classes/DataWarehouse/Export/FileManager.php +++ b/classes/DataWarehouse/Export/FileManager.php @@ -55,6 +55,20 @@ public function __construct(Log $logger = null) ]); throw new Exception('Export directory is not configured', 0, $e); } + + if (!is_dir($this->exportDir)) { + throw new Exception(sprintf( + 'Export directory "%s" does not exist', + $this->exportDir + )); + } + + if (!is_readable($this->exportDir)) { + throw new Exception(sprintf( + 'Export directory "%s" is not readable', + $this->exportDir + )); + } } /** From c5ad36515e25932313adee7883281130fa7953ca Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 21 Aug 2019 07:27:49 -0400 Subject: [PATCH 201/217] Fix comment grammar and add more error checking --- classes/DataWarehouse/Export/FileManager.php | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/classes/DataWarehouse/Export/FileManager.php b/classes/DataWarehouse/Export/FileManager.php index 6d9180bf9c..cae7f8c15e 100644 --- a/classes/DataWarehouse/Export/FileManager.php +++ b/classes/DataWarehouse/Export/FileManager.php @@ -214,10 +214,10 @@ public function createZipFile($dataFile, array $request) } // Override the name of the temporary data file with the proper - // name that will, be used in the archive file. + // name that will be used in the archive file. $localName = $this->getDataFileName($request); - if ($zip->addFile($dataFile, $localName) === false) { + if (!$zip->addFile($dataFile, $localName)) { throw new Exception(sprintf( 'Failed to add file "%s" to zip file "%s"', $dataFile, @@ -225,7 +225,12 @@ public function createZipFile($dataFile, array $request) )); } - $zip->close(); + if (!$zip->close()) { + throw new Exception(sprintf( + 'Failed to close zip file "%s"', + $zipFile + )); + } return $zipFile; } catch (Exception $e) { From 5c206e3b70568897369285c633d3b2fef46b4cb8 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 21 Aug 2019 08:41:17 -0400 Subject: [PATCH 202/217] Remove commented-out code --- classes/DataWarehouse/Data/BatchDataset.php | 22 --------------------- html/gui/js/modules/DataExport.js | 12 ----------- 2 files changed, 34 deletions(-) diff --git a/classes/DataWarehouse/Data/BatchDataset.php b/classes/DataWarehouse/Data/BatchDataset.php index e6bfa48c7e..c4b523069d 100644 --- a/classes/DataWarehouse/Data/BatchDataset.php +++ b/classes/DataWarehouse/Data/BatchDataset.php @@ -186,28 +186,6 @@ public function rewind() { $this->logger->debug('Executing query'); $this->sth = $this->query->getRawStatement(); - // Set query to be unbuffered so results are not all loaded into memory. - // @see ETL\Ingestor\pdoIngestor::multiDatabaseIngest() - /* - $this->sth->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); - $result = $this->dbh->query( - "SHOW SESSION VARIABLES WHERE Variable_name = 'net_write_timeout'" - ); - - $currentTimeout = 0; - if ( 0 != count($result) ) { - $currentTimeout = $result[0]['Value']; - $this->logger->debug("Current net_write_timeout = $currentTimeout"); - } - - $newTimeout = $numDestinationTables * $this->netWriteTimeoutSecondsPerFileChunk; - - if ( $newTimeout > $currentTimeout ) { - $sql = sprintf('SET SESSION net_write_timeout = %d', $newTimeout); - $this->executeSqlList(array($sql), $this->sourceEndpoint); - } - */ - $this->currentRowIndex = 1; $this->currentRow = $this->getNextRow(); } diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js index ffcea2772d..8be630f199 100644 --- a/html/gui/js/modules/DataExport.js +++ b/html/gui/js/modules/DataExport.js @@ -439,19 +439,7 @@ XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, { disabled: true, scope: this, handler: this.deleteExpiredRequests - }, - '->' - /* - , - { - xtype: 'paging', - store: this.store, - pageSize: this.pageSize, - displayInfo: true, - displayMsg: 'Displaying export requests {0} - {1} of {2}', - emptyMsg: 'No export requests to display' } - */ ] }); From 9247ec2058d79298c177e50c49a7dcda5b13e0a8 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 21 Aug 2019 09:18:30 -0400 Subject: [PATCH 203/217] Change raw query stat name --- classes/DataWarehouse/Export/BatchProcessor.php | 2 +- classes/DataWarehouse/Query/Jobs/JobDataset.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php index c46d4dc71b..ef3c2e3d23 100644 --- a/classes/DataWarehouse/Export/BatchProcessor.php +++ b/classes/DataWarehouse/Export/BatchProcessor.php @@ -235,7 +235,7 @@ private function getDataSet(array $request, XDUser $user) 'start_date' => $request['start_date'], 'end_date' => $request['end_date'] ], - 'accounting' + 'batch' ); $dataSet = new BatchDataset($query, $user, $this->logger); return $dataSet; diff --git a/classes/DataWarehouse/Query/Jobs/JobDataset.php b/classes/DataWarehouse/Query/Jobs/JobDataset.php index bfcd774918..0cf833823e 100644 --- a/classes/DataWarehouse/Query/Jobs/JobDataset.php +++ b/classes/DataWarehouse/Query/Jobs/JobDataset.php @@ -96,7 +96,7 @@ public function __construct( throw new Exception('invalid query parameters'); } - if ($stat == "accounting") { + if ($stat == "accounting" || $stat == 'batch') { foreach ($config['fields'] as $field) { // Replace hierarchy constants. foreach (['name', 'documentation'] as $key) { From 46349392341bd2d0812c2b372c074dd8745d64a6 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 21 Aug 2019 10:19:29 -0400 Subject: [PATCH 204/217] Change comments --- classes/DataWarehouse/Export/RealmManager.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/classes/DataWarehouse/Export/RealmManager.php b/classes/DataWarehouse/Export/RealmManager.php index a9507b3d85..c0629ec31f 100644 --- a/classes/DataWarehouse/Export/RealmManager.php +++ b/classes/DataWarehouse/Export/RealmManager.php @@ -1,7 +1,4 @@ Date: Wed, 21 Aug 2019 11:03:17 -0400 Subject: [PATCH 205/217] Add debugging --- classes/DataWarehouse/Data/BatchDataset.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/classes/DataWarehouse/Data/BatchDataset.php b/classes/DataWarehouse/Data/BatchDataset.php index c4b523069d..9e0a1fc3f3 100644 --- a/classes/DataWarehouse/Data/BatchDataset.php +++ b/classes/DataWarehouse/Data/BatchDataset.php @@ -186,6 +186,10 @@ public function rewind() { $this->logger->debug('Executing query'); $this->sth = $this->query->getRawStatement(); + $this->logger->debug(sprintf( + 'Raw query string: %s', + $this->sth->queryString + )); $this->currentRowIndex = 1; $this->currentRow = $this->getNextRow(); } From 1b3281bc6f80fe02c1e29225a08ab8a366e260c4 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 21 Aug 2019 13:29:28 -0400 Subject: [PATCH 206/217] Use explicit time zone and start time --- classes/DataWarehouse/Query/Jobs/JobDataset.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/classes/DataWarehouse/Query/Jobs/JobDataset.php b/classes/DataWarehouse/Query/Jobs/JobDataset.php index 0cf833823e..2212fdf08d 100644 --- a/classes/DataWarehouse/Query/Jobs/JobDataset.php +++ b/classes/DataWarehouse/Query/Jobs/JobDataset.php @@ -64,11 +64,12 @@ public function __construct( throw new Exception('invalid "job_identifier" query parameter'); } } elseif (isset($parameters['start_date']) && isset($parameters['end_date'])) { + date_default_timezone_set('UTC'); $startDate = date_parse_from_format('Y-m-d', $parameters['start_date']); $startDateTs = mktime( - $startDate['hour'], - $startDate['minute'], - $startDate['second'], + 0, + 0, + 0, $startDate['month'], $startDate['day'], $startDate['year'] From 90b6597d93138b060b33e5eb8c357e4500e065b8 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 21 Aug 2019 13:58:54 -0400 Subject: [PATCH 207/217] Improve comment --- classes/DataWarehouse/Export/RealmManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/DataWarehouse/Export/RealmManager.php b/classes/DataWarehouse/Export/RealmManager.php index c0629ec31f..f72889061c 100644 --- a/classes/DataWarehouse/Export/RealmManager.php +++ b/classes/DataWarehouse/Export/RealmManager.php @@ -45,7 +45,7 @@ public function __construct() public function getRealms() { // The "name" values from rawstatistics match those in - // moddb.realms.display`. + // the moddb.realms.display column. $exportable = array_map( function ($realm) { return $realm['name']; From 2faf124ce52bc6597f651532baa1592e6670af10 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Wed, 21 Aug 2019 15:24:01 -0400 Subject: [PATCH 208/217] More debugging --- classes/DataWarehouse/Data/BatchDataset.php | 1 + 1 file changed, 1 insertion(+) diff --git a/classes/DataWarehouse/Data/BatchDataset.php b/classes/DataWarehouse/Data/BatchDataset.php index 9e0a1fc3f3..18860cd292 100644 --- a/classes/DataWarehouse/Data/BatchDataset.php +++ b/classes/DataWarehouse/Data/BatchDataset.php @@ -190,6 +190,7 @@ public function rewind() 'Raw query string: %s', $this->sth->queryString )); + $this->logger->debug(sprintf('Row count: %s', $this->sth->rowCount())); $this->currentRowIndex = 1; $this->currentRow = $this->getNextRow(); } From 54160d61ab80a4a0dcd8d89767ea516963da2a84 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 22 Aug 2019 07:45:44 -0400 Subject: [PATCH 209/217] Add cron job --- configuration/cron.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configuration/cron.conf b/configuration/cron.conf index d052029151..a06ccead0a 100644 --- a/configuration/cron.conf +++ b/configuration/cron.conf @@ -1,6 +1,9 @@ # Every morning at 3:00 AM -- run the report scheduler 0 3 * * * xdmod /usr/bin/php /usr/lib/xdmod/report_schedule_manager.php >/dev/null +# Process data warehouse batch export requests. +0 4 * * * xdmod /usr/lib/xdmod/batch_export_manager.php -q + # Check for updates (monthly). 0 1 1 * * xdmod /usr/lib/xdmod/update_check.php >/dev/null From ecc3f9a70652ffaba2b24f434b0af5e0b60b4e4f Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 22 Aug 2019 09:10:14 -0400 Subject: [PATCH 210/217] Update test artifacts --- tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json | 2 +- .../xdmod/user_admin/output/get_tabs-centerdirector.json | 2 +- .../artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json | 2 +- .../artifacts/xdmod/user_admin/output/get_tabs-normaluser.json | 2 +- tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json | 2 +- .../xdmod/user_admin/output/get_tabs-test.cd.one-center.json | 2 +- .../xdmod/user_admin/output/get_tabs-test.cs.one-center.json | 2 +- .../xdmod/user_admin/output/get_tabs-test.normal-user.json | 2 +- tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json | 2 +- .../xdmod/user_admin/output/get_tabs-test.usr_dev.json | 2 +- .../xdmod/user_admin/output/get_tabs-test.usr_mgr.json | 2 +- .../xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json index 8dbb7eda00..b807cb062b 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json @@ -36,7 +36,7 @@ "tab": "data_export", "isDefault": false, "title": "Data Export", - "pos": 400, + "pos": 500, "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json index 8dbb7eda00..b807cb062b 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json @@ -36,7 +36,7 @@ "tab": "data_export", "isDefault": false, "title": "Data Export", - "pos": 400, + "pos": 500, "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json index 8dbb7eda00..b807cb062b 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json @@ -36,7 +36,7 @@ "tab": "data_export", "isDefault": false, "title": "Data Export", - "pos": 400, + "pos": 500, "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json index 8dbb7eda00..b807cb062b 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json @@ -36,7 +36,7 @@ "tab": "data_export", "isDefault": false, "title": "Data Export", - "pos": 400, + "pos": 500, "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json index 8dbb7eda00..b807cb062b 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json @@ -36,7 +36,7 @@ "tab": "data_export", "isDefault": false, "title": "Data Export", - "pos": 400, + "pos": 500, "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json index 8dbb7eda00..b807cb062b 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json @@ -36,7 +36,7 @@ "tab": "data_export", "isDefault": false, "title": "Data Export", - "pos": 400, + "pos": 500, "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json index 8dbb7eda00..b807cb062b 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json @@ -36,7 +36,7 @@ "tab": "data_export", "isDefault": false, "title": "Data Export", - "pos": 400, + "pos": 500, "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json index 8dbb7eda00..b807cb062b 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json @@ -36,7 +36,7 @@ "tab": "data_export", "isDefault": false, "title": "Data Export", - "pos": 400, + "pos": 500, "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json index 8dbb7eda00..b807cb062b 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json @@ -36,7 +36,7 @@ "tab": "data_export", "isDefault": false, "title": "Data Export", - "pos": 400, + "pos": 500, "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json index 8dbb7eda00..b807cb062b 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json @@ -36,7 +36,7 @@ "tab": "data_export", "isDefault": false, "title": "Data Export", - "pos": 400, + "pos": 500, "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json index 8dbb7eda00..b807cb062b 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json @@ -36,7 +36,7 @@ "tab": "data_export", "isDefault": false, "title": "Data Export", - "pos": 400, + "pos": 500, "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json index 8dbb7eda00..b807cb062b 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json @@ -36,7 +36,7 @@ "tab": "data_export", "isDefault": false, "title": "Data Export", - "pos": 400, + "pos": 500, "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", From 94f6cb1190c2aa4260a1e0fc13f2e9132f78e186 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 22 Aug 2019 09:22:39 -0400 Subject: [PATCH 211/217] Update test artifacts --- tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json | 2 +- .../xdmod/user_admin/output/get_tabs-centerdirector.json | 2 +- .../artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json | 2 +- .../artifacts/xdmod/user_admin/output/get_tabs-normaluser.json | 2 +- tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json | 2 +- .../xdmod/user_admin/output/get_tabs-test.cd.one-center.json | 2 +- .../xdmod/user_admin/output/get_tabs-test.cs.one-center.json | 2 +- .../xdmod/user_admin/output/get_tabs-test.normal-user.json | 2 +- tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json | 2 +- .../xdmod/user_admin/output/get_tabs-test.usr_dev.json | 2 +- .../xdmod/user_admin/output/get_tabs-test.usr_mgr.json | 2 +- .../xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json index b807cb062b..b2a0f04158 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json @@ -40,7 +40,7 @@ "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", - "tooltip": "", + "tooltip": "Data Warehouse Batch Export", "userManualSectionName": "Data Export" }, { diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json index b807cb062b..b2a0f04158 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json @@ -40,7 +40,7 @@ "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", - "tooltip": "", + "tooltip": "Data Warehouse Batch Export", "userManualSectionName": "Data Export" }, { diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json index b807cb062b..b2a0f04158 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json @@ -40,7 +40,7 @@ "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", - "tooltip": "", + "tooltip": "Data Warehouse Batch Export", "userManualSectionName": "Data Export" }, { diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json index b807cb062b..b2a0f04158 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json @@ -40,7 +40,7 @@ "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", - "tooltip": "", + "tooltip": "Data Warehouse Batch Export", "userManualSectionName": "Data Export" }, { diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json index b807cb062b..b2a0f04158 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json @@ -40,7 +40,7 @@ "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", - "tooltip": "", + "tooltip": "Data Warehouse Batch Export", "userManualSectionName": "Data Export" }, { diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json index b807cb062b..b2a0f04158 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json @@ -40,7 +40,7 @@ "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", - "tooltip": "", + "tooltip": "Data Warehouse Batch Export", "userManualSectionName": "Data Export" }, { diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json index b807cb062b..b2a0f04158 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json @@ -40,7 +40,7 @@ "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", - "tooltip": "", + "tooltip": "Data Warehouse Batch Export", "userManualSectionName": "Data Export" }, { diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json index b807cb062b..b2a0f04158 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json @@ -40,7 +40,7 @@ "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", - "tooltip": "", + "tooltip": "Data Warehouse Batch Export", "userManualSectionName": "Data Export" }, { diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json index b807cb062b..b2a0f04158 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json @@ -40,7 +40,7 @@ "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", - "tooltip": "", + "tooltip": "Data Warehouse Batch Export", "userManualSectionName": "Data Export" }, { diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json index b807cb062b..b2a0f04158 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json @@ -40,7 +40,7 @@ "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", - "tooltip": "", + "tooltip": "Data Warehouse Batch Export", "userManualSectionName": "Data Export" }, { diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json index b807cb062b..b2a0f04158 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json @@ -40,7 +40,7 @@ "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", - "tooltip": "", + "tooltip": "Data Warehouse Batch Export", "userManualSectionName": "Data Export" }, { diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json index b807cb062b..b2a0f04158 100644 --- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json +++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json @@ -40,7 +40,7 @@ "permitted_modules": null, "javascriptClass": "XDMoD.Module.DataExport", "javascriptReference": "CCR.xdmod.ui.dataExport", - "tooltip": "", + "tooltip": "Data Warehouse Batch Export", "userManualSectionName": "Data Export" }, { From 6b888f688c2d48bcfe0172f74dd5e36085b750ef Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 22 Aug 2019 09:27:20 -0400 Subject: [PATCH 212/217] Remove incorrect namespace --- .../Migration/Version812To850/ConfigFilesMigration.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php index 76aa1dcf24..5abf557660 100644 --- a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php +++ b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php @@ -100,7 +100,7 @@ public function execute() EOT ); $console->displayBlankLine(); - $exportSetup = new OpenXdmod\Setup\WarehouseExportSetup($console); + $exportSetup = new WarehouseExportSetup($console); $exportSettings = $exportSetup->promptForSettings([ 'data_warehouse_export_export_directory' => xd_utilities\getConfiguration('data_warehouse_export', 'export_directory'), 'data_warehouse_export_retention_duration_days' => xd_utilities\getConfiguration('data_warehouse_export', 'retention_duration_days') From 0e3f61635af818d78fa00a2c50a65e1038e49c43 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 22 Aug 2019 09:32:53 -0400 Subject: [PATCH 213/217] Add default configuration values --- .../Migration/Version812To850/ConfigFilesMigration.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php index 5abf557660..39c1cd1fc7 100644 --- a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php +++ b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php @@ -10,7 +10,6 @@ use OpenXdmod\Migration\ConfigFilesMigration as AbstractConfigFilesMigration; use OpenXdmod\Setup\Console; use OpenXdmod\Setup\WarehouseExportSetup; -use xd_utilities; class ConfigFilesMigration extends AbstractConfigFilesMigration { @@ -102,8 +101,8 @@ public function execute() $console->displayBlankLine(); $exportSetup = new WarehouseExportSetup($console); $exportSettings = $exportSetup->promptForSettings([ - 'data_warehouse_export_export_directory' => xd_utilities\getConfiguration('data_warehouse_export', 'export_directory'), - 'data_warehouse_export_retention_duration_days' => xd_utilities\getConfiguration('data_warehouse_export', 'retention_duration_days') + 'data_warehouse_export_export_directory' => '/var/spool/xdmod/export', + 'data_warehouse_export_retention_duration_days' => 30 ]); $this->writePortalSettingsFile(array_merge( From 78802ecdedaf0f17e1795920f28cda9f471fdc1b Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 22 Aug 2019 09:41:46 -0400 Subject: [PATCH 214/217] Update setup and test scripts --- classes/OpenXdmod/Setup/WarehouseExportSetup.php | 2 +- tests/ci/scripts/xdmod-setup-finish.tcl | 4 ++-- tests/ci/scripts/xdmod-upgrade.tcl | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/classes/OpenXdmod/Setup/WarehouseExportSetup.php b/classes/OpenXdmod/Setup/WarehouseExportSetup.php index 61f8717c63..f301cf076a 100644 --- a/classes/OpenXdmod/Setup/WarehouseExportSetup.php +++ b/classes/OpenXdmod/Setup/WarehouseExportSetup.php @@ -78,7 +78,7 @@ private function promptForRetentionDuration($defaultDuration) while (!$haveResponse) { $retentionDuration = $this->console->prompt( - 'Export File Retention Duration (Days):', + 'Export File Retention Duration in Days:', $defaultDuration ); diff --git a/tests/ci/scripts/xdmod-setup-finish.tcl b/tests/ci/scripts/xdmod-setup-finish.tcl index aa75c43b9f..4f8b2733eb 100644 --- a/tests/ci/scripts/xdmod-setup-finish.tcl +++ b/tests/ci/scripts/xdmod-setup-finish.tcl @@ -33,8 +33,8 @@ confirmFileWrite yes enterToContinue selectMenuOption 7 -provideInput {Export Directory:} {} -provideInput {Export File Retention Duration (Days):} {31} +answerQuestion {Export Directory} {} +answerQuestion {Export File Retention Duration in Days} {31} confirmFileWrite yes enterToContinue diff --git a/tests/ci/scripts/xdmod-upgrade.tcl b/tests/ci/scripts/xdmod-upgrade.tcl index a9369e9429..8a339440d6 100644 --- a/tests/ci/scripts/xdmod-upgrade.tcl +++ b/tests/ci/scripts/xdmod-upgrade.tcl @@ -22,8 +22,8 @@ set timeout 180 spawn "xdmod-upgrade" confirmUpgrade provideInput {Enable Novice User Tab*} {off} -provideInput {Export Directory:} {} -provideInput {Export File Retention Duration (Days):} {31} +answerQuestion {Export Directory} {} +answerQuestion {Export File Retention Duration in Days} {31} expect { -re "\nDo you want to run aggregation now.*\\\]" { send no\n From f27288d6341d73680c12fb9e61d226a910a6f6c7 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 22 Aug 2019 10:20:04 -0400 Subject: [PATCH 215/217] Update test scripts --- tests/ci/scripts/xdmod-setup-finish.tcl | 4 ++-- tests/ci/scripts/xdmod-upgrade.tcl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/ci/scripts/xdmod-setup-finish.tcl b/tests/ci/scripts/xdmod-setup-finish.tcl index 4f8b2733eb..cada5040ab 100644 --- a/tests/ci/scripts/xdmod-setup-finish.tcl +++ b/tests/ci/scripts/xdmod-setup-finish.tcl @@ -33,8 +33,8 @@ confirmFileWrite yes enterToContinue selectMenuOption 7 -answerQuestion {Export Directory} {} -answerQuestion {Export File Retention Duration in Days} {31} +provideInput {Export Directory*} {} +provideInput {Export File Retention Duration*} 31 confirmFileWrite yes enterToContinue diff --git a/tests/ci/scripts/xdmod-upgrade.tcl b/tests/ci/scripts/xdmod-upgrade.tcl index 8a339440d6..7a1a4bbd3f 100644 --- a/tests/ci/scripts/xdmod-upgrade.tcl +++ b/tests/ci/scripts/xdmod-upgrade.tcl @@ -22,8 +22,8 @@ set timeout 180 spawn "xdmod-upgrade" confirmUpgrade provideInput {Enable Novice User Tab*} {off} -answerQuestion {Export Directory} {} -answerQuestion {Export File Retention Duration in Days} {31} +provideInput {Export Directory*} {} +provideInput {Export File Retention Duration*} 31 expect { -re "\nDo you want to run aggregation now.*\\\]" { send no\n From 547835203ccc1316f0cce2953bc31bb7b8277a82 Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 22 Aug 2019 11:27:23 -0400 Subject: [PATCH 216/217] Update test script --- tests/ci/scripts/xdmod-upgrade-jobs.tcl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/ci/scripts/xdmod-upgrade-jobs.tcl b/tests/ci/scripts/xdmod-upgrade-jobs.tcl index 1ccb40b84a..2550589d70 100644 --- a/tests/ci/scripts/xdmod-upgrade-jobs.tcl +++ b/tests/ci/scripts/xdmod-upgrade-jobs.tcl @@ -22,6 +22,8 @@ set timeout 180 spawn "xdmod-upgrade" confirmUpgrade provideInput {Enable Novice User Tab*} {off} +provideInput {Export Directory*} {} +provideInput {Export File Retention Duration*} 31 expect { -re "\nDo you want to run aggregation now.*\\\]" { send yes\n From 4324f92b74e3678c271ae728778085d23b5909ea Mon Sep 17 00:00:00 2001 From: "Jeffrey T. Palmer" Date: Thu, 22 Aug 2019 11:53:08 -0400 Subject: [PATCH 217/217] Update test artifacts --- .../xdmod/user_admin/output/jobs/get_tabs-admin.json | 11 +++++++++++ .../output/jobs/get_tabs-centerdirector.json | 11 +++++++++++ .../user_admin/output/jobs/get_tabs-centerstaff.json | 11 +++++++++++ .../user_admin/output/jobs/get_tabs-normaluser.json | 11 +++++++++++ .../user_admin/output/jobs/get_tabs-principal.json | 11 +++++++++++ .../output/jobs/get_tabs-test.cd.one-center.json | 11 +++++++++++ .../output/jobs/get_tabs-test.cs.one-center.json | 11 +++++++++++ .../output/jobs/get_tabs-test.normal-user.json | 11 +++++++++++ .../user_admin/output/jobs/get_tabs-test.pi.json | 11 +++++++++++ .../user_admin/output/jobs/get_tabs-test.usr_dev.json | 11 +++++++++++ .../user_admin/output/jobs/get_tabs-test.usr_mgr.json | 11 +++++++++++ .../output/jobs/get_tabs-test.usr_mgr_dev.json | 11 +++++++++++ 12 files changed, 132 insertions(+) diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-admin.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-admin.json index ddc6d36311..05734dcae1 100644 --- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-admin.json +++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-admin.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 500, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "Data Warehouse Batch Export", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerdirector.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerdirector.json index ddc6d36311..05734dcae1 100644 --- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerdirector.json +++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerdirector.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 500, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "Data Warehouse Batch Export", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerstaff.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerstaff.json index ddc6d36311..05734dcae1 100644 --- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerstaff.json +++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerstaff.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 500, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "Data Warehouse Batch Export", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-normaluser.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-normaluser.json index ddc6d36311..05734dcae1 100644 --- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-normaluser.json +++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-normaluser.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 500, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "Data Warehouse Batch Export", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-principal.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-principal.json index ddc6d36311..05734dcae1 100644 --- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-principal.json +++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-principal.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 500, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "Data Warehouse Batch Export", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cd.one-center.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cd.one-center.json index ddc6d36311..05734dcae1 100644 --- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cd.one-center.json +++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cd.one-center.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 500, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "Data Warehouse Batch Export", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cs.one-center.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cs.one-center.json index ddc6d36311..05734dcae1 100644 --- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cs.one-center.json +++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cs.one-center.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 500, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "Data Warehouse Batch Export", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.normal-user.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.normal-user.json index ddc6d36311..05734dcae1 100644 --- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.normal-user.json +++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.normal-user.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 500, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "Data Warehouse Batch Export", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.pi.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.pi.json index ddc6d36311..05734dcae1 100644 --- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.pi.json +++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.pi.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 500, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "Data Warehouse Batch Export", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_dev.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_dev.json index ddc6d36311..05734dcae1 100644 --- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_dev.json +++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_dev.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 500, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "Data Warehouse Batch Export", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr.json index ddc6d36311..05734dcae1 100644 --- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr.json +++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 500, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "Data Warehouse Batch Export", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false, diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr_dev.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr_dev.json index ddc6d36311..05734dcae1 100644 --- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr_dev.json +++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr_dev.json @@ -32,6 +32,17 @@ "tooltip": "", "userManualSectionName": "Metric Explorer" }, + { + "tab": "data_export", + "isDefault": false, + "title": "Data Export", + "pos": 500, + "permitted_modules": null, + "javascriptClass": "XDMoD.Module.DataExport", + "javascriptReference": "CCR.xdmod.ui.dataExport", + "tooltip": "Data Warehouse Batch Export", + "userManualSectionName": "Data Export" + }, { "tab": "report_generator", "isDefault": false,