diff --git a/background_scripts/batch_export_manager.php b/background_scripts/batch_export_manager.php
new file mode 100755
index 0000000000..fe5d2720a8
--- /dev/null
+++ b/background_scripts/batch_export_manager.php
@@ -0,0 +1,117 @@
+#!/usr/bin/env php
+ $value) {
+ if (is_array($value)) {
+ fwrite(STDERR, "Multiple values not allowed for '$key'\n");
+ exit(1);
+ }
+
+ switch ($key) {
+ case 'h':
+ case 'help':
+ $help = true;
+ break;
+ case 'dry-run':
+ $dryRun = true;
+ break;
+ case 'q':
+ case 'quiet':
+ $logLevel = max($logLevel, Log::WARNING);
+ break;
+ case 'v':
+ case 'verbose':
+ $logLevel = max($logLevel, Log::INFO);
+ break;
+ case 'd':
+ case 'debug':
+ $logLevel = max($logLevel, Log::DEBUG);
+ break;
+ default:
+ fwrite(STDERR, "Unexpected option '$key'\n");
+ exit(1);
+ break;
+ }
+ }
+
+ // Set default log level if none was specified.
+ if ($logLevel === -1) {
+ $logLevel = Log::NOTICE;
+ }
+
+ if ($help) {
+ displayHelpText();
+ exit;
+ }
+
+ $logConf = array(
+ 'file' => false,
+ 'mail' => false,
+ 'consoleLogLevel' => $logLevel
+ );
+ $logger = Log::factory('batch-export', $logConf);
+ $logger->info('Command: ' . implode(' ', array_map('escapeshellarg', $argv)));
+ // NOTE: "process_start_time" is needed for the log summary.
+ $logger->notice(['message' => 'batch_export_manager start', 'process_start_time' => date('Y-m-d H:i:s')]);
+ $batchProcessor = new BatchProcessor($logger);
+ $batchProcessor->setDryRun($dryRun);
+ $batchProcessor->processRequests();
+ // NOTE: "process_end_time" is needed for the log summary.
+ $logger->notice(['message' => 'batch_export_manager end', 'process_end_time' => date('Y-m-d H:i:s')]);
+ exit;
+} catch (Exception $e) {
+ // Write any unexpected exceptions directly to STDERR since they may not
+ // have been logged and it may not be able to create a log instance.
+ fwrite(STDERR, "Data warehouse batch export failed\n");
+ do {
+ fwrite(STDERR, $e->getMessage() . "\n" . $e->getTraceAsString() . "\n");
+ } while ($e = $e->getPrevious());
+ exit(1);
+}
+
+function displayHelpText()
+{
+ global $argv;
+
+ echo <<<"EOMSG"
+Usage: {$argv[0]}
+
+ -h, --help
+ Display this message and exit.
+
+ -v, --verbose
+ Output info level logging.
+
+ -d, --debug
+ Output debug level logging.
+
+ -q, --quiet
+ Output warning level logging.
+
+ --dry-run
+ Perform all the processing steps, but don't generate or remove any
+ files, send any emails, or change the status of any export requests.
+
+EOMSG;
+}
diff --git a/classes/DataWarehouse/Data/BatchDataset.php b/classes/DataWarehouse/Data/BatchDataset.php
new file mode 100644
index 0000000000..18860cd292
--- /dev/null
+++ b/classes/DataWarehouse/Data/BatchDataset.php
@@ -0,0 +1,249 @@
+query = $query;
+ $this->docs = $query->getColumnDocumentation();
+ $this->dbh = DB::factory($query->_db_profile);
+
+ try {
+ $this->hashSalt = xd_utilities\getConfiguration(
+ 'data_warehouse_export',
+ 'hash_salt'
+ );
+ } catch (Exception $e) {
+ $this->logger->warning('data_warehouse_export hash_salt is not set');
+ }
+
+ foreach ($this->docs as $key => $doc) {
+ $export = isset($doc['batchExport']) ? $doc['batchExport'] : false;
+ $name = $doc['name'];
+
+ if (isset($doc['units']) && $doc['units'] === 'ts') {
+ $name .= ' (Timestamp)';
+ }
+
+ if ($export === true) {
+ $this->header[$key] = $name;
+ } elseif ($export === 'anonymize') {
+ $this->header[$key] = $name . ' (Deidentified)';
+ $this->anonymousFields[$key] = true;
+ } elseif ($export === false) {
+ // Skip field.
+ } else {
+ throw new Exception(sprintf(
+ 'Unknown "batchExport" option %s',
+ var_export($export, true)
+ ));
+ }
+ }
+
+ $this->logger->debug(sprintf(
+ 'Header: %s',
+ json_encode(array_values($this->header))
+ ));
+ $this->logger->debug(sprintf(
+ 'Anonymous fields: %s',
+ json_encode(array_keys($this->anonymousFields))
+ ));
+ }
+
+ /**
+ * Get the header row.
+ *
+ * @return string[]
+ */
+ public function getHeader()
+ {
+ return array_values($this->header);
+ }
+
+ /**
+ * Get the current row from the data set.
+ *
+ * @return mixed[]
+ */
+ public function current()
+ {
+ return $this->currentRow;
+ }
+
+ /**
+ * Get the current row index.
+ *
+ * @return int
+ */
+ public function key()
+ {
+ return $this->currentRowIndex;
+ }
+
+ /**
+ * Advance iterator to the next row.
+ *
+ * Fetches the next row.
+ */
+ public function next()
+ {
+ $this->currentRowIndex++;
+ $this->currentRow = $this->getNextRow();
+ }
+
+ /**
+ * Rewind the iterator to the beginning.
+ *
+ * Executes the underlying raw query.
+ */
+ public function rewind()
+ {
+ $this->logger->debug('Executing query');
+ $this->sth = $this->query->getRawStatement();
+ $this->logger->debug(sprintf(
+ 'Raw query string: %s',
+ $this->sth->queryString
+ ));
+ $this->logger->debug(sprintf('Row count: %s', $this->sth->rowCount()));
+ $this->currentRowIndex = 1;
+ $this->currentRow = $this->getNextRow();
+ }
+
+ /**
+ * Is this iterator valid?
+ *
+ * @return bool
+ */
+ public function valid()
+ {
+ return $this->currentRow !== false;
+ }
+
+ /**
+ * Anonymize a field.
+ *
+ * @param string $field
+ * @return string
+ */
+ private function anonymizeField($field)
+ {
+ if (array_key_exists($field, $this->hashCache)) {
+ return $this->hashCache[$field];
+ }
+
+ $hash = sha1($field . $this->hashSalt);
+ $this->hashCache[$field] = $hash;
+
+ return $hash;
+ }
+
+ /**
+ * Get the next row of data.
+ *
+ * @return array
+ */
+ private function getNextRow()
+ {
+ $rawRow = $this->sth->fetch(PDO::FETCH_ASSOC);
+
+ if ($rawRow === false) {
+ return false;
+ }
+
+ $row = [];
+
+ foreach (array_keys($this->header) as $key) {
+ $row[] = isset($this->anonymousFields[$key])
+ ? $this->anonymizeField($rawRow[$key])
+ : $rawRow[$key];
+ }
+
+ return $row;
+ }
+}
diff --git a/classes/DataWarehouse/Export/BatchProcessor.php b/classes/DataWarehouse/Export/BatchProcessor.php
new file mode 100644
index 0000000000..ef3c2e3d23
--- /dev/null
+++ b/classes/DataWarehouse/Export/BatchProcessor.php
@@ -0,0 +1,350 @@
+fileManager = new FileManager($logger);
+ parent::__construct($logger);
+ $this->dbh = DB::factory('database');
+ $this->queryHandler = new QueryHandler();
+ $this->realmManager = new RealmManager();
+ }
+
+ /**
+ * Set the logger for this object.
+ *
+ * @see \CCR\Loggable::setLogger()
+ * @param \Log $logger A logger instance or null to use the null logger.
+ * @return self This object for method chaining.
+ */
+ public function setLogger(Log $logger = null)
+ {
+ parent::setLogger($logger);
+ $this->fileManager->setLogger($logger);
+ return $this;
+ }
+
+ /**
+ * Set whether or not processing the requests will be a dry run.
+ *
+ * If this is a dry run then no export files will be generated, no emails
+ * will be sent and no changes will be made to export requests in the
+ * database.
+ *
+ * @param boolean $dryRun
+ */
+ public function setDryRun($dryRun)
+ {
+ $this->dryRun = $dryRun;
+ }
+
+ /**
+ * Process all requests.
+ */
+ public function processRequests()
+ {
+ $this->processSubmittedRequests();
+ $this->processExpiringRequests();
+ }
+
+ /**
+ * Process requests in the "Submitted" state.
+ *
+ * Generate the data export and update the request.
+ */
+ private function processSubmittedRequests()
+ {
+ $this->logger->info('Processing submitted requests');
+ foreach ($this->queryHandler->listSubmittedRecords() as $request) {
+ $this->processSubmittedRequest($request);
+ }
+ }
+
+ /**
+ * Process a single export request.
+ *
+ * @param array $request The export request data.
+ */
+ private function processSubmittedRequest(array $request)
+ {
+ $this->logger->info([
+ 'message' => 'Processing request',
+ 'batch_export_request.id' => $request['id']
+ ]);
+
+ $user = XDUser::getUserByID($request['user_id']);
+
+ if ($user === null) {
+ $this->logger->err([
+ 'message' => 'User not found',
+ 'Users.id' => $request['user_id'],
+ 'batch_export_request.id' => $request['id']
+ ]);
+ return;
+ }
+
+ try {
+ $this->dbh->beginTransaction();
+ if (!$this->dryRun) {
+ $this->queryHandler->submittedToAvailable($request['id']);
+ }
+ $dataSet = $this->getDataSet($request, $user);
+ $format = $this->dryRun ? 'null' : $request['export_file_format'];
+ $dataFile = $this->fileManager->writeDataSetToFile($dataSet, $format);
+ $this->fileManager->createZipFile($dataFile, $request);
+
+ // Query for same record to get expiration date.
+ $request = $this->queryHandler->getRequestRecord($request['id']);
+ $this->sendExportSuccessEmail($user, $request);
+ $this->dbh->commit();
+ } catch (Exception $e) {
+ $this->dbh->rollback();
+ $this->logger->err([
+ 'message' => 'Failed to export data: ' . $e->getMessage(),
+ 'stacktrace' => $e->getTraceAsString()
+ ]);
+ if (!$this->dryRun) {
+ $this->queryHandler->submittedToFailed($request['id']);
+ }
+ $this->sendExportFailureEmail($user, $request, $e);
+ }
+ }
+
+ /**
+ * Process requests in the "Available" state that should be expired.
+ *
+ * Check if the request has expired and, if so, remove expired data and
+ * update the request.
+ */
+ private function processExpiringRequests()
+ {
+ $this->logger->info('Processing expiring requests');
+ foreach ($this->queryHandler->listExpiringRecords() as $request) {
+ $this->processExpiringRequest($request);
+ }
+ }
+
+ /**
+ * Process a single export request that is expiring.
+ *
+ * @param array $request The export request data.
+ */
+ private function processExpiringRequest(array $request)
+ {
+ $this->logger->info([
+ 'message' => 'Expiring request',
+ 'batch_export_request.id' => $request['id']
+ ]);
+
+ if ($this->dryRun) {
+ $this->logger->notice('dry run: Not expiring export file');
+ return;
+ }
+
+ try {
+ $this->dbh->beginTransaction();
+ $this->queryHandler->availableToExpired($request['id']);
+ $this->fileManager->removeExportFile($request['id']);
+ $this->dbh->commit();
+ } catch (Exception $e) {
+ $this->dbh->rollback();
+ $this->logger->err([
+ 'message' => 'Failed to expire record: ' . $e->getMessage(),
+ 'stacktrace' => $e->getTraceAsString()
+ ]);
+ }
+ }
+
+ /**
+ * Get the data set for the given request.
+ *
+ * @param array $request
+ * @param \XDUser $user
+ * @return \DataWarehouse\Data\BatchDataset;
+ * @throws \Exception
+ */
+ private function getDataSet(array $request, XDUser $user)
+ {
+ $this->logger->info([
+ 'message' => 'Querying data',
+ 'Users.id' => $user->getUserID(),
+ 'user_email' => $user->getEmailAddress(),
+ 'user_first_name' => $user->getFirstName(),
+ 'user_last_name' => $user->getLastName(),
+ 'batch_export_request.id' => $request['id'],
+ 'realm' => $request['realm'],
+ 'start_date' => $request['start_date'],
+ 'end_date' => $request['end_date']
+ ]);
+
+ try {
+ $className = $this->realmManager->getRawDataQueryClass($request['realm']);
+ $this->logger->debug(sprintf('Instantiating query class "%s"', $className));
+ $query = new $className(
+ [
+ 'start_date' => $request['start_date'],
+ 'end_date' => $request['end_date']
+ ],
+ 'batch'
+ );
+ $dataSet = new BatchDataset($query, $user, $this->logger);
+ return $dataSet;
+ } catch (Exception $e) {
+ $this->logger->err([
+ 'message' => $e->getMessage(),
+ 'stacktrace' => $e->getTraceAsString()
+ ]);
+ throw new Exception('Failed to create batch export query', 0, $e);
+ }
+ }
+
+ /**
+ * Send email indicating a successful export.
+ *
+ * @param \XDUser $user The user that requested the export.
+ * @param array $request The batch request data.
+ */
+ private function sendExportSuccessEmail(XDUser $user, array $request)
+ {
+ if ($this->dryRun) {
+ $this->logger->notice('dry run: Not sending success email');
+ return;
+ }
+
+ $this->logger->info('Sending success email');
+
+ // Remove time from expiration date time.
+ list($expirationDate) = explode(' ', $request['export_expires_datetime']);
+
+ MailWrapper::sendTemplate(
+ 'batch_export_success',
+ [
+ 'subject' => 'Batch export ready for download',
+ 'toAddress' => $user->getEmailAddress(),
+ 'first_name' => $user->getFirstName(),
+ 'last_name' => $user->getLastName(),
+ 'current_date' => date('Y-m-d'),
+ 'expiration_date' => $expirationDate,
+ 'download_url' => sprintf(
+ '%s#data_export?action=download&id=%d',
+ xd_utilities\getConfigurationUrlBase('general', 'site_address'),
+ $request['id']
+ ),
+ 'maintainer_signature' => MailWrapper::getMaintainerSignature()
+ ]
+ );
+ }
+
+ /**
+ * Send email indicating a failed export.
+ *
+ * Sends one email to the user that created the request and another email
+ * to the tech support recipient.
+ *
+ * @param \XDUser $user The user that created the request.
+ * @param array $request Export request data.
+ * @param \Exception $e The exception that caused the failure.
+ */
+ private function sendExportFailureEmail(
+ XDUser $user,
+ array $request,
+ Exception $e
+ ) {
+ if ($this->dryRun) {
+ $this->logger->notice('dry run: Not sending failure email');
+ return;
+ }
+
+ $this->logger->info([
+ 'message' => 'Sending failure email',
+ 'batch_export_request.id' => $request['id']
+ ]);
+
+ $message = $e->getMessage();
+ $stackTrace = $e->getTraceAsString();
+ while ($e = $e->getPrevious()) {
+ $stackTrace .= sprintf(
+ "\n\n%s\n%s",
+ $e->getMessage(),
+ $e->getTraceAsString()
+ );
+ }
+
+ MailWrapper::sendTemplate(
+ 'batch_export_failure_admin_notice',
+ [
+ 'subject' => 'Batch export failed',
+ 'toAddress' => xd_utilities\getConfiguration('general', 'tech_support_recipient'),
+ 'user_email' => $user->getEmailAddress(),
+ 'user_username' => $user->getUsername(),
+ 'user_formal_name' => $user->getFormalName(true),
+ 'current_date' => date('Y-m-d'),
+ 'exception_message' => $message,
+ 'exception_stack_trace' => $stackTrace
+ ]
+ );
+
+ MailWrapper::sendTemplate(
+ 'batch_export_failure',
+ [
+ 'subject' => 'Batch export failed',
+ 'toAddress' => $user->getEmailAddress(),
+ 'first_name' => $user->getFirstName(),
+ 'last_name' => $user->getLastName(),
+ 'current_date' => date('Y-m-d'),
+ 'failure_reason' => $message,
+ 'maintainer_signature' => MailWrapper::getMaintainerSignature()
+ ]
+ );
+ }
+}
diff --git a/classes/DataWarehouse/Export/FileManager.php b/classes/DataWarehouse/Export/FileManager.php
new file mode 100644
index 0000000000..cae7f8c15e
--- /dev/null
+++ b/classes/DataWarehouse/Export/FileManager.php
@@ -0,0 +1,265 @@
+fileWriterFactory = new FileWriterFactory($logger);
+ parent::__construct($logger);
+
+ try {
+ $this->exportDir = xd_utilities\getConfiguration(
+ 'data_warehouse_export',
+ 'export_directory'
+ );
+ } catch (Exception $e) {
+ $this->logger->err([
+ 'message' => $e->getMessage(),
+ 'stacktrace' => $e->getTraceAsString()
+ ]);
+ throw new Exception('Export directory is not configured', 0, $e);
+ }
+
+ if (!is_dir($this->exportDir)) {
+ throw new Exception(sprintf(
+ 'Export directory "%s" does not exist',
+ $this->exportDir
+ ));
+ }
+
+ if (!is_readable($this->exportDir)) {
+ throw new Exception(sprintf(
+ 'Export directory "%s" is not readable',
+ $this->exportDir
+ ));
+ }
+ }
+
+ /**
+ * Set the logger for this object.
+ *
+ * @see \CCR\Loggable::setLogger()
+ * @param \Log $logger A logger instance or null to use the null logger.
+ * @return self This object for method chaining.
+ */
+ public function setLogger(Log $logger = null)
+ {
+ parent::setLogger($logger);
+ $this->fileWriterFactory->setLogger($logger);
+ return $this;
+ }
+
+ /**
+ * Get the batch export data file path.
+ *
+ * This is the path of the file that will be used to store the export data on
+ * the Open XDMoD server after it has been generated.
+ *
+ * @param intger $id Batch export request primary key.
+ * @return string
+ */
+ public function getExportDataFilePath($id)
+ {
+ return sprintf('%s/%s.zip', $this->exportDir, $id);
+ }
+
+ /**
+ * Get the batch export data file name.
+ *
+ * This is the name that will be used by the file that contains the
+ * exported data.
+ *
+ * @param array $request Batch export request data.
+ * @return string
+ */
+ public function getDataFileName(array $request)
+ {
+ return sprintf(
+ '%s--%s-%s.%s',
+ $request['realm'],
+ $request['start_date'],
+ $request['end_date'],
+ strtolower($request['export_file_format'])
+ );
+ }
+
+ /**
+ * Get the batch export zip file name.
+ *
+ * This is the name that will be used when the file is downloaded by a
+ * user.
+ *
+ * @param array $request Batch export request data.
+ * @return string
+ */
+ public function getZipFileName(array $request)
+ {
+ return sprintf(
+ '%s--%s-%s.zip',
+ $request['realm'],
+ $request['start_date'],
+ $request['end_date']
+ );
+ }
+
+ /**
+ * Write a data set to a temporary file.
+ *
+ * @param \DataWarehouse\Data\BatchDataset $dataSet
+ * @param string $format
+ * @return string Path to file that was written to.
+ * @throws \Exception If writing the data fails.
+ */
+ public function writeDataSetToFile(BatchDataset $dataSet, $format)
+ {
+ $this->logger->info([
+ 'message' => 'Writing data to file',
+ 'format' => $format
+ ]);
+
+ try {
+ $dataFile = tempnam(sys_get_temp_dir(), 'batch-export-');
+ $fileWriter = $this->fileWriterFactory->createFileWriter(
+ $format,
+ $dataFile
+ );
+
+ $this->logger->debug([
+ 'message' => 'Created file writer',
+ 'file_writer' => $fileWriter
+ ]);
+
+ $fileWriter->writeRecord($dataSet->getHeader());
+
+ foreach ($dataSet as $record) {
+ $fileWriter->writeRecord($record);
+ }
+
+ $fileWriter->close();
+
+ return $dataFile;
+ } catch (Exception $e) {
+ $this->logger->err([
+ 'message' => $e->getMessage(),
+ 'stacktrace' => $e->getTraceAsString()
+ ]);
+ throw new Exception('Failed to write data set to file', 0, $e);
+ }
+ }
+
+ /**
+ * Create a zip file containing a single file.
+ *
+ * @param string $dataFile Path to file that will be put in the zip file.
+ * @param array $request Batch export request data.
+ * @throws \Exception If creating the zip file fails.
+ */
+ public function createZipFile($dataFile, array $request)
+ {
+ $zipFile = $this->getExportDataFilePath($request['id']);
+
+ $this->logger->info([
+ 'message' => 'Creating zip file',
+ 'batch_export_request.id' => $request['id'],
+ 'data_file' => $dataFile,
+ 'zip_file' => $zipFile
+ ]);
+
+ try {
+ $zip = new ZipArchive();
+ $zipOpenCode = $zip->open($zipFile, ZipArchive::CREATE);
+
+ if ($zipOpenCode !== true) {
+ throw new Exception(sprintf(
+ 'Failed to open zip file "%s", error code "%s"',
+ $zipFile,
+ $zipOpenCode
+ ));
+ }
+
+ // Override the name of the temporary data file with the proper
+ // name that will be used in the archive file.
+ $localName = $this->getDataFileName($request);
+
+ if (!$zip->addFile($dataFile, $localName)) {
+ throw new Exception(sprintf(
+ 'Failed to add file "%s" to zip file "%s"',
+ $dataFile,
+ $zipFile
+ ));
+ }
+
+ if (!$zip->close()) {
+ throw new Exception(sprintf(
+ 'Failed to close zip file "%s"',
+ $zipFile
+ ));
+ }
+
+ return $zipFile;
+ } catch (Exception $e) {
+ $this->logger->err([
+ 'message' => $e->getMessage(),
+ 'stacktrace' => $e->getTraceAsString()
+ ]);
+ throw new Exception('Failed to create zip file', 0, $e);
+ }
+ }
+
+ /**
+ * Remove a batch export data file.
+ *
+ * @param int $id Export request primary key.
+ * @throws \Exception If removing the file fails.
+ */
+ public function removeExportFile($id)
+ {
+ $zipFile = $this->getExportDataFilePath($id);
+
+ $this->logger->info([
+ 'message' => 'Removing export file',
+ 'batch_export_request.id' => $id,
+ 'zip_file' => $zipFile
+ ]);
+
+ if (!unlink($zipFile)) {
+ throw new Exception(sprintf('Failed to delete "%s"', $zipFile));
+ }
+ }
+}
diff --git a/classes/DataWarehouse/Export/FileWriter/CsvFileWriter.php b/classes/DataWarehouse/Export/FileWriter/CsvFileWriter.php
new file mode 100644
index 0000000000..ef9a1a5088
--- /dev/null
+++ b/classes/DataWarehouse/Export/FileWriter/CsvFileWriter.php
@@ -0,0 +1,23 @@
+fh, $record) === false) {
+ $this->logAndThrowException(
+ sprintf('Failed to write to file "%s"', $this->file)
+ );
+ }
+ }
+}
diff --git a/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php b/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php
new file mode 100644
index 0000000000..ed54fd1392
--- /dev/null
+++ b/classes/DataWarehouse/Export/FileWriter/FileWriterFactory.php
@@ -0,0 +1,44 @@
+logger->debug([
+ 'message' => 'Creating new file writer',
+ 'format' => $format,
+ 'file' => $file
+ ]);
+
+ switch (strtolower($format)) {
+ case 'csv':
+ return new CsvFileWriter($file, $this->logger);
+ break;
+ case 'json':
+ return new JsonFileWriter($file, $this->logger);
+ break;
+ case 'null':
+ return new NullFileWriter($file, $this->logger);
+ break;
+ default:
+ $this->logAndThrowException(
+ sprintf('Unsupported format "%s"', $format)
+ );
+ break;
+ }
+ }
+}
diff --git a/classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php b/classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php
new file mode 100644
index 0000000000..60cbaa09eb
--- /dev/null
+++ b/classes/DataWarehouse/Export/FileWriter/JsonFileWriter.php
@@ -0,0 +1,84 @@
+fh, '[') === false) {
+ $this->logAndThrowException(
+ sprintf('Failed to write to file "%s"', $this->file)
+ );
+ }
+ }
+
+ /**
+ * Write the closing bracket and close the file.
+ */
+ public function close()
+ {
+ if (fwrite($this->fh, "\n]") === false) {
+ $this->logAndThrowException(
+ sprintf('Failed to write to file "%s"', $this->file)
+ );
+ }
+
+ parent::close();
+ }
+
+ /**
+ * Write a record to file formatted as JSON.
+ *
+ * @param array $record
+ */
+ public function writeRecord(array $record)
+ {
+ $json = json_encode($record, JSON_UNESCAPED_SLASHES);
+
+ if ($json === false) {
+ $this->logAndThrowException(sprintf(
+ 'Failed to encode data as JSON: %s',
+ Json::getLastErrorMessage()
+ ));
+ }
+
+ // If any records have already been written then a comma is needed
+ // before the next record.
+ $separator = ($this->recordWritten ? ',' : '') . "\n ";
+
+ if (fwrite($this->fh, $separator . $json) === false) {
+ $this->logAndThrowException(
+ sprintf('Failed to write to file "%s"', $this->file)
+ );
+ }
+
+ $this->recordWritten = true;
+ }
+}
diff --git a/classes/DataWarehouse/Export/FileWriter/NullFileWriter.php b/classes/DataWarehouse/Export/FileWriter/NullFileWriter.php
new file mode 100644
index 0000000000..6326e64170
--- /dev/null
+++ b/classes/DataWarehouse/Export/FileWriter/NullFileWriter.php
@@ -0,0 +1,40 @@
+setLogger($logger);
+ }
+
+ /**
+ * Don't close anything.
+ */
+ public function close()
+ {
+ }
+
+ /**
+ * Don't write anything.
+ *
+ * @param array $record
+ */
+ public function writeRecord(array $record)
+ {
+ }
+}
diff --git a/classes/DataWarehouse/Export/FileWriter/aFileWriter.php b/classes/DataWarehouse/Export/FileWriter/aFileWriter.php
new file mode 100644
index 0000000000..cce8e52021
--- /dev/null
+++ b/classes/DataWarehouse/Export/FileWriter/aFileWriter.php
@@ -0,0 +1,64 @@
+file = $file;
+ $this->fh = fopen($this->file, 'w');
+
+ if ($this->fh === false) {
+ $this->logAndThrowException(
+ sprintf('Failed to open file "%s"', $this->file)
+ );
+ }
+ }
+
+ /**
+ * Close file.
+ */
+ public function close()
+ {
+ if (fclose($this->fh) === false) {
+ $this->logAndThrowException(
+ sprintf('Failed to close file "%s"', $this->file)
+ );
+ }
+ }
+
+ /**
+ * Return string representation of class instance.
+ *
+ * @returns string
+ */
+ public function __toString()
+ {
+ return sprintf('%s(file: "%s")', get_class($this), $this->file);
+ }
+}
diff --git a/classes/DataWarehouse/Export/FileWriter/iFileWriter.php b/classes/DataWarehouse/Export/FileWriter/iFileWriter.php
new file mode 100644
index 0000000000..1e385c9176
--- /dev/null
+++ b/classes/DataWarehouse/Export/FileWriter/iFileWriter.php
@@ -0,0 +1,33 @@
+ Available -> Expired
+ * |
+ * v
+ * Failed
+ *
+ * ...any state can transition to Deleted.
+ */
+
+namespace DataWarehouse\Export;
+
+use Exception;
+use CCR\DB;
+
+class QueryHandler
+{
+ /**
+ * Database handle.
+ * @var \CCR\DB\iDatabase
+ */
+ private $dbh;
+
+ /**
+ * Definition of Submitted state.
+ * @var string
+ */
+ private $whereSubmitted = "WHERE export_succeeded IS NULL AND export_created_datetime IS NULL AND export_expired = 0 ";
+
+ /**
+ * Definition of Available state.
+ * @var string
+ */
+ private $whereAvailable = "WHERE export_succeeded = 1 AND export_created_datetime IS NOT NULL AND export_expired = 0 ";
+
+ /**
+ * Definition of Expired state.
+ * @var string
+ */
+ private $whereExpired = "WHERE export_succeeded = 1 AND export_created_datetime IS NOT NULL AND export_expired = 1 ";
+
+ /**
+ * Definition of Failed state.
+ * @var string
+ */
+ private $whereFailed = "WHERE export_succeeded = 0 AND export_created_datetime IS NULL AND export_expired = 0 ";
+
+ public function __construct()
+ {
+ $this->dbh = DB::factory('database');
+ }
+
+ /**
+ * Create request record for specified export request.
+ *
+ * @param integer $userId
+ * @param string $realm Realm unique identifier.
+ * @param string $startDate Start date formatted as YYYY-MM-DD.
+ * @param string $endDate End date formatted as YYYY-MM-DD.
+ * @param string $format Export format (CSV or JSON).
+ * @return integer The id for the inserted record.
+ */
+ public function createRequestRecord(
+ $userId,
+ $realm,
+ $startDate,
+ $endDate,
+ $format
+ ) {
+ $sql = "INSERT INTO batch_export_requests
+ (requested_datetime, user_id, realm, start_date, end_date, export_file_format)
+ VALUES
+ (NOW(), :user_id, :realm, :start_date, :end_date, :export_file_format)";
+
+ $params = array(
+ 'user_id' => $userId,
+ 'realm' => $realm,
+ 'start_date' => $startDate,
+ 'end_date' => $endDate,
+ 'export_file_format' => $format
+ );
+
+ return $this->dbh->insert($sql, $params);
+ }
+
+ /**
+ * Get a single export request record.
+ *
+ * @param integer $id Export request primary key.
+ * @return array
+ */
+ public function getRequestRecord($id)
+ {
+ $sql = 'SELECT id,
+ user_id
+ realm,
+ start_date,
+ end_date,
+ export_file_format,
+ requested_datetime
+ export_succeeded,
+ export_created_datetime,
+ downloaded_datetime,
+ export_expires_datetime,
+ export_expired,
+ last_modified
+ FROM batch_export_requests
+ WHERE id = :id';
+ list($record) = $this->dbh->query($sql, ['id' => $id]);
+ return $record;
+ }
+
+ /**
+ * Transition specified export request from Submitted state to Failed state.
+ *
+ * @param integer $id Export request primary key.
+ * @return integer Count of affected rows--should be 1 if successful.
+ */
+ public function submittedToFailed($id)
+ {
+ $sql = "UPDATE batch_export_requests
+ SET export_succeeded = 0 " .
+ $this->whereSubmitted .
+ "AND id = :id";
+ return $this->dbh->execute($sql, array('id' => $id));
+ }
+
+ /**
+ * Transition specified export request from Submitted state to Available state.
+ *
+ * @param integer $id Export request primary key.
+ * @return integer Count of affected rows--should be 1 if successful.
+ */
+ public function submittedToAvailable($id)
+ {
+ // read export retention duration from config file. Value is stored in days.
+ $expires_in_days = \xd_utilities\getConfiguration('data_warehouse_export', 'retention_duration_days');
+
+ $sql = "UPDATE batch_export_requests
+ SET export_created_datetime = NOW(),
+ export_expires_datetime = DATE_ADD(NOW(), INTERVAL :expires_in_days DAY),
+ export_succeeded = 1 " .
+ $this->whereSubmitted . "AND id = :id";
+
+ $params = array(
+ 'expires_in_days' => $expires_in_days,
+ 'id' => $id
+ );
+
+ return $this->dbh->execute($sql, $params);
+ }
+
+ /**
+ * Transition specified export request from Available state to Expired state.
+ *
+ * @param integer $userId
+ * @return integer Count of affected rows--should be 1 if successful.
+ */
+ public function availableToExpired($id)
+ {
+ $sql = "UPDATE batch_export_requests SET export_expired = 1 " .
+ $this->whereAvailable . 'AND id = :id';
+ return $this->dbh->execute($sql, array('id' => $id));
+ }
+
+ /**
+ * Return count of all export requests presently in Submitted state.
+ *
+ * @return integer Count of rows.
+ */
+ public function countSubmittedRecords()
+ {
+ $sql = "SELECT COUNT(id) AS row_count FROM batch_export_requests " . $this->whereSubmitted;
+ $result = $this->dbh->query($sql);
+ return $result[0]['row_count'];
+ }
+
+ /**
+ * Return details of all export requests presently in Submitted state.
+ *
+ * @return array
+ */
+ public function listSubmittedRecords()
+ {
+ $sql = "SELECT id, user_id, realm, start_date, end_date, export_file_format, requested_datetime
+ FROM batch_export_requests " . $this->whereSubmitted . ' ORDER BY requested_datetime, id';
+ return $this->dbh->query($sql);
+ }
+
+ /**
+ * Return export requests in Available state that should expire.
+ *
+ * @return array
+ */
+ public function listExpiringRecords()
+ {
+ $sql = 'SELECT id,
+ user_id,
+ realm,
+ start_date,
+ end_date,
+ export_file_format,
+ requested_datetime,
+ export_succeeded,
+ export_created_datetime,
+ export_expires_datetime
+ FROM batch_export_requests
+ ' . $this->whereAvailable . '
+ AND export_expires_datetime IS NOT NULL
+ AND export_expires_datetime < NOW()
+ ORDER BY requested_datetime, id';
+ return $this->dbh->query($sql);
+ }
+
+ /**
+ * Return details of export requests made by specified user.
+ *
+ * @param integer $userId
+ * @return array All of the user's export requests.
+ */
+ public function listRequestsForUser($userId)
+ {
+ $sql = "SELECT id,
+ realm,
+ start_date,
+ end_date,
+ export_succeeded,
+ export_expired,
+ export_expires_datetime,
+ export_created_datetime,
+ export_file_format,
+ requested_datetime
+ FROM batch_export_requests
+ WHERE user_id = :user_id
+ ORDER BY requested_datetime, id";
+ return $this->dbh->query($sql, array('user_id' => $userId));
+ }
+
+ /**
+ * Return details (including state) of export requests made by specified user.
+ *
+ * @param integer $userId
+ * @return array All of the user's export requests (including state field).
+ */
+ public function listUserRequestsByState($userId)
+ {
+ $attributes = "SELECT id,
+ realm,
+ start_date,
+ end_date,
+ export_succeeded,
+ export_expired,
+ export_expires_datetime,
+ export_created_datetime,
+ export_file_format,
+ requested_datetime,
+ ";
+ $fromTable = "FROM batch_export_requests ";
+ $userClause = "AND user_id = :user_id ";
+
+ $sql = $attributes . "'Submitted' AS state " . $fromTable . $this->whereSubmitted . $userClause . "UNION " .
+ $attributes . "'Available' AS state " . $fromTable . $this->whereAvailable . $userClause . "UNION " .
+ $attributes . "'Expired' AS state " . $fromTable . $this->whereExpired . $userClause . "UNION " .
+ $attributes . "'Failed' AS state " . $fromTable . $this->whereFailed . $userClause . "ORDER BY requested_datetime, id";
+
+ return $this->dbh->query($sql, array('user_id' => $userId));
+ }
+
+ /**
+ * Delete specified record from the database, regardless of its state.
+ *
+ * Only the user who submitted the request may delete it.
+ *
+ * @param integer $id Export request primary key.
+ * @param integer $userId
+ * @return integer Count of deleted rows--should be 1 if successful.
+ */
+ public function deleteRequest($id, $userId)
+ {
+ $sql = "DELETE FROM batch_export_requests WHERE id = :request_id AND user_id = :user_id";
+ return $this->dbh->execute($sql, array('request_id' => $id, 'user_id' => $userId));
+ }
+}
diff --git a/classes/DataWarehouse/Export/RealmManager.php b/classes/DataWarehouse/Export/RealmManager.php
new file mode 100644
index 0000000000..f72889061c
--- /dev/null
+++ b/classes/DataWarehouse/Export/RealmManager.php
@@ -0,0 +1,133 @@
+dbh = DB::factory('database');
+ $this->config = XdmodConfiguration::assocArrayFactory(
+ 'rawstatistics.json',
+ CONFIG_DIR
+ );
+ }
+
+ /**
+ * Get an array of all the batch exportable realms.
+ *
+ * @return \Models\Realm[]
+ */
+ public function getRealms()
+ {
+ // The "name" values from rawstatistics match those in
+ // the moddb.realms.display column.
+ $exportable = array_map(
+ function ($realm) {
+ return $realm['name'];
+ },
+ $this->config['realms']
+ );
+
+ return array_filter(
+ Realms::getRealms(),
+ function ($realm) use ($exportable) {
+ return in_array($realm->getDisplay(), $exportable);
+ }
+ );
+ }
+
+ /**
+ * Get an array of all the batch exportable realms for a user.
+ *
+ * @param \XDUser $user
+ * @return \Models\Realm[]
+ */
+ public function getRealmsForUser(XDUser $user)
+ {
+ // Returns data from moddb.realms.display column.
+ $userRealms = Realms::getRealmsForUser($user);
+
+ return array_filter(
+ $this->getRealms(),
+ function ($realm) use ($userRealms) {
+ return in_array($realm->getDisplay(), $userRealms);
+ }
+ );
+ }
+
+ /**
+ * Get the raw data query class for the given realm.
+ *
+ * @param string $realmName The realm name used in moddb.realms.name.
+ * @return string The fully qualified name of the query class.
+ */
+ public function getRawDataQueryClass($realmName)
+ {
+ // The query classes use the "name" from the rawstatistics
+ // configuration, but the realm name is taken from moddb.realms.name.
+ // These use the same "display" name so that is used to find the
+ // correct class name.
+
+ // Realm model.
+ $realmObj = null;
+
+ foreach ($this->getRealms() as $realm) {
+ if ($realm->getName() == $realmName) {
+ $realmObj = $realm;
+ break;
+ }
+ }
+
+ if ($realmObj === null) {
+ throw new Exception(
+ sprintf('Failed to find model for realm "%s"', $realmName)
+ );
+ }
+
+ // Realm rawstatistics configuration.
+ $realmConfig = null;
+
+ foreach ($this->config['realms'] as $realm) {
+ if ($realm['display'] == $realmObj->getDisplay()) {
+ $realmConfig = $realm;
+ break;
+ }
+ }
+
+ if ($realmConfig === null) {
+ throw new Exception(
+ sprintf(
+ 'Failed to find rawstatistics configuration for realm "%s"',
+ $realmName
+ )
+ );
+ }
+
+ return sprintf('\DataWarehouse\Query\%s\JobDataset', $realmConfig['name']);
+ }
+}
diff --git a/classes/DataWarehouse/Query/Jobs/JobDataset.php b/classes/DataWarehouse/Query/Jobs/JobDataset.php
index be3c708b18..2212fdf08d 100644
--- a/classes/DataWarehouse/Query/Jobs/JobDataset.php
+++ b/classes/DataWarehouse/Query/Jobs/JobDataset.php
@@ -1,11 +1,13 @@
getDataTable();
- $joblistTable = new Table($dataTable->getSchema(), $dataTable->getName() . "_joblist", "jl");
- $factTable = new Table(new Schema('modw'), 'job_tasks', 'jt');
+ // The data table is always aliased to "jf".
+ $tables = ['jf' => $this->getDataTable()];
- $this->addTable($joblistTable);
- $this->addTable($factTable);
+ foreach ($config['tables'] as $tableDef) {
+ $alias = $tableDef['alias'];
+ $table = new Table(
+ new Schema($tableDef['schema']),
+ $tableDef['name'],
+ $alias
+ );
+ $tables[$alias] = $table;
+ $this->addTable($table);
- $this->addWhereCondition(new WhereCondition(
- new TableField($joblistTable, "agg_id"),
- "=",
- new TableField($dataTable, "id")
- ));
- $this->addWhereCondition(new WhereCondition(
- new TableField($joblistTable, "jobid"),
- "=",
- new TableField($factTable, "job_id")
- ));
+ $join = $tableDef['join'];
+ $this->addWhereCondition(new WhereCondition(
+ new TableField($table, $join['primaryKey']),
+ '=',
+ new TableField($tables[$join['foreignTableAlias']], $join['foreignKey'])
+ ));
+ }
+
+ // This table is defined in the configuration file, but used in the section below.
+ $factTable = $tables['jt'];
if (isset($parameters['primary_key'])) {
$this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'job_id'), "=", $parameters['primary_key']));
- } else {
+ } elseif (isset($parameters['job_identifier'])) {
$matches = array();
if (preg_match('/^(\d+)(?:[\[_](\d+)\]?)?$/', $parameters['job_identifier'], $matches)) {
$this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'resource_id'), '=', $parameters['resource_id']));
@@ -53,41 +61,70 @@ public function __construct(
$this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'local_job_id_raw'), '=', $matches[1]));
}
} else {
- throw new \Exception('invalid query parameters');
+ throw new Exception('invalid "job_identifier" query parameter');
+ }
+ } elseif (isset($parameters['start_date']) && isset($parameters['end_date'])) {
+ date_default_timezone_set('UTC');
+ $startDate = date_parse_from_format('Y-m-d', $parameters['start_date']);
+ $startDateTs = mktime(
+ 0,
+ 0,
+ 0,
+ $startDate['month'],
+ $startDate['day'],
+ $startDate['year']
+ );
+ if ($startDateTs === false) {
+ throw new Exception('invalid "start_date" query parameter');
+ }
+
+ $endDate = date_parse_from_format('Y-m-d', $parameters['end_date']);
+ $endDateTs = mktime(
+ 23,
+ 59,
+ 59,
+ $endDate['month'],
+ $endDate['day'],
+ $endDate['year']
+ );
+ if ($startDateTs === false) {
+ throw new Exception('invalid "end_date" query parameter');
}
+
+ $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_time_ts'), ">=", $startDateTs));
+ $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'end_time_ts'), "<=", $endDateTs));
+ } else {
+ throw new Exception('invalid query parameters');
}
- if ($stat == "accounting") {
- $i = 0;
- foreach ($config['modw.job_tasks'] as $sdata) {
- $sfield = $sdata['key'];
- if ($sdata['dtype'] == 'accounting') {
- $this->addField(new TableField($factTable, $sfield));
- $this->documentation[$sfield] = $sdata;
- } elseif ($sdata['dtype'] == 'foreignkey') {
- if (isset($sdata['join'])) {
- $info = $sdata['join'];
- $i += 1;
- $tmptable = new Table(new Schema($info['schema']), $info['table'], "ft$i");
- $this->addTable($tmptable);
- $this->addWhereCondition(new WhereCondition(new TableField($factTable, $sfield), '=', new TableField($tmptable, "id")));
- $fcol = isset($info['column']) ? $info['column'] : 'name';
- $this->addField(new TableField($tmptable, $fcol, $sdata['name']));
-
- $this->documentation[ $sdata['name'] ] = $sdata;
+ if ($stat == "accounting" || $stat == 'batch') {
+ foreach ($config['fields'] as $field) {
+ // Replace hierarchy constants.
+ foreach (['name', 'documentation'] as $key) {
+ $value = $field[$key];
+ if (strpos($value, 'HIERARCHY_') === 0 && defined($value)) {
+ $field[$key] = constant($value);
}
}
+
+ $alias = $field['name'];
+ if (isset($field['tableAlias']) && isset($field['column'])) {
+ $this->addField(new TableField(
+ $tables[$field['tableAlias']],
+ $field['column'],
+ $alias
+ ));
+ } elseif (isset($field['formula'])) {
+ $this->addField(new FormulaField($field['formula'], $alias));
+ } else {
+ throw new Exception(sprintf(
+ 'Missing tableAlias and column or formula for "%s", definition: %s',
+ $alias,
+ json_encode($field)
+ ));
+ }
+ $this->documentation[$alias] = $field;
}
- $rf = new Table(new Schema('modw'), 'resourcefact', 'rf');
- $this->addTable($rf);
- $this->addWhereCondition(new WhereCondition(new TableField($factTable, 'resource_id'), '=', new TableField($rf, 'id')));
- $this->addField(new TableField($rf, 'timezone'));
- $this->documentation['timezone'] = array(
- "name" => "Timezone",
- "documentation" => "The timezone of the resource.",
- "group" => "Administration",
- 'visibility' => 'public',
- "per" => "resource");
} else {
$this->addField(new TableField($factTable, "job_id", "jobid"));
$this->addField(new TableField($factTable, "local_jobid", "local_job_id"));
diff --git a/classes/DataWarehouse/Query/Model/Field.php b/classes/DataWarehouse/Query/Model/Field.php
index 28c2b2fdd4..91f2495607 100644
--- a/classes/DataWarehouse/Query/Model/Field.php
+++ b/classes/DataWarehouse/Query/Model/Field.php
@@ -67,7 +67,7 @@ public function getQualifiedName($show_alias = false)
$ret = $this->_def;
if ($show_alias == true && $this->getAlias() != '') {
- $ret .= ' as ' . $this->getAlias();
+ $ret .= " as '" . $this->getAlias() . "'";
}
return $ret;
diff --git a/classes/DataWarehouse/Query/Query.php b/classes/DataWarehouse/Query/Query.php
index 97e4cb39a8..03d0a2b8d8 100644
--- a/classes/DataWarehouse/Query/Query.php
+++ b/classes/DataWarehouse/Query/Query.php
@@ -676,7 +676,7 @@ public function cloneParameters(Query $other)
$this->roleParameterDescriptions = $other->roleParameterDescriptions;
}
- private function getLeftJoinSql()
+ protected function getLeftJoinSql()
{
$stmt = '';
foreach ($this->leftJoins as $joincond) {
diff --git a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php
index 5f06509739..39c1cd1fc7 100644
--- a/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php
+++ b/classes/OpenXdmod/Migration/Version812To850/ConfigFilesMigration.php
@@ -9,6 +9,7 @@
use CCR\Json;
use OpenXdmod\Migration\ConfigFilesMigration as AbstractConfigFilesMigration;
use OpenXdmod\Setup\Console;
+use OpenXdmod\Setup\WarehouseExportSetup;
class ConfigFilesMigration extends AbstractConfigFilesMigration
{
@@ -92,8 +93,21 @@ public function execute()
array('on', 'off')
);
- $this->writePortalSettingsFile(array(
- 'features_novice_user' => $novice_user
+ $console->displayMessage(<<<"EOT"
+This release of XDMoD includes support for batch exporting of data from the
+data warehouse.
+EOT
+ );
+ $console->displayBlankLine();
+ $exportSetup = new WarehouseExportSetup($console);
+ $exportSettings = $exportSetup->promptForSettings([
+ 'data_warehouse_export_export_directory' => '/var/spool/xdmod/export',
+ 'data_warehouse_export_retention_duration_days' => 30
+ ]);
+
+ $this->writePortalSettingsFile(array_merge(
+ ['features_novice_user' => $novice_user],
+ $exportSettings
));
}
diff --git a/classes/OpenXdmod/Setup/WarehouseExportSetup.php b/classes/OpenXdmod/Setup/WarehouseExportSetup.php
new file mode 100644
index 0000000000..f301cf076a
--- /dev/null
+++ b/classes/OpenXdmod/Setup/WarehouseExportSetup.php
@@ -0,0 +1,225 @@
+
+ */
+
+namespace OpenXdmod\Setup;
+
+use Exception;
+
+/**
+ * Data warehouse batch export setup.
+ */
+class WarehouseExportSetup extends SetupItem
+{
+ /**
+ * Configure data warehouse export.
+ *
+ * @see \OpenXdmod\Setup\SetupItem::handle()
+ */
+ public function handle()
+ {
+ $this->console->displaySectionHeader('Data Warehouse Batch Export');
+ $newSettings = $this->promptForSettings($this->loadIniConfig('portal_settings'));
+ $this->saveIniConfig($newSettings, 'portal_settings');
+ }
+
+ /**
+ * Prompt user for settings.
+ *
+ * This function is public so that it may also be used during the upgrade
+ * process.
+ *
+ * @param array $settings Current settings.
+ * @return array New settings.
+ */
+ public function promptForSettings(array $settings)
+ {
+ $this->console->displayMessage(<<<'MSG'
+The data warehouse batch export feature allows users to create requests to
+export data which is then generated by a cron job and stored on the server.
+The directory where this data is stored and the duration that the data will be
+retained are configurable.
+MSG
+ );
+ $this->console->displayBlankLine();
+ $exportDir = $this->console->prompt(
+ 'Export Directory:',
+ $settings['data_warehouse_export_export_directory']
+ );
+
+ try {
+ $this->checkExportDirectory($exportDir);
+ } catch (Exception $e) {
+ $this->console->displayMessage('There was an error while updating the export directory: ' . $e->getMessage());
+ $this->console->displayMessage('You must manually create the directory and set permissions');
+ }
+ $settings['data_warehouse_export_export_directory'] = $exportDir;
+
+ $settings['data_warehouse_export_retention_duration_days'] = $this->promptForRetentionDuration($settings['data_warehouse_export_retention_duration_days']);
+
+ if (empty($settings['data_warehouse_export_hash_salt'])) {
+ $settings['data_warehouse_export_hash_salt'] = bin2hex(random_bytes(32));
+ }
+
+ return $settings;
+ }
+
+ /**
+ * Prompt the user for the export file retention duration.
+ *
+ * @param int $defaultDuration The default retention duration.
+ * @return int The user's response.
+ */
+ private function promptForRetentionDuration($defaultDuration)
+ {
+ $haveResponse = false;
+ $retentionDuration = $defaultDuration;
+
+ while (!$haveResponse) {
+ $retentionDuration = $this->console->prompt(
+ 'Export File Retention Duration in Days:',
+ $defaultDuration
+ );
+
+ if (filter_var($retentionDuration, FILTER_VALIDATE_INT) !== false
+ && $retentionDuration > 0) {
+ $haveResponse = true;
+ } else {
+ $this->console->displayMessage('The export file retention duration must be a positive integer.');
+ }
+ }
+
+ return $retentionDuration;
+ }
+
+ /**
+ * Check the export directory.
+ *
+ * Checks that the export directory exists, has the correct ownership and
+ * permissions. Prompts the user if the directory needs to be created or
+ * changed.
+ *
+ * @param string $dir Path of the export directory.
+ */
+ private function checkExportDirectory($dir)
+ {
+ // Desired attributes.
+ $desiredPerms = 0570;
+ $desiredUser = 'apache';
+ $desiredGroup = 'xdmod';
+
+ $this->console->displayMessage(<<<"MSG"
+If the export directory does not exist, it must be created and assigned the
+correct permissions and ownership. It must be readable by the web server and
+both readable and writable by the user that is used to generate the export
+files. By default, the web server user is expected to be {$desiredUser} and
+the group is expected to be {$desiredGroup}. If your system uses a different
+user and group then the automatic process will fail and you must set the
+permissions manually.
+MSG
+ );
+ $this->console->displayBlankLine();
+
+ if (!is_dir($dir)) {
+ $response = $this->console->prompt(
+ 'Export directory does not exist. Create and set permissions?',
+ 'yes',
+ ['yes', 'no']
+ );
+ if ($response !== 'yes') {
+ return;
+ }
+ if (!mkdir($dir, $desiredPerms, true)) {
+ throw new Exception(sprintf(
+ 'Failed to create directory "%s"',
+ $dir
+ ));
+ }
+ if (!chmod($dir, $desiredPerms)) {
+ throw new Exception(sprintf(
+ 'Failed to change permissions of "%s"',
+ $dir
+ ));
+ }
+ if (!chown($dir, $desiredUser)) {
+ throw new Exception(sprintf(
+ 'Failed to change owner of "%s"',
+ $dir
+ ));
+ }
+ if (!chgrp($dir, $desiredGroup)) {
+ throw new Exception(sprintf(
+ 'Failed to change group of "%s"',
+ $dir
+ ));
+ }
+ }
+
+ $perms = fileperms($dir) & 0777;
+ if ($perms != $desiredPerms) {
+ $this->console->displayMessage(sprintf(
+ 'Directory permissions are "%o", expected "%o".',
+ $perms,
+ $desiredPerms
+ ));
+ $response = $this->console->prompt(
+ 'Update permissions?',
+ 'yes',
+ ['yes', 'no']
+ );
+ if ($response != 'no') {
+ if (!chmod($dir, $desiredPerms)) {
+ throw new Exception(sprintf(
+ 'Failed to change permissions of "%s"',
+ $dir
+ ));
+ }
+ }
+ }
+
+ $user = posix_getpwuid(fileowner($dir))['name'];
+ if ($user != $desiredUser) {
+ $this->console->displayMessage(sprintf(
+ 'Directory owner is "%s", expected "%s".',
+ $user,
+ $desiredUser
+ ));
+ $response = $this->console->prompt(
+ 'Update owner?',
+ 'yes',
+ ['yes', 'no']
+ );
+ if ($response != 'no') {
+ if (!chown($dir, $desiredUser)) {
+ throw new Exception(sprintf(
+ 'Failed to change owner of "%s"',
+ $dir
+ ));
+ }
+ }
+ }
+
+ $group = posix_getgrgid(filegroup($dir))['name'];
+ if ($group != $desiredGroup) {
+ $this->console->displayMessage(sprintf(
+ 'Directory group is "%s", expected "%s".',
+ $group,
+ $desiredGroup
+ ));
+ $response = $this->console->prompt(
+ 'Update group?',
+ 'yes',
+ ['yes', 'no']
+ );
+ if ($response != 'no') {
+ if (!chgrp($dir, $desiredGroup)) {
+ throw new Exception(sprintf(
+ 'Failed to change group of "%s"',
+ $dir
+ ));
+ }
+ }
+ }
+ }
+}
diff --git a/classes/Rest/Controllers/BaseControllerProvider.php b/classes/Rest/Controllers/BaseControllerProvider.php
index 67549c3447..f18718e049 100644
--- a/classes/Rest/Controllers/BaseControllerProvider.php
+++ b/classes/Rest/Controllers/BaseControllerProvider.php
@@ -563,6 +563,52 @@ protected function getDateTimeFromUnixParam(Request $request, $name, $mandatory
);
}
+ /**
+ * Attempt to get a date parameter value from a request where it is
+ * submitted as a ISO 8601 (YYYY-MM-DD) date.
+ *
+ * @param Request $request The request to extract the parameter from.
+ * @param string $name The name of the parameter.
+ * @param boolean $mandatory (Optional) If true, an exception will be
+ * thrown if the parameter is missing from the
+ * request. (Defaults to false.)
+ * @param mixed $default (Optional) The value to return if the
+ * parameter was not specified and the parameter
+ * is not mandatory. (Defaults to null.)
+ * @return mixed If available and valid, the parameter value
+ * as a DateTime. Otherwise, if it is missing
+ * and not mandatory, the given default.
+ *
+ * @throws BadRequestHttpException If the parameter was not available
+ * and the parameter was deemed mandatory,
+ * or if the parameter value could not be
+ * converted to a DateTime.
+ */
+ protected function getDateFromISO8601Param(
+ Request $request,
+ $name,
+ $mandatory = false,
+ $default = null
+ ) {
+ return $this->getParam(
+ $request,
+ $name,
+ $mandatory,
+ $default,
+ FILTER_CALLBACK,
+ [
+ 'options' => function ($value) {
+ $value_dt = \DateTime::createFromFormat('Y-m-d', $value);
+ if ($value_dt === false) {
+ return null;
+ }
+ return $value_dt;
+ },
+ ],
+ 'ISO 8601 Date'
+ );
+ }
+
/**
* Get the best match for the acceptable content type for the request, given a
* list of supported content types.
diff --git a/classes/Rest/Controllers/WarehouseExportControllerProvider.php b/classes/Rest/Controllers/WarehouseExportControllerProvider.php
new file mode 100644
index 0000000000..7a49d51430
--- /dev/null
+++ b/classes/Rest/Controllers/WarehouseExportControllerProvider.php
@@ -0,0 +1,341 @@
+fileManager = new FileManager();
+ $this->realmManager = new RealmManager();
+ $this->queryHandler = new QueryHandler();
+ }
+
+ /**
+ * Set up data warehouse export routes.
+ *
+ * @param Application $app
+ * @param ControllerCollection $controller
+ */
+ public function setupRoutes(
+ Application $app,
+ ControllerCollection $controller
+ ) {
+ $root = $this->prefix;
+ $current = get_class($this);
+ $conversions = '\Rest\Utilities\Conversions';
+
+ $controller->get("$root/realms", "$current::getRealms");
+ $controller->post("$root/request", "$current::createRequest");
+ $controller->get("$root/requests", "$current::getRequests");
+ $controller->delete("$root/requests", "$current::deleteRequests");
+
+ $controller->get("$root/download/{id}", "$current::getExportedDataFile")
+ ->assert('id', '\d+')
+ ->convert('id', "$conversions::toInt");
+
+ $controller->delete("$root/request/{id}", "$current::deleteRequest")
+ ->assert('id', '\d+')
+ ->convert('id', "$conversions::toInt");
+ }
+
+ /**
+ * Get all the realms available for exporting for the current user.
+ *
+ * @param Request $request
+ * @param Application $app
+ * @return \Symfony\Component\HttpFoundation\JsonResponse
+ * @throws \Symfony\Component\HttpKernel\Exception\UnauthorizedHttpException
+ */
+ public function getRealms(Request $request, Application $app)
+ {
+ $user = $this->authorize($request);
+ $realms = array_map(
+ function ($realm) {
+ return [
+ 'id' => $realm->getName(),
+ 'name' => $realm->getDisplay()
+ ];
+ },
+ $this->realmManager->getRealmsForUser($user)
+ );
+
+ return $app->json(
+ [
+ 'success' => true,
+ 'data' => array_values($realms),
+ 'total' => count($realms)
+ ]
+ );
+ }
+
+ /**
+ * Get all the existing export requests for the current user.
+ *
+ * @param Request $request
+ * @param Application $app
+ * @return \Symfony\Component\HttpFoundation\JsonResponse
+ * @throws \Symfony\Component\HttpKernel\Exception\UnauthorizedHttpException
+ */
+ public function getRequests(Request $request, Application $app)
+ {
+ $user = $this->authorize($request);
+ $results = $this->queryHandler->listUserRequestsByState($user->getUserId());
+ return $app->json(
+ [
+ 'success' => true,
+ 'data' => $results,
+ 'total' => count($results)
+ ]
+ );
+ }
+
+ /**
+ * Create a new export request for the current user.
+ *
+ * @param Request $request
+ * @param Application $app
+ * @return \Symfony\Component\HttpFoundation\JsonResponse
+ * @throws \Symfony\Component\HttpKernel\Exception\UnauthorizedHttpException
+ * @throws BadRequestHttpException
+ */
+ public function createRequest(Request $request, Application $app)
+ {
+ $user = $this->authorize($request);
+ $realm = $this->getStringParam($request, 'realm', true);
+
+ $realms = array_map(
+ function ($realm) {
+ return $realm->getName();
+ },
+ $this->realmManager->getRealmsForUser($user)
+ );
+ if (!in_array($realm, $realms)) {
+ throw new BadRequestHttpException('Invalid realm');
+ }
+
+ $startDate = $this->getDateFromISO8601Param($request, 'start_date', true);
+ $endDate = $this->getDateFromISO8601Param($request, 'end_date', true);
+ $now = new DateTime();
+
+ if ($startDate > $now) {
+ throw new BadRequestHttpException('Start date cannot be in the future');
+ }
+
+ if ($endDate > $now) {
+ throw new BadRequestHttpException('End date cannot be in the future');
+ }
+
+ $interval = $startDate->diff($endDate);
+
+ if ($interval === false) {
+ throw new BadRequestHttpException('Failed to calculate date interval');
+ }
+
+ if ($interval->invert === 1) {
+ throw new BadRequestHttpException('Start date must be before end date');
+ }
+
+ $format = strtoupper($this->getStringParam($request, 'format', true));
+
+ if (!in_array($format, ['CSV', 'JSON'])) {
+ throw new BadRequestHttpException('format must be CSV or JSON');
+ }
+
+ $id = $this->queryHandler->createRequestRecord(
+ $user->getUserId(),
+ $realm,
+ $startDate->format('Y-m-d'),
+ $endDate->format('Y-m-d'),
+ $format
+ );
+
+ return $app->json([
+ 'success' => true,
+ 'message' => 'Created export request',
+ 'data' => [['id' => $id]],
+ 'total' => 1
+ ]);
+ }
+
+ /**
+ * Get the requested data.
+ *
+ * @param Request $request
+ * @param Application $app
+ * @param int $id
+ * @return \Symfony\Component\HttpFoundation\BinaryFileResponse
+ * @throws \Symfony\Component\HttpKernel\Exception\UnauthorizedHttpException
+ * @throws AccessDeniedHttpException
+ * @throws NotFoundHttpException
+ * @throws BadRequestHttpException
+ */
+ public function getExportedDataFile(Request $request, Application $app, $id)
+ {
+ $user = $this->authorize($request);
+
+ $requests = array_filter(
+ $this->queryHandler->listUserRequestsByState($user->getUserId()),
+ function ($request) use ($id) {
+ return $request['id'] == $id;
+ }
+ );
+
+ if (count($requests) === 0) {
+ throw new NotFoundHttpException('Export request not found');
+ }
+
+ // Using `array_shift` because `array_filter` preserves keys so the
+ // request may not be at index 0.
+ $request = array_shift($requests);
+
+ if ($request['state'] !== 'Available') {
+ throw new BadRequestHttpException('Requested data is not available');
+ }
+
+ $file = $this->fileManager->getExportDataFilePath($id);
+
+ if (!is_file($file)) {
+ throw new NotFoundHttpException('Exported data not found');
+ }
+
+ if (!is_readable($file)) {
+ throw new AccessDeniedHttpException('Exported data is not readable');
+ }
+
+ return $app->sendFile(
+ $file,
+ 200,
+ [
+ 'Content-type' => 'application/zip',
+ 'Content-Disposition' => sprintf(
+ 'attachment; filename="%s"',
+ $this->fileManager->getZipFileName($request)
+ )
+ ]
+ );
+ }
+
+ /**
+ * Delete a single request.
+ *
+ * @param Request $request
+ * @param Application $app
+ * @param int $id
+ * @return \Symfony\Component\HttpFoundation\JsonResponse
+ * @throws \Symfony\Component\HttpKernel\Exception\UnauthorizedHttpException
+ * @throws NotFoundHttpException
+ */
+ public function deleteRequest(Request $request, Application $app, $id)
+ {
+ $user = $this->authorize($request);
+ $count = $this->queryHandler->deleteRequest($id, $user->getUserId());
+
+ if ($count === 0) {
+ throw new NotFoundHttpException('Export request not found');
+ }
+
+ return $app->json([
+ 'success' => true,
+ 'message' => 'Deleted export request',
+ 'data' => [['id' => $id]],
+ 'total' => 1
+ ]);
+ }
+
+ /**
+ * Delete multiple requests.
+ *
+ * The request body content must be a JSON encoded array of request IDs.
+ *
+ * @param Request $request
+ * @param Application $app
+ * @return \Symfony\Component\HttpFoundation\JsonResponse
+ * @throws \Symfony\Component\HttpKernel\Exception\UnauthorizedHttpException
+ * @throws NotFoundHttpException
+ */
+ public function deleteRequests(Request $request, Application $app)
+ {
+ $user = $this->authorize($request);
+
+ $requestIds = [];
+
+ try {
+ $requestIds = @json_decode($request->getContent());
+
+ if ($requestIds === null) {
+ throw new Exception('Failed to decode JSON');
+ }
+
+ if (!is_array($requestIds)) {
+ throw new Exception('Export request IDs must be in an array');
+ }
+
+ foreach ($requestIds as $id) {
+ if (!is_int($id)) {
+ throw new Exception('Export request IDs must integers');
+ }
+ }
+ } catch (Exception $e) {
+ throw new BadRequestHttpException(
+ 'Malformed HTTP request content: ' . $e->getMessage()
+ );
+ }
+
+ try {
+ $dbh = DB::factory('database');
+ $dbh->beginTransaction();
+
+ foreach ($requestIds as $id) {
+ $count = $this->queryHandler->deleteRequest($id, $user->getUserId());
+ if ($count === 0) {
+ throw new NotFoundHttpException('Export request not found');
+ }
+ }
+
+ $dbh->commit();
+ } catch (NotFoundHttpException $e) {
+ $dbh->rollBack();
+ throw $e;
+ } catch (Exception $e) {
+ $dbh->rollBack();
+ throw new BadRequestHttpException('Failed to delete export requests');
+ }
+
+ return $app->json([
+ 'success' => true,
+ 'message' => 'Deleted export requests',
+ 'data' => array_map(
+ function ($id) {
+ return ['id' => $id];
+ },
+ $requestIds
+ ),
+ 'total' => count($requestIds)
+ ]);
+ }
+}
diff --git a/configuration/cron.conf b/configuration/cron.conf
index d052029151..a06ccead0a 100644
--- a/configuration/cron.conf
+++ b/configuration/cron.conf
@@ -1,6 +1,9 @@
# Every morning at 3:00 AM -- run the report scheduler
0 3 * * * xdmod /usr/bin/php /usr/lib/xdmod/report_schedule_manager.php >/dev/null
+# Process data warehouse batch export requests.
+0 4 * * * xdmod /usr/lib/xdmod/batch_export_manager.php -q
+
# Check for updates (monthly).
0 1 1 * * xdmod /usr/lib/xdmod/update_check.php >/dev/null
diff --git a/configuration/etl/etl.d/xdb.json b/configuration/etl/etl.d/xdb.json
index 171c175977..a4bef98dd0 100644
--- a/configuration/etl/etl.d/xdb.json
+++ b/configuration/etl/etl.d/xdb.json
@@ -33,7 +33,7 @@
"class": "ManageTables",
"namespace": "ETL\\Maintenance",
"options_class": "MaintenanceOptions",
- "# order-matters": "Because foreign key constraints exist between user-roles and users the order matters",
+ "# order-matters": "Because of foreign key constraints",
"definition_file_list": [
"xdb/account-requests.json",
"xdb/api-keys.json",
@@ -52,7 +52,8 @@
"xdb/users.json",
"xdb/user-profiles.json",
"xdb/user-types.json",
- "xdb/version-check.json"
+ "xdb/version-check.json",
+ "xdb/batch-export-requests.json"
]
},
{
diff --git a/configuration/etl/etl.d/xdmod-migration-8_1_2-8_5_0.json b/configuration/etl/etl.d/xdmod-migration-8_1_2-8_5_0.json
index 576ee2df3f..497c467043 100644
--- a/configuration/etl/etl.d/xdmod-migration-8_1_2-8_5_0.json
+++ b/configuration/etl/etl.d/xdmod-migration-8_1_2-8_5_0.json
@@ -51,6 +51,7 @@
"class": "ManageTables",
"options_class": "MaintenanceOptions",
"definition_file_list": [
+ "xdb/batch-export-requests.json",
"xdb/reports.json"
],
"endpoints": {
diff --git a/configuration/etl/etl_tables.d/xdb/batch-export-requests.json b/configuration/etl/etl_tables.d/xdb/batch-export-requests.json
new file mode 100644
index 0000000000..ad6c015ce0
--- /dev/null
+++ b/configuration/etl/etl_tables.d/xdb/batch-export-requests.json
@@ -0,0 +1,133 @@
+{
+ "table_definition": {
+ "name": "batch_export_requests",
+ "comment": "Data warehouse batch export requests.",
+ "engine": "InnoDB",
+ "columns": [
+ {
+ "name": "id",
+ "type": "int(11)",
+ "nullable": false,
+ "extra": "auto_increment"
+ },
+ {
+ "name": "user_id",
+ "type": "int(11)",
+ "nullable": false,
+ "comment": "References the user that requested the export."
+ },
+ {
+ "name": "realm",
+ "type": "varchar(255)",
+ "nullable": false,
+ "comment": "The realm from which data will be exported."
+ },
+ {
+ "name": "start_date",
+ "type": "date",
+ "nullable": false,
+ "comment": "Start date for the date range of the data that will be exported."
+ },
+ {
+ "name": "end_date",
+ "type": "date",
+ "nullable": false,
+ "comment": "End date for the date range of the data that will be exported."
+ },
+ {
+ "name": "export_file_format",
+ "type": "enum('CSV','JSON')",
+ "nullable": false,
+ "comment": "File format that will be used to store the exported data."
+ },
+ {
+ "name": "requested_datetime",
+ "type": "datetime",
+ "nullable": false,
+ "comment": "Date and time the export request was created."
+ },
+ {
+ "name": "export_succeeded",
+ "type": "tinyint(1)",
+ "nullable": true,
+ "comment": "True if the export was a success, false if not, null if the request has not yet been processed."
+ },
+ {
+ "name": "export_created_datetime",
+ "type": "datetime",
+ "nullable": true,
+ "comment": "Date and time the export data was generated."
+ },
+ {
+ "name": "downloaded_datetime",
+ "type": "datetime",
+ "nullable": true,
+ "comment": "Date and time of the first download of the exported data."
+ },
+ {
+ "name": "export_expires_datetime",
+ "type": "datetime",
+ "nullable": true,
+ "comment": "Date and time the export data will expire."
+ },
+ {
+ "name": "export_expired",
+ "type": "tinyint(1)",
+ "nullable": false,
+ "default": 0,
+ "comment": "True if the export has expired, false if not."
+ },
+ {
+ "name": "last_modified",
+ "type": "timestamp",
+ "nullable": false,
+ "default": "CURRENT_TIMESTAMP",
+ "extra": "ON UPDATE CURRENT_TIMESTAMP"
+ }
+ ],
+ "indexes": [
+ {
+ "name": "PRIMARY",
+ "columns": [
+ "id"
+ ],
+ "is_unique": true
+ },
+ {
+ "name": "idx_user_id",
+ "columns": [
+ "user_id"
+ ]
+ },
+ {
+ "name": "idx_requested_datetime",
+ "columns": [
+ "requested_datetime"
+ ]
+ }
+ ],
+ "foreign_key_constraints": [
+ {
+ "name": "fk_user_id",
+ "columns": [
+ "user_id"
+ ],
+ "referenced_table": "Users",
+ "referenced_columns": [
+ "id"
+ ],
+ "on_delete": "CASCADE"
+ }
+ ],
+ "triggers": [
+ {
+ "schema": "moddb",
+ "table": "batch_export_requests",
+ "name": "batch_export_requests_before_insert",
+ "time": "BEFORE",
+ "event": "INSERT",
+ "body": "SET NEW.requested_datetime = NOW();"
+ }
+ ]
+ }
+}
diff --git a/configuration/portal_settings.ini b/configuration/portal_settings.ini
index a1befc1a4d..79c950985b 100644
--- a/configuration/portal_settings.ini
+++ b/configuration/portal_settings.ini
@@ -150,3 +150,12 @@ database = "mod_hpcdb"
[slurm]
sacct = "sacct"
+
+; Configuration for data warehouse export functionality.
+[data_warehouse_export]
+; Exported data files will be stored in this directory.
+export_directory = "/var/spool/xdmod/export"
+; Length of time in days that files will be retained before automatic deletion.
+retention_duration_days = 30
+; Salt used during deidentification.
+hash_salt = ""
diff --git a/configuration/rawstatistics.d/20_jobs.json b/configuration/rawstatistics.d/20_jobs.json
index 5481ec84c3..3362dde0e8 100644
--- a/configuration/rawstatistics.d/20_jobs.json
+++ b/configuration/rawstatistics.d/20_jobs.json
@@ -5,207 +5,342 @@
"display": "Jobs"
}
],
- "modw.job_tasks": [
- {
- "key": "local_jobid",
- "name": "Local Job Id",
- "dtype": "accounting",
- "group": "Administration",
- "documentation": "The unique identifier assigned to the job by the job scheduler."
- },
- {
- "key": "resource_id",
- "name": "Resource",
- "group": "Administration",
- "dtype": "foreignkey",
- "join": {
+ "Jobs": {
+ "tables": [
+ {
+ "schema": "modw_aggregates",
+ "name": "jobfact_by_day_joblist",
+ "alias": "jl",
+ "join": {
+ "primaryKey": "agg_id",
+ "foreignTableAlias": "jf",
+ "foreignKey": "id"
+ }
+ },
+ {
"schema": "modw",
- "table": "resourcefact"
+ "name": "job_tasks",
+ "alias": "jt",
+ "join": {
+ "primaryKey": "job_id",
+ "foreignTableAlias": "jl",
+ "foreignKey": "jobid"
+ }
},
- "documentation": "The resource that ran the job."
- },
- {
- "key": "systemaccount_id",
- "name": "System Username",
- "group": "Administration",
- "dtype": "foreignkey",
- "join": {
+ {
"schema": "modw",
- "table": "systemaccount",
- "column": "username"
+ "name": "job_records",
+ "alias": "jr",
+ "join": {
+ "primaryKey": "job_record_id",
+ "foreignTableAlias": "jt",
+ "foreignKey": "job_record_id"
+ }
},
- "documentation": "The username on the resource of the user that ran the job. May be a UID or string username depending on the resource."
- },
- {
- "key": "person_id",
- "name": "User",
- "group": "Administration",
- "dtype": "foreignkey",
- "join": {
+ {
"schema": "modw",
- "table": "person",
- "column": "long_name"
+ "name": "resourcefact",
+ "alias": "rf",
+ "join": {
+ "primaryKey": "id",
+ "foreignTableAlias": "jt",
+ "foreignKey": "resource_id"
+ }
},
- "documentation": "The name of the job owner."
- },
- {
- "key": "person_organization_id",
- "name": "Organization",
- "group": "Administration",
- "dtype": "foreignkey",
- "join": {
+ {
"schema": "modw",
- "table": "organization"
+ "name": "systemaccount",
+ "alias": "sa",
+ "join": {
+ "primaryKey": "id",
+ "foreignTableAlias": "jt",
+ "foreignKey": "systemaccount_id"
+ }
},
- "documentation": "The organization of the person who ran the task"
- },
- {
- "key": "name",
- "name": "Name",
- "documentation": "The name of the job as reported by the job scheduler.",
- "dtype": "accounting",
- "group": "Executable"
- },
- {
- "key": "submit_time_ts",
- "name": "Submit Time",
- "dtype": "accounting",
- "group": "Timing",
- "units": "ts",
- "documentation": "Task submission time"
- },
- {
- "key": "start_time_ts",
- "name": "Start Time",
- "dtype": "accounting",
- "group": "Timing",
- "units": "ts",
- "documentation": "The time that the job started running."
- },
- {
- "key": "end_time_ts",
- "name": "End Time",
- "units": "ts",
- "dtype": "accounting",
- "group": "Timing",
- "documentation": "The time that the job ended."
- },
- {
- "key": "eligible_time_ts",
- "name": "Eligible Time",
- "units": "ts",
- "dtype": "accounting",
- "group": "Timing",
- "documentation": "The time that the job was eligible for scheduling by the resource manager."
- },
- {
- "key": "node_count",
- "name": "Nodes",
- "dtype": "foreignkey",
- "group": "Allocated Resource",
- "join": {
+ {
"schema": "modw",
- "table": "nodecount",
- "column": "nodes"
+ "name": "person",
+ "alias": "p",
+ "join": {
+ "primaryKey": "id",
+ "foreignTableAlias": "jt",
+ "foreignKey": "person_id"
+ }
},
- "documentation": "The number of nodes that were assigned to the job."
- },
- {
- "key": "processor_count",
- "name": "Cores",
- "dtype": "accounting",
- "group": "Allocated Resource",
- "documentation": "The number of cores that were assigned to the job."
- },
- {
- "key": "memory_kb",
- "name": "Memory Used",
- "dtype": "accounting",
- "group": "Allocated resource",
- "units": "kilobyte",
- "documentation": "Memory consumed as reported by the resource manager."
- },
- {
- "key": "wallduration",
- "name": "Wall Time",
- "dtype": "accounting",
- "group": "Timing",
- "units": "seconds",
- "documentation": "Overall job duration."
- },
- {
- "key": "waitduration",
- "name": "Wait Time",
- "dtype": "accounting",
- "group": "Timing",
- "units": "seconds",
- "documentation": "Time the job waited in the queue"
- },
- {
- "key": "cpu_time",
- "name": "Core Time",
- "dtype": "accounting",
- "group": "Allocated resource",
- "units": "seconds",
- "documentation": "The amount of CPU core time (Core Count * Wall Time)"
- },
- {
- "key": "group_name",
- "name": "UNIX group name",
- "dtype": "accounting",
- "group": "Administration",
- "documentation": "The name of the group that ran the job."
- },
- {
- "key": "gid_number",
- "name": "UNIX group GID",
- "dtype": "accounting",
- "group": "Administration",
- "documentation": "The GID of the group that ran the job."
- },
- {
- "key": "uid_number",
- "name": "UNIX UID",
- "dtype": "accounting",
- "group": "Administration",
- "documentation": "The UID of the user that ran the job."
- },
- {
- "key": "exit_code",
- "name": "Exit Code",
- "dtype": "accounting",
- "group": "Executable",
- "documentation": "The code that the job exited with."
- },
- {
- "key": "exit_state",
- "name": "Exit State",
- "dtype": "accounting",
- "group": "Executable",
- "documentation": "The state of the job when it completed."
- },
- {
- "key": "cpu_req",
- "name": "Requested Cores",
- "dtype": "accounting",
- "group": "Requested resource",
- "documentation": "The number of CPUs required by the job."
- },
- {
- "key": "mem_req",
- "name": "Requested memory",
- "dtype": "accounting",
- "group": "Requested resource",
- "units": "bytes",
- "documentation": "The amount of memory required by the job."
- },
- {
- "key": "timelimit",
- "name": "Requested Wall Time",
- "dtype": "accounting",
- "group": "Requested resource",
- "units": "seconds",
- "documentation": "The time limit of the job."
- }
- ]
+ {
+ "schema": "modw",
+ "name": "organization",
+ "alias": "o",
+ "join": {
+ "primaryKey": "id",
+ "foreignTableAlias": "jt",
+ "foreignKey": "person_organization_id"
+ }
+ },
+ {
+ "schema": "modw",
+ "name": "nodecount",
+ "alias": "nc",
+ "join": {
+ "primaryKey": "id",
+ "foreignTableAlias": "jt",
+ "foreignKey": "node_count"
+ }
+ },
+ {
+ "schema": "modw",
+ "name": "fieldofscience_hierarchy",
+ "alias": "fos",
+ "join": {
+ "primaryKey": "id",
+ "foreignTableAlias": "jr",
+ "foreignKey": "fos_id"
+ }
+ }
+ ],
+ "fields": [
+ {
+ "name": "Local Job Id",
+ "formula": "IF(jt.local_job_array_index = -1, jt.local_jobid, CONCAT(jt.local_jobid, '[', jt.local_job_array_index, ']'))",
+ "group": "Administration",
+ "documentation": "The unique identifier assigned to the job by the job scheduler.",
+ "batchExport": true
+ },
+ {
+ "name": "Resource",
+ "tableAlias": "rf",
+ "column": "name",
+ "group": "Administration",
+ "documentation": "The resource that ran the job.",
+ "batchExport": true
+ },
+ {
+ "name": "Timezone",
+ "tableAlias": "rf",
+ "column": "timezone",
+ "group": "Administration",
+ "documentation": "The timezone of the resource.",
+ "batchExport": true
+ },
+ {
+ "name": "System Username",
+ "tableAlias": "sa",
+ "column": "username",
+ "group": "Administration",
+ "visibility": "non-public",
+ "documentation": "The username on the resource of the user that ran the job. May be a UID or string username depending on the resource.",
+ "batchExport": "anonymize"
+ },
+ {
+ "name": "User",
+ "tableAlias": "p",
+ "column": "long_name",
+ "group": "Administration",
+ "documentation": "The name of the job owner.",
+ "batchExport": true
+ },
+ {
+ "name": "Organization",
+ "tableAlias": "o",
+ "column": "name",
+ "group": "Administration",
+ "documentation": "The organization of the person who ran the task",
+ "batchExport": true
+ },
+ {
+ "name": "Name",
+ "tableAlias": "jt",
+ "column": "name",
+ "documentation": "The name of the job as reported by the job scheduler.",
+ "group": "Executable",
+ "batchExport": false
+ },
+ {
+ "name": "Submit Time",
+ "tableAlias": "jt",
+ "column": "submit_time_ts",
+ "group": "Timing",
+ "units": "ts",
+ "documentation": "Task submission time",
+ "batchExport": true
+ },
+ {
+ "name": "Start Time",
+ "tableAlias": "jt",
+ "column": "start_time_ts",
+ "group": "Timing",
+ "units": "ts",
+ "documentation": "The time that the job started running.",
+ "batchExport": true
+ },
+ {
+ "name": "End Time",
+ "tableAlias": "jt",
+ "column": "end_time_ts",
+ "units": "ts",
+ "group": "Timing",
+ "documentation": "The time that the job ended.",
+ "batchExport": true
+ },
+ {
+ "name": "Eligible Time",
+ "tableAlias": "jt",
+ "column": "eligible_time_ts",
+ "units": "ts",
+ "group": "Timing",
+ "documentation": "The time that the job was eligible for scheduling by the resource manager.",
+ "batchExport": true
+ },
+ {
+ "name": "Nodes",
+ "tableAlias": "nc",
+ "column": "nodes",
+ "group": "Allocated Resource",
+ "documentation": "The number of nodes that were assigned to the job.",
+ "batchExport": true
+ },
+ {
+ "name": "Cores",
+ "tableAlias": "jt",
+ "column": "processor_count",
+ "group": "Allocated Resource",
+ "documentation": "The number of cores that were assigned to the job.",
+ "batchExport": true
+ },
+ {
+ "name": "Memory Used",
+ "tableAlias": "jt",
+ "column": "memory_kb",
+ "group": "Allocated Resource",
+ "units": "kilobyte",
+ "documentation": "Memory consumed as reported by the resource manager.",
+ "batchExport": true
+ },
+ {
+ "name": "Wall Time",
+ "tableAlias": "jt",
+ "column": "wallduration",
+ "group": "Timing",
+ "units": "seconds",
+ "documentation": "Overall job duration.",
+ "batchExport": true
+ },
+ {
+ "name": "Wait Time",
+ "tableAlias": "jt",
+ "column": "waitduration",
+ "group": "Timing",
+ "units": "seconds",
+ "documentation": "Time the job waited in the queue",
+ "batchExport": true
+ },
+ {
+ "name": "Core Time",
+ "tableAlias": "jt",
+ "column": "cpu_time",
+ "group": "Allocated Resource",
+ "units": "seconds",
+ "documentation": "The amount of CPU core time (Core Count * Wall Time)",
+ "batchExport": true
+ },
+ {
+ "name": "UNIX group name",
+ "tableAlias": "jt",
+ "column": "group_name",
+ "group": "Administration",
+ "documentation": "The name of the group that ran the job.",
+ "batchExport": false
+ },
+ {
+ "name": "UNIX group GID",
+ "tableAlias": "jt",
+ "column": "gid_number",
+ "group": "Administration",
+ "documentation": "The GID of the group that ran the job.",
+ "batchExport": false
+ },
+ {
+ "name": "UNIX UID",
+ "tableAlias": "jt",
+ "column": "uid_number",
+ "group": "Administration",
+ "documentation": "The UID of the user that ran the job.",
+ "batchExport": false
+ },
+ {
+ "name": "Exit Code",
+ "tableAlias": "jt",
+ "column": "exit_code",
+ "group": "Executable",
+ "documentation": "The code that the job exited with.",
+ "batchExport": true
+ },
+ {
+ "name": "Exit State",
+ "tableAlias": "jt",
+ "column": "exit_state",
+ "group": "Executable",
+ "documentation": "The state of the job when it completed.",
+ "batchExport": true
+ },
+ {
+ "name": "Requested Cores",
+ "tableAlias": "jt",
+ "column": "cpu_req",
+ "group": "Requested Resource",
+ "documentation": "The number of CPUs required by the job.",
+ "batchExport": true
+ },
+ {
+ "name": "Requested memory",
+ "tableAlias": "jt",
+ "column": "mem_req",
+ "group": "Requested Resource",
+ "units": "bytes",
+ "documentation": "The amount of memory required by the job.",
+ "batchExport": true
+ },
+ {
+ "name": "Requested Wall Time",
+ "tableAlias": "jt",
+ "column": "timelimit",
+ "group": "Requested Resource",
+ "units": "seconds",
+ "documentation": "The time limit of the job.",
+ "batchExport": true
+ },
+ {
+ "name": "Queue",
+ "tableAlias": "jr",
+ "column": "queue",
+ "group": "Requested Resource",
+ "documentation": "The name of the queue to which the job was submitted.",
+ "batchExport": true
+ },
+ {
+ "name": "HIERARCHY_TOP_LEVEL_LABEL",
+ "tableAlias": "fos",
+ "column": "directorate_description",
+ "group": "Requested Resource",
+ "documentation": "HIERARCHY_TOP_LEVEL_INFO",
+ "batchExport": true
+ },
+ {
+ "name": "HIERARCHY_MIDDLE_LEVEL_LABEL",
+ "tableAlias": "fos",
+ "column": "parent_description",
+ "group": "Requested Resource",
+ "documentation": "HIERARCHY_MIDDLE_LEVEL_INFO",
+ "batchExport": true
+ },
+ {
+ "name": "HIERARCHY_BOTTOM_LEVEL_LABEL",
+ "tableAlias": "fos",
+ "column": "description",
+ "group": "Requested Resource",
+ "documentation": "HIERARCHY_BOTTOM_LEVEL_INFO",
+ "batchExport": true
+ }
+ ]
+ }
}
diff --git a/configuration/rest.d/warehouse_export.json b/configuration/rest.d/warehouse_export.json
new file mode 100644
index 0000000000..f080b22b00
--- /dev/null
+++ b/configuration/rest.d/warehouse_export.json
@@ -0,0 +1,6 @@
+{
+ "warehouse_export": {
+ "prefix": "warehouse/export",
+ "controller": "Rest\\Controllers\\WarehouseExportControllerProvider"
+ }
+}
diff --git a/configuration/roles.json b/configuration/roles.json
index 1b89c8a4c0..57b7528f6e 100644
--- a/configuration/roles.json
+++ b/configuration/roles.json
@@ -30,6 +30,15 @@
"userManualSectionName": "Metric Explorer",
"tooltip": ""
},
+ {
+ "name": "data_export",
+ "title": "Data Export",
+ "position": 500,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "userManualSectionName": "Data Export",
+ "tooltip": "Data Warehouse Batch Export"
+ },
{
"name": "report_generator",
"title": "Report Generator",
diff --git a/configuration/setup.json b/configuration/setup.json
index 0a1b7711e7..cbd81b5d04 100644
--- a/configuration/setup.json
+++ b/configuration/setup.json
@@ -30,6 +30,11 @@
"label": "Hierarchy",
"handler": "HierarchySetup"
},
+ {
+ "position": 65,
+ "label": "Data Warehouse Batch Export",
+ "handler": "WarehouseExportSetup"
+ },
{
"position": 70,
"label": "Automatically Check for Updates",
diff --git a/email_templates/batch_export_failure.template b/email_templates/batch_export_failure.template
new file mode 100644
index 0000000000..d5d646fe9b
--- /dev/null
+++ b/email_templates/batch_export_failure.template
@@ -0,0 +1,10 @@
+Dear [:first_name:],
+
+Your batch export request failed on [:current_date:] for the following reason:
+
+ [:failure_reason:]
+
+The technical staff has been notified and will investigate the issue.
+
+Sincerely,
+[:maintainer_signature:]
diff --git a/email_templates/batch_export_failure_admin_notice.template b/email_templates/batch_export_failure_admin_notice.template
new file mode 100644
index 0000000000..cddea0f996
--- /dev/null
+++ b/email_templates/batch_export_failure_admin_notice.template
@@ -0,0 +1,14 @@
+Dear Tech Support,
+
+There was an error while processing a data warehouse batch export on [:current_date:].
+
+Person Details -----------------------------------
+Name: [:user_formal_name:]
+Username: [:user_username:]
+E-Mail: [:user_email:]
+
+Exception Details --------------------------------
+Message: [:exception_message:]
+
+Stack Trace:
+[:exception_stack_trace:]
diff --git a/email_templates/batch_export_success.template b/email_templates/batch_export_success.template
new file mode 100644
index 0000000000..d3c6ba78bb
--- /dev/null
+++ b/email_templates/batch_export_success.template
@@ -0,0 +1,11 @@
+Dear [:first_name:],
+
+Your requested data was generated on [:current_date:] and is available for
+download at the following link:
+
+ [:download_url:]
+
+This data will be available until [:expiration_date:].
+
+Sincerely,
+[:maintainer_signature:]
diff --git a/etl/js/lib/etl_profile.js b/etl/js/lib/etl_profile.js
index f121ac4dd3..4e00f77f7d 100644
--- a/etl/js/lib/etl_profile.js
+++ b/etl/js/lib/etl_profile.js
@@ -713,6 +713,7 @@ ETLProfile.prototype.integrateWithXDMoD = function () {
var dtype = columns[c].dtype ? columns[c].dtype : (columns[c].queries ? "foreignkey" : "statistic" );
var group = columns[c].group ? columns[c].group : "misc";
var visibility = columns[c].visibility ? columns[c].visibility : 'public';
+ var batchExport = columns[c].batchExport ? columns[c].batchExport : false;
var name = extractandsubst(columns[c], "name");
if(!name) {
@@ -727,6 +728,7 @@ ETLProfile.prototype.integrateWithXDMoD = function () {
documentation: columns[c].comments,
dtype: dtype,
visibility: visibility,
+ batchExport: batchExport,
group: group
});
}
diff --git a/html/gui/css/viewer.css b/html/gui/css/viewer.css
index d9608602df..c71a6fc464 100644
--- a/html/gui/css/viewer.css
+++ b/html/gui/css/viewer.css
@@ -801,3 +801,12 @@ div.no-data-info {
.jobviewer_helpcontent strong {
font-weight: bold;
}
+
+/* Data warehouse export module */
+img.data-export-action-icon {
+ margin-right: 3px;
+}
+
+img.data-export-action-icon-hidden {
+ display: none;
+}
diff --git a/html/gui/js/CCR.js b/html/gui/js/CCR.js
index 45b3ba9150..2eef739fe4 100644
--- a/html/gui/js/CCR.js
+++ b/html/gui/js/CCR.js
@@ -587,7 +587,8 @@ CCR.xdmod.reporting.dirtyState = false;
CCR.xdmod.catalog = {
metric_explorer: {},
- report_generator: {}
+ report_generator: {},
+ data_export: {}
};
CCR.xdmod.ui.invertColor = function (hexTripletColor) {
diff --git a/html/gui/js/modules/DataExport.js b/html/gui/js/modules/DataExport.js
new file mode 100644
index 0000000000..8be630f199
--- /dev/null
+++ b/html/gui/js/modules/DataExport.js
@@ -0,0 +1,641 @@
+Ext.ns('XDMoD.Module.DataExport');
+
+/**
+ * Data warehouse export module.
+ */
+XDMoD.Module.DataExport = Ext.extend(XDMoD.PortalModule, {
+ module_id: 'data_export',
+ title: 'Data Export',
+ usesToolbar: false,
+
+ /**
+ * The default number of results to retrieve during paging operations.
+ *
+ * @var {Number}
+ */
+ defaultPageSize: 24,
+
+ initComponent: function () {
+ this.requestsStore = new XDMoD.Module.DataExport.RequestsStore();
+
+ this.realmsStore = new Ext.data.JsonStore({
+ url: 'rest/v1/warehouse/export/realms',
+ root: 'data',
+ fields: [
+ { name: 'id', type: 'string' },
+ { name: 'name', type: 'string' }
+ ]
+ });
+
+ this.requestForm = new XDMoD.Module.DataExport.RequestForm({
+ title: 'Create Bulk Data Export Request',
+ bodyStyle: 'padding: 5px 5px 0 5px',
+ border: false,
+ region: 'north',
+ realmsStore: this.realmsStore
+ });
+
+ this.requestsGrid = new XDMoD.Module.DataExport.RequestsGrid({
+ title: 'Status of Export Requests',
+ region: 'center',
+ margins: '2 2 2 0',
+ pageSize: this.defaultPageSize,
+ realmsStore: this.realmsStore,
+ store: this.requestsStore
+ });
+
+ // Defer loading of realms so they are not loaded immediately.
+ this.on('beforerender', this.realmsStore.load, this.realmsStore, { single: true });
+
+ // Open the download window if this is a download URL.
+ this.on('activate', function () {
+ var token = CCR.tokenize(document.location.hash);
+ var params = Ext.urlDecode(token.params);
+
+ if (params.action === 'download') {
+ // Update history so the download URL is no longer present.
+ Ext.History.add(this.id);
+
+ // A confirmation message is used because the download cannot
+ // be initiated automatically as it would be blocked as a
+ // pop-up.
+ Ext.Msg.confirm(
+ 'Data Export',
+ 'Download exported data now?',
+ function () {
+ XDMoD.Module.DataExport.openDownloadWindow(params.id);
+ }
+ );
+ }
+ }, this, { single: true });
+
+ // Load the requests after the realms have loaded. This is necessary so
+ // that the realm name can be determined from its ID when displayed in
+ // the grid.
+ this.realmsStore.on('load', this.requestsStore.load, this.requestsStore, { single: true });
+
+ // Reload the requests every time a new request is submitted.
+ this.requestForm.on('actioncomplete', this.requestsStore.reload, this.requestsStore);
+
+ // Display alert every time a new request is submitted.
+ this.requestForm.on(
+ 'actioncomplete',
+ function () {
+ Ext.Msg.alert(
+ 'Request Submitted',
+ XDMoD.Module.DataExport.requestSubmittedText
+ );
+ }
+ );
+
+ this.items = [
+ {
+ xtype: 'panel',
+ border: true,
+ width: 375,
+ split: true,
+ region: 'west',
+ margins: '2 0 0 2',
+ layout: 'vbox',
+ layoutConfig: {
+ align: 'stretch'
+ },
+ items: [
+ this.requestForm,
+ {
+ // Spacer panel
+ xtype: 'panel',
+ border: false,
+ flex: 1
+ }
+ ]
+ },
+ this.requestsGrid
+ ];
+
+ XDMoD.Module.DataExport.superclass.initComponent.call(this);
+ }
+});
+
+/**
+ * Open a new window to download the requested data.
+ */
+XDMoD.Module.DataExport.openDownloadWindow = function (requestId) {
+ window.open('rest/v1/warehouse/export/download/' + requestId);
+};
+
+/**
+ * Data export request form.
+ */
+XDMoD.Module.DataExport.RequestForm = Ext.extend(Ext.form.FormPanel, {
+ initComponent: function () {
+ this.maxDateRangeText = '1 year';
+ this.maxDateRangeInMilliseconds = 1000 * 60 * 60 * 24 * 365;
+
+ Ext.apply(this.initialConfig, {
+ method: 'POST',
+ url: 'rest/v1/warehouse/export/request'
+ });
+
+ Ext.apply(this, {
+ monitorValid: true,
+ tools: [
+ {
+ id: 'help',
+ qtip: XDMoD.Module.DataExport.createRequestHelpText
+ }
+ ],
+ items: [
+ {
+ xtype: 'fieldset',
+ style: {
+ margin: '0'
+ },
+ columnWidth: 1,
+ items: [
+ {
+ xtype: 'combo',
+ hiddenName: 'realm',
+ fieldLabel: 'Realm',
+ emptyText: 'Select a realm',
+ valueField: 'id',
+ displayField: 'name',
+ allowBlank: false,
+ editable: false,
+ triggerAction: 'all',
+ mode: 'local',
+ store: this.realmsStore
+ },
+ {
+ xtype: 'datefield',
+ name: 'start_date',
+ fieldLabel: 'Start Date',
+ emptyText: 'Start Date',
+ format: 'Y-m-d',
+ allowBlank: false,
+ validator: this.validateStartDate.bind(this)
+ },
+ {
+ xtype: 'datefield',
+ name: 'end_date',
+ fieldLabel: 'End Date',
+ emptyText: 'End Date',
+ format: 'Y-m-d',
+ allowBlank: false,
+ validator: this.validateEndDate.bind(this)
+ },
+ {
+ xtype: 'combo',
+ name: 'format',
+ fieldLabel: 'Format',
+ emptyText: 'Select an export format',
+ allowBlank: false,
+ editable: false,
+ triggerAction: 'all',
+ mode: 'local',
+ store: ['CSV', 'JSON']
+ }
+ ]
+ }
+ ],
+ buttons: [
+ {
+ xtype: 'button',
+ text: 'Submit Request',
+ formBind: true,
+ disabled: true,
+ scope: this,
+ handler: function () {
+ this.getForm().submit();
+ }
+ }
+ ]
+ });
+
+ XDMoD.Module.DataExport.RequestForm.superclass.initComponent.call(this);
+
+ this.getForm().on('actionfailed', function (form, action) {
+ switch (action.failureType) {
+ case Ext.form.Action.CLIENT_INVALID:
+ // It shouldn't be possible to submit and invalid form, but it
+ // does happen display an error message.
+ Ext.Msg.alert('Error', 'Validation failed, please check input values and resubmit.');
+ break;
+ case Ext.form.Action.CONNECT_FAILURE:
+ case Ext.form.Action.SERVER_INVALID:
+ var response = action.response;
+ Ext.Msg.alert(
+ response.statusText || 'Error',
+ JSON.parse(response.responseText).message || 'Unknown Error'
+ );
+ break;
+ case Ext.form.Action.LOAD_FAILURE:
+ // This error occurs when the server doesn't return anything.
+ Ext.Msg.alert('Submission Error', 'Failed to submit request, try again later.');
+ break;
+ default:
+ Ext.Msg.alert('Unknown Error', 'An unknown error occured, try again later.');
+ }
+ });
+ },
+
+ validateStartDate: function (date) {
+ var startDate;
+ try {
+ startDate = this.parseDate(date);
+ } catch (e) {
+ return e.message;
+ }
+
+ var endDate = this.getForm().getFieldValues().end_date;
+
+ if (endDate === '') {
+ return true;
+ }
+
+ if (startDate > Date.now()) {
+ return 'Start date cannot be in the future';
+ }
+
+ if (startDate > endDate) {
+ return 'Start date must be before the end date';
+ }
+
+ if (endDate - startDate > this.maxDateRangeInMilliseconds) {
+ return 'Date range must be less than ' + this.maxDateRangeText;
+ }
+
+ return true;
+ },
+
+ validateEndDate: function (date) {
+ var endDate;
+ try {
+ endDate = this.parseDate(date);
+ } catch (e) {
+ return e.message;
+ }
+
+ var startDate = this.getForm().getFieldValues().start_date;
+
+ if (startDate === '') {
+ return true;
+ }
+
+ if (endDate > Date.now()) {
+ return 'End date cannot be in the future';
+ }
+
+ if (startDate > endDate) {
+ return 'End date must be after the start date';
+ }
+
+ if (endDate - startDate > this.maxDateRangeInMilliseconds) {
+ return 'Date range must be less than ' + this.maxDateRangeText;
+ }
+
+ return true;
+ },
+
+ parseDate: function (date) {
+ if (Ext.isDate(date)) {
+ return date;
+ }
+
+ var format = 'Y-m-d';
+ var parsedDate = Date.parseDate(date, format);
+
+ if (parsedDate === undefined) {
+ throw new Error(date + ' is not a valid date - it must be in the format ' + format);
+ }
+
+ return parsedDate;
+ }
+});
+
+/**
+ * Data export request grid.
+ */
+XDMoD.Module.DataExport.RequestsGrid = Ext.extend(Ext.grid.GridPanel, {
+ initComponent: function () {
+ Ext.apply(this, {
+ loadMask: true,
+ tools: [
+ {
+ id: 'help',
+ qtip: XDMoD.Module.DataExport.exportStatusHelpText
+ }
+ ],
+ columns: [
+ {
+ header: 'Request Date',
+ dataIndex: 'requested_datetime',
+ xtype: 'datecolumn',
+ format: 'Y-m-d'
+ },
+ {
+ header: 'State',
+ dataIndex: 'state',
+ renderer: function (value, metaData) {
+ switch (value) {
+ case 'Available':
+ metaData.attr = 'style="background-color:#0f0"'; // eslint-disable-line no-param-reassign
+ break;
+ case 'Expired':
+ case 'Failed':
+ metaData.attr = 'style="background-color:#f00"'; // eslint-disable-line no-param-reassign
+ break;
+ case 'Submitted':
+ metaData.attr = 'style="background-color:yellow"'; // eslint-disable-line no-param-reassign
+ break;
+ default:
+ }
+ return value;
+ }
+ },
+ {
+ header: 'Realm',
+ dataIndex: 'realm_id',
+ scope: this,
+ renderer: function (value) {
+ return this.realmsStore.getById(value).get('name');
+ }
+ },
+ {
+ header: 'Data Start Date',
+ dataIndex: 'start_date',
+ xtype: 'datecolumn',
+ format: 'Y-m-d'
+ },
+ {
+ header: 'Data End Date',
+ dataIndex: 'end_date',
+ xtype: 'datecolumn',
+ format: 'Y-m-d'
+ },
+ {
+ header: 'Format',
+ dataIndex: 'export_file_format'
+ },
+ {
+ header: 'Expiration Date',
+ dataIndex: 'export_expires_datetime',
+ xtype: 'datecolumn',
+ format: 'Y-m-d'
+ },
+ {
+ header: 'Actions',
+ xtype: 'actioncolumn',
+ dataIndex: 'state',
+ scope: this,
+ // XXX: The first argument to the getClass callback should
+ // always contain the value specified by the dataIndex, but
+ // the ActionColumn renderer alters it and so is only
+ // accurate in the renderer callback. Store this value in
+ // the metaData object so it can be used by the icons.
+ //
+ // See https://docs.sencha.com/extjs/3.4.0/source/Column2.html#Ext-grid-ActionColumn-method-constructor
+ renderer: function (state, metaData) {
+ metaData.rowState = state; // eslint-disable-line no-param-reassign
+ return '';
+ },
+ items: [
+ {
+ icon: 'gui/images/report_generator/delete_report.png',
+ tooltip: 'Delete Request',
+ iconCls: 'data-export-action-icon',
+ handler: function (grid, rowIndex) {
+ this.deleteRequest(grid.store.getAt(rowIndex));
+ }
+ },
+ {
+ icon: 'gui/images/report_generator/download_report.png',
+ tooltip: 'Download Exported Data',
+ getClass: function (v, metaData) {
+ return 'data-export-action-icon' + (metaData.rowState !== 'Available' ? '-hidden' : '');
+ },
+ handler: function (grid, rowIndex) {
+ this.downloadRequest(grid.store.getAt(rowIndex));
+ }
+ },
+ {
+ icon: 'gui/images/arrow_redo.png',
+ tooltip: 'Resubmit Request',
+ getClass: function (v, metaData) {
+ return 'data-export-action-icon' + (metaData.rowState !== 'Expired' && metaData.rowState !== 'Failed' ? '-hidden' : '');
+ },
+ handler: function (grid, rowIndex) {
+ this.resubmitRequest(grid.store.getAt(rowIndex));
+ }
+ }
+ ]
+ }
+ ],
+ bbar: [
+ {
+ xtype: 'button',
+ id: 'delete-all-expired-requests-button',
+ text: 'Delete all expired requests',
+ disabled: true,
+ scope: this,
+ handler: this.deleteExpiredRequests
+ }
+ ]
+ });
+
+ XDMoD.Module.DataExport.RequestsGrid.superclass.initComponent.call(this);
+
+ // Update elements that should be enabled/disabled or masked/unmasked
+ // after the store loads.
+ this.store.on('load', function () {
+ Ext.getCmp('delete-all-expired-requests-button').setDisabled(
+ this.getExpiredRequestIds().length === 0
+ );
+
+ if (this.store.getCount() === 0) {
+ this.el.mask('No Current Requests');
+ } else {
+ this.el.unmask();
+ }
+ }, this);
+ },
+
+ getExpiredRequestIds: function () {
+ var requestIds = [];
+
+ this.store.each(function (record) {
+ if (record.get('state') === 'Expired') {
+ requestIds.push(record.get('id'));
+ }
+ });
+
+ return requestIds;
+ },
+
+ deleteExpiredRequests: function () {
+ Ext.Msg.confirm(
+ 'Delete All Expired Requests',
+ 'Are you sure that you want to delete all expired requests? You cannot undo this operation.',
+ function (selection) {
+ if (selection === 'yes') {
+ Ext.Ajax.request({
+ method: 'DELETE',
+ url: 'rest/v1/warehouse/export/requests',
+ jsonData: this.getExpiredRequestIds(),
+ scope: this,
+ success: function () {
+ this.store.reload();
+ Ext.Msg.alert(
+ 'Request Submitted',
+ XDMoD.Module.DataExport.requestSubmittedText
+ );
+ },
+ failure: function (response) {
+ Ext.Msg.alert(
+ response.statusText || 'Deletion Failure',
+ JSON.parse(response.responseText).message || 'Unknown Error'
+ );
+ }
+ });
+ }
+ },
+ this
+ );
+ },
+
+ deleteRequest: function (record) {
+ Ext.Msg.confirm(
+ 'Delete Request',
+ 'Are you sure that you want to delete this request? You cannot undo this operation.',
+ function (selection) {
+ if (selection === 'yes') {
+ Ext.Ajax.request({
+ method: 'DELETE',
+ url: 'rest/v1/warehouse/export/request/' + record.get('id'),
+ scope: this,
+ success: function () {
+ this.store.reload();
+ },
+ failure: function (response) {
+ Ext.Msg.alert(
+ response.statusText || 'Deletion Failure',
+ JSON.parse(response.responseText).message || 'Unknown Error'
+ );
+ }
+ });
+ }
+ },
+ this
+ );
+ },
+
+ downloadRequest: function (record) {
+ XDMoD.Module.DataExport.openDownloadWindow(record.get('id'));
+ },
+
+ resubmitRequest: function (record) {
+ Ext.Ajax.request({
+ method: 'POST',
+ url: 'rest/v1/warehouse/export/request',
+ params: {
+ realm: record.get('realm_id'),
+ start_date: record.get('start_date').format('Y-m-d'),
+ end_date: record.get('end_date').format('Y-m-d'),
+ format: record.get('export_file_format')
+ },
+ scope: this,
+ success: function () {
+ this.store.reload();
+ },
+ failure: function (response) {
+ Ext.Msg.alert(
+ response.statusText || 'Resubmission Failure',
+ JSON.parse(response.responseText).message || 'Unknown Error'
+ );
+ }
+ });
+ }
+});
+
+XDMoD.Module.DataExport.createRequestHelpText =
+'Create a new request to bulk export data from the data warehouse. Date ' +
+'ranges are inclusive and are limited to one year. When the exported data is ' +
+'ready you will receive an email notification.';
+
+XDMoD.Module.DataExport.exportStatusHelpText =
+'Bulk data export requests and their current statuses.
' +
+'Status descriptions:
' +
+'Submitted: Request has been submitted, but exported data is not yet ' +
+'available.
' +
+'Available: Requested data is available for download.
' +
+'Expired: Requested data has expired and is no longer available.
' +
+'Failed: Data export failed. Submit a support request for more ' +
+'information.';
+
+XDMoD.Module.DataExport.requestSubmittedText =
+'Your bulk data export request has been successfully submitted. Requests ' +
+'are typically fulfilled in 24 hours. You will recieve an email notifying ' +
+'you when your data is available.';
+
+/**
+ * Data warehouse export batch requests data store.
+ */
+XDMoD.Module.DataExport.RequestsStore = Ext.extend(Ext.data.JsonStore, {
+ constructor: function (c) {
+ var config = c || {};
+ Ext.apply(config, {
+ url: 'rest/v1/warehouse/export/requests',
+ root: 'data',
+ fields: [
+ {
+ name: 'id',
+ type: 'int'
+ },
+ {
+ name: 'realm_id',
+ type: 'string',
+ mapping: 'realm'
+ },
+ {
+ name: 'start_date',
+ type: 'date',
+ dateFormat: 'Y-m-d'
+ },
+ {
+ name: 'end_date',
+ type: 'date',
+ dateFormat: 'Y-m-d'
+ },
+ {
+ name: 'export_file_format',
+ type: 'string'
+ },
+ {
+ name: 'requested_datetime',
+ type: 'date',
+ dateFormat: 'Y-m-d H:i:s'
+ },
+ {
+ name: 'export_created_datetime',
+ type: 'date',
+ dateFormat: 'Y-m-d H:i:s'
+ },
+ {
+ name: 'export_expires_datetime',
+ type: 'date',
+ dateFormat: 'Y-m-d H:i:s'
+ },
+ {
+ name: 'export_expired',
+ type: 'boolean'
+ },
+ {
+ name: 'state',
+ type: 'string'
+ }
+ ]
+ });
+
+ XDMoD.Module.DataExport.RequestsStore.superclass.constructor.call(this, config);
+ }
+});
diff --git a/html/index.php b/html/index.php
index 50deb64140..86be2faa3d 100644
--- a/html/index.php
+++ b/html/index.php
@@ -467,6 +467,7 @@ function ($item) {
+
diff --git a/open_xdmod/modules/xdmod/xdmod.spec.in b/open_xdmod/modules/xdmod/xdmod.spec.in
index ceae625bf2..ff8a3914cd 100644
--- a/open_xdmod/modules/xdmod/xdmod.spec.in
+++ b/open_xdmod/modules/xdmod/xdmod.spec.in
@@ -27,6 +27,8 @@ from resource managers in high-performance computing environments.
XDMoD presents resource utilization over set time periods and provides
detailed interactive charts, graphs, and tables.
+%define xdmod_export_dir %{_var}/spool/%{name}/export
+
%prep
%setup -q -n %{name}-%{version}__PRERELEASE__
@@ -49,6 +51,7 @@ DESTDIR=$RPM_BUILD_ROOT ./install \
--httpdconfdir=%{_sysconfdir}/httpd/conf.d \
--logrotatedconfdir=%{_sysconfdir}/logrotate.d \
--crondconfdir=%{_sysconfdir}/cron.d
+mkdir -p $RPM_BUILD_ROOT%{xdmod_export_dir}
%post
# Create log files that need to be writable by both Apache and Open XDMoD scripts.
@@ -87,9 +90,12 @@ rm -rf $RPM_BUILD_ROOT
%config(noreplace) %{_sysconfdir}/cron.d/%{name}
%config(noreplace) %{_datadir}/%{name}/html/robots.txt
+%dir %attr(0570,apache,xdmod) %{xdmod_export_dir}
+
%changelog
* Sun Jul 28 2019 XDMoD
- Remove `node_modules` management from `%pre`
+ - Add data warehouse batch export directory `/var/spool/xdmod/export`
* Mon May 06 2019 XDMoD 8.1.2-1.0
- Update `%pre` with steps for `node_modules` on 8.1.2 release
* Thu May 02 2019 XDMoD 8.1.1-1.0
diff --git a/templates/portal_settings.template b/templates/portal_settings.template
index 0f6bf0505d..162c852020 100644
--- a/templates/portal_settings.template
+++ b/templates/portal_settings.template
@@ -150,3 +150,12 @@ database = "mod_hpcdb"
[slurm]
sacct = "[:slurm_sacct:]"
+
+; Configuration for data warehouse export functionality.
+[data_warehouse_export]
+; Exported data files will be stored in this directory.
+export_directory = "[:data_warehouse_export_export_directory:]"
+; Length of time in days that files will be retained before automatic deletion.
+retention_duration_days = [:data_warehouse_export_retention_duration_days:]
+; Salt used during deidentification.
+hash_salt = "[:data_warehouse_export_hash_salt:]"
diff --git a/tests/artifacts/xdmod/component/export/realm_manager/output/query-class-for-realm.json b/tests/artifacts/xdmod/component/export/realm_manager/output/query-class-for-realm.json
new file mode 100644
index 0000000000..73fef7817a
--- /dev/null
+++ b/tests/artifacts/xdmod/component/export/realm_manager/output/query-class-for-realm.json
@@ -0,0 +1,6 @@
+{
+ "Jobs realm": [
+ "jobs",
+ "\\DataWarehouse\\Query\\Jobs\\JobDataset"
+ ]
+}
diff --git a/tests/artifacts/xdmod/component/export/realm_manager/output/realms-for-user.json b/tests/artifacts/xdmod/component/export/realm_manager/output/realms-for-user.json
new file mode 100644
index 0000000000..38f92b3b9b
--- /dev/null
+++ b/tests/artifacts/xdmod/component/export/realm_manager/output/realms-for-user.json
@@ -0,0 +1,11 @@
+{
+ "Normal user": [
+ "usr",
+ [
+ {
+ "name": "jobs",
+ "display": "Jobs"
+ }
+ ]
+ ]
+}
diff --git a/tests/artifacts/xdmod/component/export/realm_manager/output/realms.json b/tests/artifacts/xdmod/component/export/realm_manager/output/realms.json
new file mode 100644
index 0000000000..925dc05cf0
--- /dev/null
+++ b/tests/artifacts/xdmod/component/export/realm_manager/output/realms.json
@@ -0,0 +1,10 @@
+{
+ "Batch exportable realms": [
+ [
+ {
+ "name": "jobs",
+ "display": "Jobs"
+ }
+ ]
+ ]
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/create-request.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/create-request.json
new file mode 100644
index 0000000000..6564af9e7e
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/create-request.json
@@ -0,0 +1,174 @@
+{
+ "Normal user": [
+ "usr",
+ {
+ "realm": "jobs",
+ "start_date": "2018-01-01",
+ "end_date": "2018-12-31",
+ "format": "CSV"
+ },
+ 200,
+ "POST-request"
+ ],
+ "Normal user (JSON format)": [
+ "usr",
+ {
+ "realm": "jobs",
+ "start_date": "2017-02-01",
+ "end_date": "2017-02-28",
+ "format": "JSON"
+ },
+ 200,
+ "POST-request"
+ ],
+ "PI": [
+ "pi",
+ {
+ "realm": "jobs",
+ "start_date": "2017-01-01",
+ "end_date": "2017-12-31",
+ "format": "CSV"
+ },
+ 200,
+ "POST-request"
+ ],
+ "Center director": [
+ "cd",
+ {
+ "realm": "jobs",
+ "start_date": "2016-01-01",
+ "end_date": "2016-12-31",
+ "format": "CSV"
+ },
+ 200,
+ "POST-request"
+ ],
+ "Missing realm": [
+ "usr",
+ {
+ "start_date": "2018-01-01",
+ "end_date": "2018-12-31",
+ "format": "CSV"
+ },
+ 400,
+ "error"
+ ],
+ "Missing start date": [
+ "usr",
+ {
+ "realm": "jobs",
+ "end_date": "2018-12-31",
+ "format": "CSV"
+ },
+ 400,
+ "error"
+ ],
+ "Missing end date": [
+ "usr",
+ {
+ "realm": "jobs",
+ "start_date": "2018-01-01",
+ "format": "CSV"
+ },
+ 400,
+ "error"
+ ],
+ "Missing format": [
+ "usr",
+ {
+ "realm": "jobs",
+ "start_date": "2018-01-01",
+ "end_date": "2018-12-31"
+ },
+ 400,
+ "error"
+ ],
+ "Invalid realm": [
+ "usr",
+ {
+ "realm": "foo",
+ "start_date": "2018-01-01",
+ "end_date": "2018-12-31",
+ "format": "CSV"
+ },
+ 400,
+ "error"
+ ],
+ "Invalid start date": [
+ "usr",
+ {
+ "realm": "jobs",
+ "start_date": "1 January 2018",
+ "end_date": "2018-12-31",
+ "format": "CSV"
+ },
+ 400,
+ "error"
+ ],
+ "Invalid end date": [
+ "usr",
+ {
+ "realm": "jobs",
+ "start_date": "2018-01-01",
+ "end_date": "31 December 2018",
+ "format": "CSV"
+ },
+ 400,
+ "error"
+ ],
+ "Invalid format": [
+ "usr",
+ {
+ "realm": "jobs",
+ "start_date": "2018-01-01",
+ "end_date": "2018-12-31",
+ "format": "bar"
+ },
+ 400,
+ "error"
+ ],
+ "Future start date": [
+ "usr",
+ {
+ "realm": "jobs",
+ "start_date": "9999-12-31",
+ "end_date": "2018-12-31",
+ "format": "CSV"
+ },
+ 400,
+ "error"
+ ],
+ "Future end date": [
+ "usr",
+ {
+ "realm": "jobs",
+ "start_date": "2018-01-01",
+ "end_date": "9999-12-31",
+ "format": "CSV"
+ },
+ 400,
+ "error"
+ ],
+ "Start date before end date": [
+ "usr",
+ {
+ "realm": "jobs",
+ "start_date": "2018-12-31",
+ "end_date": "2018-01-01",
+ "format": "CSV"
+ },
+ 400,
+ "error"
+ ],
+ "Public user": [
+ "pub",
+ {
+ "realm": "jobs",
+ "start_date": "2018-01-01",
+ "end_date": "2018-12-31",
+ "format": "CSV"
+ },
+ 401,
+ "error"
+ ]
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-request.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-request.json
new file mode 100644
index 0000000000..cc68732f07
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-request.json
@@ -0,0 +1,35 @@
+{
+ "Normal user": [
+ "usr",
+ {
+ "realm": "jobs",
+ "start_date": "2018-01-01",
+ "end_date": "2018-12-31",
+ "format": "CSV"
+ },
+ 200,
+ "DELETE-request"
+ ],
+ "PI": [
+ "pi",
+ {
+ "realm": "jobs",
+ "start_date": "2018-01-01",
+ "end_date": "2018-12-31",
+ "format": "CSV"
+ },
+ 200,
+ "DELETE-request"
+ ],
+ "Center director": [
+ "cd",
+ {
+ "realm": "jobs",
+ "start_date": "2018-01-01",
+ "end_date": "2018-12-31",
+ "format": "CSV"
+ },
+ 200,
+ "DELETE-request"
+ ]
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-requests.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-requests.json
new file mode 100644
index 0000000000..e83a6aef5b
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/delete-requests.json
@@ -0,0 +1,17 @@
+{
+ "Normal user": [
+ "usr",
+ 200,
+ "DELETE-requests"
+ ],
+ "PI": [
+ "pi",
+ 200,
+ "DELETE-requests"
+ ],
+ "Center director": [
+ "cd",
+ 200,
+ "DELETE-requests"
+ ]
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-realms.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-realms.json
new file mode 100644
index 0000000000..e4c16f7d97
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-realms.json
@@ -0,0 +1,63 @@
+{
+ "Normal user": [
+ "usr",
+ 200,
+ "GET-realms",
+ [
+ {
+ "id": "jobs",
+ "name": "Jobs"
+ }
+ ]
+ ],
+ "PI": [
+ "pi",
+ 200,
+ "GET-realms",
+ [
+ {
+ "id": "jobs",
+ "name": "Jobs"
+ }
+ ]
+ ],
+ "Center staff": [
+ "cs",
+ 200,
+ "GET-realms",
+ [
+ {
+ "id": "jobs",
+ "name": "Jobs"
+ }
+ ]
+ ],
+ "Center director": [
+ "cd",
+ 200,
+ "GET-realms",
+ [
+ {
+ "id": "jobs",
+ "name": "Jobs"
+ }
+ ]
+ ],
+ "Administrative user": [
+ "mgr",
+ 200,
+ "GET-realms",
+ [
+ {
+ "id": "jobs",
+ "name": "Jobs"
+ }
+ ]
+ ],
+ "Public user": [
+ "pub",
+ 401,
+ "error",
+ []
+ ]
+}
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-request.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-request.json
new file mode 100644
index 0000000000..fe51488c70
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-request.json
@@ -0,0 +1 @@
+[]
diff --git a/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-requests.json b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-requests.json
new file mode 100644
index 0000000000..cc4993d48b
--- /dev/null
+++ b/tests/artifacts/xdmod/integration/rest/warehouse-export/input/get-requests.json
@@ -0,0 +1,79 @@
+{
+ "Normal user": [
+ "usr",
+ 200,
+ "GET-requests",
+ [
+ {
+ "realm": "jobs",
+ "start_date": "2018-01-01",
+ "end_date": "2018-12-31",
+ "export_succeeded": null,
+ "export_expired": "0",
+ "export_expires_datetime": null,
+ "export_created_datetime": null,
+ "export_file_format": "CSV",
+ "state": "Submitted"
+ },
+ {
+ "realm": "jobs",
+ "start_date": "2017-02-01",
+ "end_date": "2017-02-28",
+ "export_succeeded": null,
+ "export_expired": "0",
+ "export_expires_datetime": null,
+ "export_created_datetime": null,
+ "export_file_format": "JSON",
+ "state": "Submitted"
+ }
+ ]
+ ],
+ "PI": [
+ "pi",
+ 200,
+ "GET-requests",
+ [
+ {
+ "realm": "jobs",
+ "start_date": "2017-01-01",
+ "end_date": "2017-12-31",
+ "export_succeeded": null,
+ "export_expired": "0",
+ "export_expires_datetime": null,
+ "export_created_datetime": null,
+ "export_file_format": "CSV",
+ "state": "Submitted"
+ }
+ ]
+ ],
+ "Center staff": [
+ "cs",
+ 200,
+ "GET-requests",
+ []
+ ],
+ "Center director": [
+ "cd",
+ 200,
+ "GET-requests",
+ [
+ {
+ "realm": "jobs",
+ "start_date": "2016-01-01",
+ "end_date": "2016-12-31",
+ "export_succeeded": null,
+ "export_expired": "0",
+ "export_expires_datetime": null,
+ "export_created_datetime": null,
+ "export_file_format": "CSV",
+ "state": "Submitted"
+ }
+ ]
+ ],
+ "Public user": [
+ "pub",
+ 401,
+ "error",
+ []
+ ]
+}
diff --git a/tests/artifacts/xdmod/schema/warehouse-export/DELETE-request.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/DELETE-request.schema.json
new file mode 100644
index 0000000000..3ff5b76e73
--- /dev/null
+++ b/tests/artifacts/xdmod/schema/warehouse-export/DELETE-request.schema.json
@@ -0,0 +1,26 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Response from DELETE rest/warehouse/export/request/{id}",
+ "type": "object",
+ "properties": {
+ "success": {
+ "type": "boolean"
+ },
+ "message": {
+ "type": "string"
+ },
+ "data": {
+ "type": "array",
+ "items": {
+ "type": "object"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ },
+ "required": [
+ "success",
+ "message"
+ ]
+}
diff --git a/tests/artifacts/xdmod/schema/warehouse-export/DELETE-requests.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/DELETE-requests.schema.json
new file mode 100644
index 0000000000..f467bd4985
--- /dev/null
+++ b/tests/artifacts/xdmod/schema/warehouse-export/DELETE-requests.schema.json
@@ -0,0 +1,26 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Response from DELETE rest/warehouse/export/requests",
+ "type": "object",
+ "properties": {
+ "success": {
+ "type": "boolean"
+ },
+ "message": {
+ "type": "string"
+ },
+ "data": {
+ "type": "array",
+ "items": {
+ "type": "object"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ },
+ "required": [
+ "success",
+ "message"
+ ]
+}
diff --git a/tests/artifacts/xdmod/schema/warehouse-export/GET-realms.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/GET-realms.schema.json
new file mode 100644
index 0000000000..58d7caac95
--- /dev/null
+++ b/tests/artifacts/xdmod/schema/warehouse-export/GET-realms.schema.json
@@ -0,0 +1,42 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Response from GET rest/warehouse/export/realms",
+ "type": "object",
+ "properties": {
+ "success": {
+ "type": "boolean"
+ },
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/realm"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ },
+ "required": [
+ "success",
+ "data"
+ ],
+ "additionalProperties": false,
+ "definitions": {
+ "realm": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "id",
+ "name"
+ ],
+ "additionalProperties": false
+ }
+ }
+}
diff --git a/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json
new file mode 100644
index 0000000000..219dfaf2ac
--- /dev/null
+++ b/tests/artifacts/xdmod/schema/warehouse-export/GET-requests.schema.json
@@ -0,0 +1,117 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Response from GET rest/warehouse/export/requests",
+ "type": "object",
+ "properties": {
+ "success": {
+ "type": "boolean"
+ },
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/request"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ },
+ "required": [
+ "success",
+ "data"
+ ],
+ "additionalProperties": false,
+ "definitions": {
+ "request": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "type": "integer"
+ },
+ "realm": {
+ "type": "string"
+ },
+ "start_date": {
+ "type": "string",
+ "format": "date"
+ },
+ "end_date": {
+ "type": "string",
+ "format": "date"
+ },
+ "export_succeeded": {
+ "anyOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 1
+ }
+ ]
+ },
+ "export_expired": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 1
+ },
+ "export_expires_datetime": {
+ "anyOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "string",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$"
+ }
+ ]
+ },
+ "export_created_datetime": {
+ "anyOf": [
+ {
+ "type": "null"
+ },
+ {
+ "type": "string",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$"
+ }
+ ]
+ },
+ "export_file_format": {
+ "enum": [
+ "CSV",
+ "JSON"
+ ]
+ },
+ "requested_datetime": {
+ "type": "string",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$"
+ },
+ "state": {
+ "type": "string",
+ "enum": [
+ "Submitted",
+ "Available",
+ "Failed",
+ "Expired"
+ ]
+ }
+ },
+ "required": [
+ "id",
+ "realm",
+ "start_date",
+ "end_date",
+ "export_succeeded",
+ "export_expired",
+ "export_expires_datetime",
+ "export_created_datetime",
+ "export_file_format",
+ "requested_datetime",
+ "state"
+ ],
+ "additionalProperties": false
+ }
+ }
+}
diff --git a/tests/artifacts/xdmod/schema/warehouse-export/POST-request.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/POST-request.schema.json
new file mode 100644
index 0000000000..cf034bd6af
--- /dev/null
+++ b/tests/artifacts/xdmod/schema/warehouse-export/POST-request.schema.json
@@ -0,0 +1,26 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Response from POST rest/warehouse/export/request",
+ "type": "object",
+ "properties": {
+ "success": {
+ "type": "boolean"
+ },
+ "message": {
+ "type": "string"
+ },
+ "data": {
+ "type": "array",
+ "items": {
+ "type": "object"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ },
+ "required": [
+ "success",
+ "message"
+ ]
+}
diff --git a/tests/artifacts/xdmod/schema/warehouse-export/error.schema.json b/tests/artifacts/xdmod/schema/warehouse-export/error.schema.json
new file mode 100644
index 0000000000..8f23aba744
--- /dev/null
+++ b/tests/artifacts/xdmod/schema/warehouse-export/error.schema.json
@@ -0,0 +1,18 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Generic error response",
+ "type": "object",
+ "properties": {
+ "success": {
+ "type": "boolean"
+ },
+ "message": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "success",
+ "message"
+ ],
+ "additionalProperties": true
+}
diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json
index 0368786312..b2a0f04158 100644
--- a/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json
+++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-admin.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json
index 0368786312..b2a0f04158 100644
--- a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json
+++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerdirector.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json
index 0368786312..b2a0f04158 100644
--- a/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json
+++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-centerstaff.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json
index 0368786312..b2a0f04158 100644
--- a/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json
+++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-normaluser.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json
index 0368786312..b2a0f04158 100644
--- a/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json
+++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-principal.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json
index 0368786312..b2a0f04158 100644
--- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json
+++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cd.one-center.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json
index 0368786312..b2a0f04158 100644
--- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json
+++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.cs.one-center.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json
index 0368786312..b2a0f04158 100644
--- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json
+++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.normal-user.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json
index 0368786312..b2a0f04158 100644
--- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json
+++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.pi.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json
index 0368786312..b2a0f04158 100644
--- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json
+++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_dev.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json
index 0368786312..b2a0f04158 100644
--- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json
+++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json
index 0368786312..b2a0f04158 100644
--- a/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json
+++ b/tests/artifacts/xdmod/user_admin/output/get_tabs-test.usr_mgr_dev.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-admin.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-admin.json
index ddc6d36311..05734dcae1 100644
--- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-admin.json
+++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-admin.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerdirector.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerdirector.json
index ddc6d36311..05734dcae1 100644
--- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerdirector.json
+++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerdirector.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerstaff.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerstaff.json
index ddc6d36311..05734dcae1 100644
--- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerstaff.json
+++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-centerstaff.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-normaluser.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-normaluser.json
index ddc6d36311..05734dcae1 100644
--- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-normaluser.json
+++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-normaluser.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-principal.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-principal.json
index ddc6d36311..05734dcae1 100644
--- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-principal.json
+++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-principal.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cd.one-center.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cd.one-center.json
index ddc6d36311..05734dcae1 100644
--- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cd.one-center.json
+++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cd.one-center.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cs.one-center.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cs.one-center.json
index ddc6d36311..05734dcae1 100644
--- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cs.one-center.json
+++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.cs.one-center.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.normal-user.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.normal-user.json
index ddc6d36311..05734dcae1 100644
--- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.normal-user.json
+++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.normal-user.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.pi.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.pi.json
index ddc6d36311..05734dcae1 100644
--- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.pi.json
+++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.pi.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_dev.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_dev.json
index ddc6d36311..05734dcae1 100644
--- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_dev.json
+++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_dev.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr.json
index ddc6d36311..05734dcae1 100644
--- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr.json
+++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr_dev.json b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr_dev.json
index ddc6d36311..05734dcae1 100644
--- a/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr_dev.json
+++ b/tests/artifacts/xdmod/user_admin/output/jobs/get_tabs-test.usr_mgr_dev.json
@@ -32,6 +32,17 @@
"tooltip": "",
"userManualSectionName": "Metric Explorer"
},
+ {
+ "tab": "data_export",
+ "isDefault": false,
+ "title": "Data Export",
+ "pos": 500,
+ "permitted_modules": null,
+ "javascriptClass": "XDMoD.Module.DataExport",
+ "javascriptReference": "CCR.xdmod.ui.dataExport",
+ "tooltip": "Data Warehouse Batch Export",
+ "userManualSectionName": "Data Export"
+ },
{
"tab": "report_generator",
"isDefault": false,
diff --git a/tests/ci/scripts/xdmod-setup-finish.tcl b/tests/ci/scripts/xdmod-setup-finish.tcl
index 6fc0ccac8b..cada5040ab 100644
--- a/tests/ci/scripts/xdmod-setup-finish.tcl
+++ b/tests/ci/scripts/xdmod-setup-finish.tcl
@@ -32,6 +32,12 @@ provideInput {Bottom Level Description:} {PI Group}
confirmFileWrite yes
enterToContinue
+selectMenuOption 7
+provideInput {Export Directory*} {}
+provideInput {Export File Retention Duration*} 31
+confirmFileWrite yes
+enterToContinue
+
selectMenuOption q
lassign [wait] pid spawnid os_error_flag value
diff --git a/tests/ci/scripts/xdmod-upgrade-jobs.tcl b/tests/ci/scripts/xdmod-upgrade-jobs.tcl
index 1ccb40b84a..2550589d70 100644
--- a/tests/ci/scripts/xdmod-upgrade-jobs.tcl
+++ b/tests/ci/scripts/xdmod-upgrade-jobs.tcl
@@ -22,6 +22,8 @@ set timeout 180
spawn "xdmod-upgrade"
confirmUpgrade
provideInput {Enable Novice User Tab*} {off}
+provideInput {Export Directory*} {}
+provideInput {Export File Retention Duration*} 31
expect {
-re "\nDo you want to run aggregation now.*\\\]" {
send yes\n
diff --git a/tests/ci/scripts/xdmod-upgrade.tcl b/tests/ci/scripts/xdmod-upgrade.tcl
index 39e404c720..7a1a4bbd3f 100644
--- a/tests/ci/scripts/xdmod-upgrade.tcl
+++ b/tests/ci/scripts/xdmod-upgrade.tcl
@@ -22,6 +22,8 @@ set timeout 180
spawn "xdmod-upgrade"
confirmUpgrade
provideInput {Enable Novice User Tab*} {off}
+provideInput {Export Directory*} {}
+provideInput {Export File Retention Duration*} 31
expect {
-re "\nDo you want to run aggregation now.*\\\]" {
send no\n
diff --git a/tests/component/lib/Export/BatchProcessTest.php b/tests/component/lib/Export/BatchProcessTest.php
new file mode 100644
index 0000000000..ee8ffe2a1c
--- /dev/null
+++ b/tests/component/lib/Export/BatchProcessTest.php
@@ -0,0 +1,174 @@
+ 0) {
+ $line = array_shift($lines);
+ // Check for a new message.
+ if (substr($line, 0, 5) === 'From ') {
+ // Store previous email.
+ if (!empty($currentEmail)) {
+ // Remove trailing newline.
+ $body = substr($body, 0, -1);
+ // Undo "From " escaping.
+ $body = preg_replace("/\n>([>]*From )/", "\n$1", $body);
+ $currentEmail['body'] = $body;
+ $emails[] = $currentEmail;
+ }
+ $currentEmail = [];
+ $headers = [];
+ // Parse headers.
+ while (count($lines) > 0 && $lines[0] != "\n") {
+ $line = substr(array_shift($lines), 0, -1);
+ list($key, $value) = explode(': ', $line, 2);
+ while ($lines[0][0] === "\t") {
+ $value .= ' ' . substr(substr(array_shift($lines), 0, -1), 1);
+ }
+ $headers[$key] = $value;
+ }
+ $currentEmail['headers'] = $headers;
+ $body = '';
+ // Skip blank line before body.
+ array_shift($lines);
+ } else {
+ $body .= $line;
+ }
+ }
+ // Store last email.
+ if (!empty($currentEmail)) {
+ // Remove trailing newline.
+ $body = substr($body, 0, -1);
+ // Undo "From " escaping.
+ $body = preg_replace("/\n>([>]*From )/", "\n$1", $body);
+ $currentEmail['body'] = $body;
+ $emails[] = $currentEmail;
+ }
+ return $emails;
+ }
+
+ /**
+ * Get information about all the files in the export directory.
+ *
+ * @return array[]
+ */
+ private static function getExportFiles()
+ {
+ $dir = self::$exportDirectory;
+
+ return array_map(
+ function ($file) use ($dir) {
+ return stat($dir . DIRECTORY_SEPARATOR . $file);
+ },
+ // Filter out files starting with ".".
+ array_filter(
+ scandir($dir),
+ function ($file) {
+ return $file[0] !== '.';
+ }
+ )
+ );
+ }
+
+ /**
+ * Test the batch processor dry run option.
+ */
+ public function testDryRun()
+ {
+ $batchProcessor = new BatchProcessor();
+ $batchProcessor->setDryRun(true);
+
+ // Capture state before processing requests.
+ $emails = self::getEmails();
+ $files = self::getExportFiles();
+ $submittedRequests = self::$queryHandler->listSubmittedRecords();
+ $expiringRequests = self::$queryHandler->listExpiringRecords();
+
+ $batchProcessor->processRequests();
+
+ $this->assertEquals($emails, self::getEmails(), 'No new emails');
+ $this->assertEquals($files, self::getExportFiles(), 'No new export files');
+ $this->assertEquals(
+ $submittedRequests,
+ self::$queryHandler->listSubmittedRecords(),
+ 'No submitted requests changed'
+ );
+ $this->assertEquals(
+ $expiringRequests,
+ self::$queryHandler->listExpiringRecords(),
+ 'No expiring requests changed'
+ );
+
+ $this->markTestIncomplete('This test has not been implemented yet.');
+ }
+
+ /**
+ * Test processing batch export requests.
+ */
+ public function testRequestProcessing()
+ {
+ $batchProcessor = new BatchProcessor();
+ $batchProcessor->processRequests();
+ $this->markTestIncomplete('This test has not been implemented yet.');
+ }
+}
diff --git a/tests/component/lib/Export/ExportDBTest.php b/tests/component/lib/Export/ExportDBTest.php
new file mode 100644
index 0000000000..775e87e3dc
--- /dev/null
+++ b/tests/component/lib/Export/ExportDBTest.php
@@ -0,0 +1,527 @@
+getUserID();
+ }
+
+ private function findSubmittedRecord()
+ {
+ // Find a record in Submitted status
+ return static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE export_succeeded IS NULL')[0]['id'];
+ }
+
+ private function findAvailableRecord()
+ {
+ // Find a record in Available status
+ return static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE
+ export_succeeded = 1
+ AND export_expired = 0')[0]['id'];
+ }
+
+ private function findExpiredRecord()
+ {
+ // Find a record in Expired status
+ return static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE
+ export_expired = 1')[0]['id'];
+ }
+
+ private function findFailedRecord()
+ {
+ // Find a record in Failed status
+ return static::$dbh->query('SELECT MAX(id) AS id FROM batch_export_requests WHERE
+ export_succeeded = 0')[0]['id'];
+ }
+
+ private function countSubmittedRecords()
+ {
+ // Count records in Submitted state
+ return static::$dbh->query('SELECT COUNT(id) AS count FROM batch_export_requests WHERE export_succeeded IS NULL')[0]['count'];
+ }
+
+ private function countAvailableRecords()
+ {
+ // Count records in Available state
+ return static::$dbh->query('SELECT COUNT(id) AS count FROM batch_export_requests WHERE export_succeeded = 1 and export_expired = 0')[0]['count'];
+ }
+
+ private function countExpiredRecords()
+ {
+ // Count records in Expired state
+ return static::$dbh->query('SELECT COUNT(id) AS count FROM batch_export_requests WHERE export_succeeded = 1 and export_expired = 1')[0]['count'];
+ }
+
+ private function countFailedRecords()
+ {
+ // List ids of records in Failed state
+ return static::$dbh->query('SELECT COUNT(id) AS count FROM batch_export_requests WHERE export_succeeded = 0 and export_expired = 0')[0]['count'];
+ }
+
+ private function countUserRequests()
+ {
+ // Determine number of requests placed by this user
+ $params= array('user_id' => $this->acquireUserId());
+ $sql = 'SELECT COUNT(id) AS count FROM batch_export_requests WHERE user_id=:user_id';
+ $retval = static::$dbh->query($sql, $params);
+ return $retval[0]['count'];
+ }
+
+ /* *********** PUBLIC TESTS *********** */
+
+ // Create three new records in Submitted state.
+ public function testNewRecordCreation()
+ {
+ $query = new QueryHandler();
+ $userId = $this->acquireUserId();
+
+ // Find the Submitted record count
+ $initialCount = $query->countSubmittedRecords();
+
+ // Add new record and verify
+ $requestId = $query->createRequestRecord($userId, 'Jobs', '2019-01-01', '2019-03-01', 'CSV');
+ $this->assertNotNull($requestId);
+
+ // Add another new record and verify
+ $requestId2 = $query->createRequestRecord($userId, 'Accounts', '2016-12-01', '2017-01-01', 'JSON');
+ $this->assertNotNull($requestId2);
+
+ // Add another new record and verify
+ $requestId3 = $query->createRequestRecord($userId, 'Jobs', '2014-01-05', '2014-01-26', 'CSV');
+ $this->assertNotNull($requestId3);
+
+ // Determine final count
+ $finalCount = $query->countSubmittedRecords();
+ $finalCountTest = $this->countSubmittedRecords();
+
+ $this->assertEquals(3, $finalCount - $initialCount, 'Verify final Submitted count. Should have added 3 records');
+ $this->assertEquals($finalCount, $finalCountTest, 'Verify test and class methods return same Submitted counts');
+
+ // debug
+ if (self::$debug)
+ {
+ print("\n".__FUNCTION__.": initialCount=$initialCount finalCount=$finalCount requestId=$requestId
+ requestId2=$requestId2 requestId3=$requestId3\n");
+ }
+ }
+
+ // Verify counts of Submitted records
+ public function testCountSubmitted()
+ {
+ $query = new QueryHandler();
+ $submittedCount = $query->countSubmittedRecords();
+ $submittedCountTest = $this->countSubmittedRecords();
+
+ $this->assertEquals($submittedCount, $submittedCountTest);
+ $this->assertNotNull($submittedCount);
+ $this->assertGreaterThanOrEqual(0, $submittedCount);
+
+ // debug
+ if (self::$debug)
+ {
+ print("\n".__FUNCTION__.": submittedRecords=$submittedCount\n");
+ }
+ }
+
+ // Verify field list returned from listSubmittedRecords()
+ public function testSubmittedRecordFieldList()
+ {
+ $query = new QueryHandler();
+
+ // Expect these keys from the associative array
+ $expectedKeys = array(
+ 'id',
+ 'user_id',
+ 'realm',
+ 'start_date',
+ 'end_date',
+ 'export_file_format',
+ 'requested_datetime'
+ );
+
+ // List all records in Submitted state:
+ $actual = $query->listSubmittedRecords();
+
+ $this->assertEquals($expectedKeys, array_keys($actual[0]), 'the expected fields are returned from the query');
+ $this->assertEquals($this->countSubmittedRecords(), count($actual), 'the expected number of records is returned from the query');
+ }
+
+ public function testSubmittedToFailed()
+ {
+ $query = new QueryHandler();
+
+ // initial counts
+ $submittedCountInitial = $this->countSubmittedRecords();
+ $failedCountInitial = $this->countFailedRecords();
+
+ // Find a record in submitted status to transition
+ $maxSubmitted = $this->findSubmittedRecord();
+ $result = $query->submittedToFailed($maxSubmitted);
+
+ // final counts
+ $submittedCountFinal = $this->countSubmittedRecords();
+ $failedCountFinal = $this->countFailedRecords();
+
+ $this->assertEquals(1, $result, 'Exactly one record was transitioned');
+ $this->assertEquals($submittedCountInitial - 1, $submittedCountFinal, 'There is one fewer Submitted record');
+ $this->assertEquals($failedCountInitial + 1, $failedCountFinal, 'There is one more Failed record');
+
+ // debug
+ if (self::$debug)
+ {
+ print("\n".__FUNCTION__.": transitioned Id=$maxSubmitted\n");
+ }
+ }
+
+ public function testSubmittedToExpired()
+ {
+ $query = new QueryHandler();
+
+ // initial counts
+ $submittedCountInitial = $this->countSubmittedRecords();
+ $expiredCountInitial = $this->countExpiredRecords();
+
+ // Find a record in Submitted status to transition
+ $maxSubmitted = $this->findSubmittedRecord();
+ $result = $query->availableToExpired($maxSubmitted);
+
+ // final counts
+ $submittedCountFinal = $this->countSubmittedRecords();
+ $expiredCountFinal = $this->countExpiredRecords();
+
+ $this->assertEquals(0, $result, 'Exactly zero records transitioned');
+ $this->assertEquals($submittedCountInitial, $submittedCountFinal, 'No change in Submitted counts occurred');
+ $this->assertEquals($expiredCountInitial, $expiredCountFinal, 'No change in Expired state counts occurred');
+
+ // debug
+ if (self::$debug)
+ {
+ print("\n".__FUNCTION__.": NON transitioned Id=$maxSubmitted\n");
+ }
+ }
+
+ public function testSubmittedToAvailable()
+ {
+ $query = new QueryHandler();
+
+ // initial counts
+ $submittedCountInitial = $this->countSubmittedRecords();
+ $availCountInitial = $this->countAvailableRecords();
+
+ // Find a record in Submitted status to transition
+ $maxSubmitted = $this->findSubmittedRecord();
+ $result = $query->submittedToAvailable($maxSubmitted);
+
+ // final counts
+ $submittedCountFinal = $this->countSubmittedRecords();
+ $availCountFinal = $this->countAvailableRecords();
+
+ $this->assertEquals(1, $result, 'Exactly one record was transitioned');
+ $this->assertEquals($submittedCountInitial - 1, $submittedCountFinal, 'There is one fewer Submitted record');
+ $this->assertEquals($availCountInitial + 1, $availCountFinal, 'There is one more Available record');
+
+ // debug
+ if (self::$debug)
+ {
+ print("\n".__FUNCTION__.": transitioned Id=$maxSubmitted\n");
+ }
+ }
+
+ public function testAvailableToFailed()
+ {
+ $query = new QueryHandler();
+
+ // initial counts
+ $availCountInitial = $this->countAvailableRecords();
+ $failCountInitial = $this->countFailedRecords();
+
+ // Find a record in Available status to transition
+ $maxAvailable = $this->findAvailableRecord();
+ $result = $query->submittedToFailed($maxAvailable);
+
+ // final counts
+ $availCountFinal = $this->countAvailableRecords();
+ $failCountFinal = $this->countFailedRecords();
+
+ $this->assertEquals(0, $result, 'Exactly zero records were transitioned');
+ $this->assertEquals($availCountInitial, $availCountFinal, 'No change in Available state counts occurred');
+ $this->assertEquals($failCountInitial, $failCountFinal, 'No change in Failed state counts occurred');
+
+ // debug
+ if (self::$debug)
+ {
+ print("\n".__FUNCTION__.": NON transitioned Id=$maxAvailable\n");
+ }
+ }
+
+ public function testAvailableToExpired()
+ {
+ $query = new QueryHandler();
+
+ // initial counts
+ $availCountInitial = $this->countAvailableRecords();
+ $expiredCountInitial = $this->countExpiredRecords();
+
+ // Find a record in Available status to transition
+ $maxAvailable = $this->findAvailableRecord();
+ $result = $query->availableToExpired($maxAvailable);
+
+ // final counts
+ $availCountFinal = $this->countAvailableRecords();
+ $expiredCountFinal = $this->countExpiredRecords();
+
+ $this->assertEquals(1, $result, 'Exactly one record was transitioned');
+ $this->assertEquals($availCountInitial - 1, $availCountFinal, 'There is one fewer Available record');
+ $this->assertEquals($expiredCountInitial + 1, $expiredCountFinal, 'There is one more Expired record');
+
+ // debug
+ if (self::$debug)
+ {
+ print("\n".__FUNCTION__.": transitioned Id=$maxAvailable\n");
+ }
+ }
+
+ public function testExpiredToFailed()
+ {
+ $query = new QueryHandler();
+
+ // initial counts
+ $expiredCountInitial = $this->countExpiredRecords();
+ $failCountInitial = $this->countFailedRecords();
+
+ // Find a record in Expired status to transition
+ $maxExpired = $this->findExpiredRecord();
+ $result = $query->submittedToFailed($maxExpired);
+
+ // final counts
+ $expiredCountFinal = $this->countExpiredRecords();
+ $failCountFinal = $this->countFailedRecords();
+
+ $this->assertEquals(0, $result, 'Exactly zero records were transitioned');
+ $this->assertEquals($expiredCountInitial, $expiredCountFinal, 'No change in Expired state counts occurred');
+ $this->assertEquals($failCountInitial, $failCountFinal, 'No change in Failed state counts occurred');
+
+ // debug
+ if (self::$debug)
+ {
+ print("\n".__FUNCTION__.": NON transitioned Id=$maxExpired\n");
+ }
+ }
+
+ public function testFailedToExpired()
+ {
+ $query = new QueryHandler();
+
+ // initial counts
+ $expiredCountInitial = $this->countExpiredRecords();
+ $failCountInitial = $this->countFailedRecords();
+
+ // Find or create a record in Failed status to transition
+ $maxFailed = $this->findFailedRecord();
+ $result = $query->availableToExpired($maxFailed);
+
+ // final counts
+ $expiredCountFinal = $this->countExpiredRecords();
+ $failCountFinal = $this->countFailedRecords();
+
+ $this->assertEquals(0, $result, 'Exactly zero records transitioned');
+ $this->assertEquals($expiredCountInitial, $expiredCountFinal, 'No change in Expired state counts occurred');
+ $this->assertEquals($failCountInitial, $failCountFinal, 'No change in Failed state counts occurred');
+
+ // debug
+ if (self::$debug)
+ {
+ print("\n".__FUNCTION__.": NON transitioned Id=$maxFailed\n");
+ }
+ }
+
+ public function testExpiredToAvailable()
+ {
+ $query = new QueryHandler();
+
+ // initial counts
+ $expiredCountInitial = $this->countExpiredRecords();
+ $availCountInitial = $this->countAvailableRecords();
+
+ // Find a record in Expired status to transition
+ $maxExpired = $this->findExpiredRecord();
+ $result = $query->submittedToAvailable($maxExpired);
+
+ // final counts
+ $expiredCountFinal = $this->countExpiredRecords();
+ $availCountFinal = $this->countAvailableRecords();
+
+ $this->assertEquals(0, $result, 'Exactly zero records transitioned');
+ $this->assertEquals($expiredCountInitial, $expiredCountFinal, 'No change in Expired state counts occurred');
+ $this->assertEquals($availCountInitial, $availCountFinal, 'No change in Available state counts occurred');
+
+ // debug
+ if (self::$debug)
+ {
+ print("\n".__FUNCTION__.": NON transitioned Id=$maxExpired\n");
+ }
+ }
+
+ public function testFailedToAvailable()
+ {
+ $query = new QueryHandler();
+
+ // initial counts
+ $availCountInitial = $this->countAvailableRecords();
+ $failCountInitial = $this->countFailedRecords();
+
+ // Find a record in Failed status to transition
+ $maxFailed = $this->findFailedRecord();
+ $result = $query->submittedToAvailable($maxFailed);
+
+ // final counts
+ $availCountFinal = $this->countAvailableRecords();
+ $failCountFinal = $this->countFailedRecords();
+
+ $this->assertEquals(0, $result, 'Exactly zero records transitioned');
+ $this->assertEquals($availCountInitial, $availCountFinal, 'No change in Available state counts occurred');
+ $this->assertEquals($failCountInitial, $failCountFinal, 'No change in Failed state counts occurred');
+
+ // debug
+ if (self::$debug)
+ {
+ print("\n".__FUNCTION__.": NON transitioned Id=$maxFailed\n");
+ }
+ }
+
+ // Verify field list returned from listRequestsForUser()
+ public function testUserRecordFieldList()
+ {
+ $query = new QueryHandler();
+ $userId = $this->acquireUserId();
+
+ // Expect these keys from the associative array
+ $expectedKeys = array(
+ 'id',
+ 'realm',
+ 'start_date',
+ 'end_date',
+ 'export_succeeded',
+ 'export_expired',
+ 'export_expires_datetime',
+ 'export_created_datetime',
+ 'export_file_format',
+ 'requested_datetime'
+ );
+
+ // Requests via this user have been created as part of these tests
+ $actual = $query->listRequestsForUser($userId);
+
+ $this->assertEquals($expectedKeys, array_keys($actual[0]), 'the expected fields are returned from the query');
+ $this->assertEquals($this->countUserRequests(), count($actual), 'the expected number of records is returned from the query');
+ }
+
+ // Verify field list returned from listUserRequestsByState()
+ public function testUserRecordReportStates()
+ {
+ $query = new QueryHandler();
+ $userId = $this->acquireUserId();
+
+ // Expect these keys from the associative array
+ $expectedKeys = array(
+ 'id',
+ 'realm',
+ 'start_date',
+ 'end_date',
+ 'export_succeeded',
+ 'export_expired',
+ 'export_expires_datetime',
+ 'export_created_datetime',
+ 'export_file_format',
+ 'requested_datetime',
+ 'state'
+ );
+
+ // Requests via this user have been created as part of these tests
+ $actual = $query->listUserRequestsByState($userId);
+
+ $this->assertEquals($expectedKeys, array_keys($actual[0]), 'the expected fields are returned from the query');
+ $this->assertEquals($this->countUserRequests(), count($actual), 'the expected number of records is returned from the query');
+ }
+
+ // Verify that user that did not create request cannot delete it
+ public function testRecordDeleteIncorrectUser()
+ {
+ $query = new QueryHandler();
+ $userId = $this->acquireUserId();
+ $wrongUserId = XDUser::getUserByUserName(self::CENTER_STAFF_USER_NAME)->getUserID();
+
+ // Requests via $userId have been created as part of these tests
+ $maxSubmitted = $this->findSubmittedRecord();
+
+ // Provided that we have specified two different users here:
+ if ($userId != $wrongUserId) {
+
+ // try to delete the request:
+ $actual = $query->deleteRequest($maxSubmitted, $wrongUserId);
+
+ $this->assertEquals(0, $actual, 'the delete attempt affected 0 rows');
+
+ if (self::$debug)
+ {
+ print("\n".__FUNCTION__.": deleted record id=".$maxSubmitted." ? $actual\n");
+ }
+ }
+ }
+
+ // Verify that user that created request can delete it
+ public function testRecordDeleteCorrectUser()
+ {
+ $query = new QueryHandler();
+ $userId = $this->acquireUserId();
+
+ // Requests via $userId have been created as part of these tests
+ $maxSubmitted = $this->findSubmittedRecord();
+
+ // try to delete the request:
+ $actual = $query->deleteRequest($maxSubmitted, $userId);
+
+ $this->assertEquals(1, $actual, 'the delete affected 1 row');
+
+ if (self::$debug)
+ {
+ print("\n".__FUNCTION__.": deleted record id=".$maxSubmitted." ? $actual\n");
+ }
+ }
+
+ public static function setUpBeforeClass()
+ {
+ // setup needed to use NORMAL_USER_USER_NAME or the like
+ parent::setUpBeforeClass();
+
+ // determine initial max id to enable cleanup after testing
+ static::$dbh = DB::factory('database');
+ static::$maxId = static::$dbh->query('SELECT COALESCE(MAX(id), 0) AS id FROM batch_export_requests')[0]['id'];
+ }
+
+ public static function tearDownAfterClass()
+ {
+ // Reset the batch_export_requests database table to its initial contents
+ static::$dbh->execute('DELETE FROM batch_export_requests WHERE id > :id', array('id' => static::$maxId));
+ }
+}
diff --git a/tests/component/lib/Export/RealmManagerTest.php b/tests/component/lib/Export/RealmManagerTest.php
new file mode 100644
index 0000000000..42ea882c7d
--- /dev/null
+++ b/tests/component/lib/Export/RealmManagerTest.php
@@ -0,0 +1,144 @@
+ 'Public User',
+ 'usr' => 'normaluser',
+ 'pi' => 'principal',
+ 'cs' => 'centerstaff',
+ 'cd' => 'centerdirector',
+ 'mgr' => 'admin'
+ ];
+
+ /**
+ * User for each role.
+ * @var XDUser[]
+ */
+ private static $users = [];
+
+ /**
+ */
+ public static function setUpBeforeClass()
+ {
+ parent::setUpBeforeClass();
+
+ self::$realmManager = new RealmManager();
+
+ foreach (self::$userRoles as $role => $username) {
+ //if ($role !== 'pub') {
+ self::$users[$role] = XDUser::getUserByUserName($username);
+ //}
+ }
+ }
+
+ /**
+ * Convert a realm model object to an array.
+ *
+ * Only includes the relevant properties from the realm that are used by
+ * the realm manager class.
+ *
+ * @param \Model\Realm $realm
+ * @return array
+ */
+ private function convertRealmToArray(Realm $realm)
+ {
+ return [
+ 'name' => $realm->getName(),
+ 'display' => $realm->getDisplay()
+ ];
+ }
+
+ /**
+ * Test which realms may be exported.
+ *
+ * @covers ::getRealms
+ * @dataProvider getRealmsProvider
+ */
+ public function testGetRealms($realms)
+ {
+ $this->assertEquals(
+ $realms,
+ array_map(
+ [$this, 'convertRealmToArray'],
+ self::$realmManager->getRealms()
+ ),
+ 'getRealms returns expected realms'
+ );
+ }
+
+ /**
+ * Test which realms may be exported for a given user.
+ *
+ * @covers ::getRealmsForUser
+ * @dataProvider getRealmsForUserProvider
+ */
+ public function testGetRealmsForUser($role, $realms)
+ {
+ $this->assertEquals(
+ $realms,
+ array_map(
+ [$this, 'convertRealmToArray'],
+ self::$realmManager->getRealmsForUser(self::$users[$role])
+ ),
+ "getRealmsForUser returns expected realms for role $role"
+ );
+ }
+
+ /**
+ * Test what query class should be used for each realm.
+ *
+ * @covers ::getRawDataQueryClass
+ * @dataProvider getRawDataQueryClassProvider
+ */
+ public function testGetRawDataQueryClassProvider($realmName, $queryClassName)
+ {
+ $this->assertEquals(
+ $queryClassName,
+ self::$realmManager->getRawDataQueryClass($realmName),
+ "getRawDataQueryClass returns expected query class for realm $realmName"
+ );
+ }
+
+ public function getRealmsProvider()
+ {
+ return $this->getTestFiles()->loadJsonFile(self::TEST_GROUP, 'realms', 'output');
+ }
+
+ public function getRealmsForUserProvider()
+ {
+ return $this->getTestFiles()->loadJsonFile(self::TEST_GROUP, 'realms-for-user', 'output');
+ }
+
+ public function getRawDataQueryClassProvider()
+ {
+ return $this->getTestFiles()->loadJsonFile(self::TEST_GROUP, 'query-class-for-realm', 'output');
+ }
+}
diff --git a/tests/component/phpunit.xml.dist b/tests/component/phpunit.xml.dist
index 210d3b5f8c..e1a733180e 100644
--- a/tests/component/phpunit.xml.dist
+++ b/tests/component/phpunit.xml.dist
@@ -18,6 +18,11 @@
stopOnSkipped="false"
testSuiteLoaderClass="PHPUnit_Runner_StandardTestSuiteLoader"
verbose="true">
+
+ lib/Export
+ lib/ETL
+ lib/Roles
+
lib/Roles
lib/ETL
diff --git a/tests/component/runtests.sh b/tests/component/runtests.sh
index a91b25e6e4..78084189a7 100755
--- a/tests/component/runtests.sh
+++ b/tests/component/runtests.sh
@@ -55,6 +55,8 @@ if [ 0 -ne ${#INCLUDE_GROUPS[@]} ]; then
INCLUDE_GROUP_OPTION="--group "$(implode_array , ${INCLUDE_ONLY_GROUPS[@]})
fi
+$phpunit ${PHPUNITARGS} --testsuite=Export -v $EXCLUDE_GROUP_OPTION $INCLUDE_GROUP_OPTION
+
# This test suite runs everything in lib/Roles
$phpunit ${PHPUNITARGS} --testsuite=Roles -v $EXCLUDE_GROUP_OPTION $INCLUDE_GROUP_OPTION
diff --git a/tests/integration/lib/Database/DataWarehouseExportTest.php b/tests/integration/lib/Database/DataWarehouseExportTest.php
new file mode 100644
index 0000000000..800be67a38
--- /dev/null
+++ b/tests/integration/lib/Database/DataWarehouseExportTest.php
@@ -0,0 +1,61 @@
+db = DB::factory('database');
+ $this->dbHelper = MySQLHelper::factory($this->db);
+ }
+
+ /**
+ * Test that the table used by the data warehouse export exists.
+ */
+ public function testTableExists()
+ {
+ $this->assertTrue(
+ $this->dbHelper->tableExists(self::EXPORT_REQUEST_TABLE_NAME),
+ sprintf('Table `%s` exists', self::EXPORT_REQUEST_TABLE_NAME)
+ );
+ }
+
+ /**
+ * Test that the table used by the data warehouse export is empty.
+ *
+ * @depends testTableExists
+ */
+ public function testTableEmpty()
+ {
+ list($row) = $this->db->query(
+ sprintf(
+ 'SELECT COUNT(*) AS count FROM `%s`',
+ self::EXPORT_REQUEST_TABLE_NAME
+ )
+ );
+ $this->assertEquals(
+ 0,
+ $row['count'],
+ sprintf('Table `%s` is empty', self::EXPORT_REQUEST_TABLE_NAME)
+ );
+ }
+}
diff --git a/tests/integration/lib/Rest/WarehouseExportControllerTest.php b/tests/integration/lib/Rest/WarehouseExportControllerTest.php
new file mode 100644
index 0000000000..ea2e7a39dc
--- /dev/null
+++ b/tests/integration/lib/Rest/WarehouseExportControllerTest.php
@@ -0,0 +1,435 @@
+ null,
+ 'usr' => 'normaluser',
+ 'pi' => 'principal',
+ 'cs' => 'centerstaff',
+ 'cd' => 'centerdirector',
+ 'mgr' => 'admin'
+ ];
+
+ /**
+ * User for each role.
+ * @var XDUser[]
+ */
+ private static $users = [];
+
+ /**
+ * Instances of XdmodTestHelper for each user role.
+ * @var XdmodTestHelper[]
+ */
+ private static $helpers = [];
+
+ /**
+ * Database handle.
+ * @var iDatabase
+ */
+ private static $dbh;
+
+ /**
+ * Database handle.
+ * @var QueryHandler
+ */
+ private static $queryHandler;
+
+ /**
+ * Data warehouse export file manager.
+ * @var FileManager
+ */
+ private static $fileManager;
+
+ /**
+ * JSON schema validator.
+ * @var Validator
+ */
+ private static $schemaValidator;
+
+ /**
+ * JSON schema objects.
+ * @var stdClass[]
+ */
+ private static $schemaCache = [];
+
+ /**
+ * @var TestFiles
+ */
+ private static $testFiles;
+
+ /**
+ * @return TestFiles
+ */
+ private static function getTestFiles()
+ {
+ if (!isset(self::$testFiles)) {
+ self::$testFiles = new TestFiles(__DIR__ . '/../../../');
+ }
+
+ return self::$testFiles;
+ }
+
+ /**
+ * Instantiate fixtures and authenticate helpers.
+ */
+ public static function setUpBeforeClass()
+ {
+ foreach (self::$userRoles as $role => $username) {
+ self::$helpers[$role] = new XdmodTestHelper();
+
+ if ($role !== 'pub') {
+ self::$users[$role] = XDUser::getUserByUserName($username);
+ self::$helpers[$role]->authenticate($role);
+ }
+ }
+
+ self::$dbh = DB::factory('database');
+
+ list($row) = self::$dbh->query('SELECT COUNT(*) AS count FROM batch_export_requests');
+ if ($row['count'] > 0) {
+ error_log(sprintf('Expected 0 rows in moddb.batch_export_requests, found %d', $row['count']));
+ }
+
+ self::$schemaValidator = new Validator();
+ self::$queryHandler = new QueryHandler();
+ self::$fileManager = new FileManager();
+ }
+
+ /**
+ * Logout and unset fixtures.
+ */
+ public static function tearDownAfterClass()
+ {
+ foreach (self::$helpers as $helper) {
+ $helper->logout();
+ }
+
+ // Delete any requests that weren't already deleted.
+ self::$dbh->execute('DELETE FROM batch_export_requests');
+
+ self::$users = null;
+ self::$helpers = null;
+ self::$dbh = null;
+ self::$schemaValidator = null;
+ self::$queryHandler = null;
+ self::$testFiles = null;
+ }
+
+ /**
+ * Load a JSON schema file.
+ *
+ * @return stdClass
+ */
+ private static function getSchema($schema)
+ {
+ if (!array_key_exists($schema, self::$schemaCache)) {
+ static::$schemaCache[$schema] = self::getTestFiles()->loadJsonFile(
+ 'schema',
+ $schema . '.schema',
+ 'warehouse-export',
+ false
+ );
+ }
+
+ return self::$schemaCache[$schema];
+ }
+
+ /**
+ * Validate content against a JSON schema.
+ *
+ * Test the results of the validation with an assertion.
+ *
+ * @param mixed $content The content to validate.
+ * @param string $schema The name of the schema file (without ".schema.json").
+ * @param string $message The message to use in the assertion.
+ */
+ public function validateAgainstSchema(
+ $content,
+ $schema,
+ $message = 'Validate against JSON schema'
+ ) {
+ // The content may have been decoded as an associative array so it needs
+ // to be encoded and decoded again as a stdClass before it is validated.
+ $normalizedContent = json_decode(json_encode($content));
+
+ // Data (numbers, etc.) are returned from MySQL as strings and likewise
+ // returned from the REST endpoint as string. Using
+ // CHECK_MODE_COERCE_TYPES to allow these values.
+ self::$schemaValidator->validate(
+ $normalizedContent,
+ self::getSchema($schema),
+ Constraint::CHECK_MODE_COERCE_TYPES
+ );
+
+ $errors = self::$schemaValidator->getErrors();
+ $this->assertCount(
+ 0,
+ $errors,
+ $message . "\n" . implode("\n", array_map(
+ function ($error) {
+ return sprintf("[%s] %s", $error['property'], $error['message']);
+ },
+ $errors
+ ))
+ );
+ }
+
+ /**
+ * Test getting the list of exportable realms.
+ *
+ * @param string $role Role to use during test.
+ * @param int $httpCode Expected HTTP response code.
+ * @param string $schema Name of JSON schema file that will be used
+ * to validate returned data.
+ * @param array $realms The name of the realms that are expected to
+ * be in the returned data.
+ * @covers ::getRealms
+ * @dataProvider getRealmsProvider
+ */
+ public function testGetRealms($role, $httpCode, $schema, array $realms)
+ {
+ list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/realms');
+ $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header');
+ $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code');
+ $this->validateAgainstSchema($content, $schema);
+
+ // Only check data for successful requests.
+ if ($httpCode == 200) {
+ $this->assertEquals($realms, $content['data'], 'Data contains realms');
+ }
+ }
+
+ /**
+ * Test creating a new export request.
+ *
+ * @param string $role Role to use during test.
+ * @param int $httpCode Expected HTTP response code.
+ * @param string $schema Name of JSON schema file that will be used
+ * to validate returned data.
+ * @covers ::createRequest
+ * @dataProvider createRequestProvider
+ */
+ public function testCreateRequest($role, array $params, $httpCode, $schema)
+ {
+ list($content, $info, $headers) = self::$helpers[$role]->post('rest/warehouse/export/request', null, $params);
+ $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header');
+ $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code');
+ $this->validateAgainstSchema($content, $schema);
+ }
+
+ /**
+ * Test getting the list of export requests.
+ *
+ * @param string $role Role to use during test.
+ * @param int $httpCode Expected HTTP response code.
+ * @param string $schema Name of JSON schema file that will be used
+ * to validate returned data.
+ * @param array $requests Export requests expected to exist.
+ * @covers ::getRequests
+ * @depends testCreateRequest
+ * @dataProvider getRequestsProvider
+ */
+ public function testGetRequests(
+ $role,
+ $httpCode,
+ $schema,
+ array $requests
+ ) {
+ list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/requests');
+ $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header');
+ $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code');
+ $this->validateAgainstSchema($content, $schema);
+
+ // Only check data for successful requests.
+ if ($httpCode == 200) {
+ $this->assertArraySubset($requests, $content['data'], 'Data contains requests');
+ }
+ }
+
+ /**
+ * Test getting the exported data.
+ *
+ * @covers ::getExportedDataFile
+ */
+ public function testDownloadExportedDataFile()
+ {
+ $role = 'usr';
+ $zipContent = 'Mock Zip File';
+ $id = self::$queryHandler->createRequestRecord(self::$users[$role]->getUserID(), 'jobs', '2019-01-01', '2019-01-31', 'CSV');
+ self::$queryHandler->submittedToAvailable($id);
+ @file_put_contents(self::$fileManager->getExportDataFilePath($id), $zipContent);
+ list($content, $info, $headers) = self::$helpers[$role]->get('rest/warehouse/export/download/' . $id);
+ $this->assertRegExp('#\bapplication/zip\b#', $headers['Content-Type'], 'Content type header');
+ $this->assertEquals(200, $info['http_code'], 'HTTP response code');
+ $this->assertEquals($zipContent, $content, 'Download content');
+ self::$fileManager->removeExportFile($id);
+ self::$queryHandler->deleteRequest($id, self::$users[$role]->getUserID());
+ }
+
+ /**
+ * Test deleting an export request.
+ *
+ * Creates an export and then deletes it.
+ *
+ * @param string $role Role to use during test.
+ * @param array $params Parameters to create an export request.
+ * @param int $httpCode Expected HTTP response code.
+ * @param string $schema Name of JSON schema file that will be used
+ * to validate returned data.
+ * @covers ::deleteRequest
+ * @uses ::createRequest
+ * @uses ::getRequests
+ * @dataProvider deleteRequestProvider
+ */
+ public function testDeleteRequest($role, array $params, $httpCode, $schema)
+ {
+ // Get list of requests before deletion.
+ list($beforeContent) = self::$helpers[$role]->get('rest/warehouse/export/requests');
+ $dataBefore = $beforeContent['data'];
+
+ list($createContent) = self::$helpers[$role]->post('rest/warehouse/export/request', null, $params);
+ $id = $createContent['data'][0]['id'];
+
+ list($content, $info, $headers) = self::$helpers[$role]->delete('rest/warehouse/export/request/' . $id);
+ $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header');
+ $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code');
+ $this->validateAgainstSchema($content, $schema);
+ $this->assertEquals($id, $content['data'][0]['id'], 'Deleted ID is in response');
+
+ // Get list of requests after deletion
+ list($afterContent) = self::$helpers[$role]->get('rest/warehouse/export/requests');
+ $dataAfter = $afterContent['data'];
+
+ $this->assertEquals($dataBefore, $dataAfter, 'Data before and after creation/deletion are the same.');
+ }
+
+ /**
+ * Test deleting an export request in cases where it is expected to fail.
+ *
+ * @covers ::deleteRequest
+ */
+ public function testDeleteRequestErrors()
+ {
+ // Public user can't delete anything.
+ list($content, $info, $headers) = self::$helpers['pub']->delete('rest/warehouse/export/request/1');
+ $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header');
+ $this->assertEquals(401, $info['http_code'], 'HTTP response code');
+ $this->validateAgainstSchema($content, 'error');
+
+ // Non-integer ID.
+ list($content, $info, $headers) = self::$helpers['usr']->delete('rest/warehouse/export/request/abc');
+ $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header');
+ $this->assertEquals(404, $info['http_code'], 'HTTP response code');
+ $this->validateAgainstSchema($content, 'error');
+
+ // Trying to delete a non-existent request.
+ list($row) = self::$dbh->query('SELECT MAX(id) + 1 AS id FROM batch_export_requests');
+ list($content, $info, $headers) = self::$helpers['usr']->delete('rest/warehouse/export/request/' . $row['id']);
+ $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header');
+ $this->assertEquals(404, $info['http_code'], 'HTTP response code');
+ $this->validateAgainstSchema($content, 'error');
+
+ // Trying to delete another user's request.
+ list($row) = self::$dbh->query('SELECT id FROM batch_export_requests WHERE user_id = :user_id LIMIT 1', ['user_id' => self::$users['pi']->getUserId()]);
+ list($content, $info, $headers) = self::$helpers['usr']->delete('rest/warehouse/export/request/' . $row['id']);
+ $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header');
+ $this->assertEquals(404, $info['http_code'], 'HTTP response code');
+ $this->validateAgainstSchema($content, 'error');
+ }
+
+ /**
+ * Test deleting multiple export requests at a time.
+ *
+ * @param string $role Role to use during test.
+ * @param int $httpCode Expected HTTP response code.
+ * @param string $schema Name of JSON schema file that will be used
+ * to validate returned data.
+ * @covers ::deleteRequests
+ * @uses ::getRequests
+ * @dataProvider deleteRequestsProvider
+ */
+ public function testDeleteRequests($role, $httpCode, $schema)
+ {
+ // Get list of requests before deletion.
+ list($beforeContent) = self::$helpers[$role]->get('rest/warehouse/export/requests');
+
+ // Gather ID values and also convert to integers for the array
+ // comparison done below.
+ $ids = [];
+ foreach ($beforeContent['data'] as &$datum) {
+ $datum['id'] = (int)$datum['id'];
+ $ids[] = $datum['id'];
+ }
+ $data = json_encode($ids);
+ //$this->assertTrue(false, json_encode($beforeContent['data']));
+
+ // Delete all existing requests.
+ list($content, $info, $headers) = self::$helpers[$role]->delete('rest/warehouse/export/requests', null, $data);
+ $this->assertRegExp('#\bapplication/json\b#', $headers['Content-Type'], 'Content type header');
+ $this->assertEquals($httpCode, $info['http_code'], 'HTTP response code');
+ $this->validateAgainstSchema($content, $schema);
+ $this->assertArraySubset($content['data'], $beforeContent['data'], 'Deleted IDs are in response');
+
+ // Get list of requests after deletion
+ list($afterContent) = self::$helpers[$role]->get('rest/warehouse/export/requests');
+ $this->assertEquals([], $afterContent['data'], 'Data after deletion is empty.');
+ }
+
+ public function getRealmsProvider()
+ {
+ return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'get-realms', 'input');
+ }
+
+ public function createRequestProvider()
+ {
+ return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'create-request', 'input');
+ }
+
+ public function getRequestsProvider()
+ {
+ return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'get-requests', 'input');
+ }
+
+ public function getRequestProvider()
+ {
+ return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'get-request', 'input');
+ }
+
+ public function deleteRequestProvider()
+ {
+ return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'delete-request', 'input');
+ }
+
+ public function deleteRequestsProvider()
+ {
+ return self::getTestFiles()->loadJsonFile(self::TEST_GROUP, 'delete-requests', 'input');
+ }
+}
diff --git a/tests/integration/lib/TestHarness/TestFiles.php b/tests/integration/lib/TestHarness/TestFiles.php
index 570b7f585a..9f5899c542 100644
--- a/tests/integration/lib/TestHarness/TestFiles.php
+++ b/tests/integration/lib/TestHarness/TestFiles.php
@@ -2,6 +2,8 @@
namespace TestHarness;
+use CCR\Json;
+
class TestFiles
{
const TEST_ARTIFACT_OUTPUT_PATH = './artifacts';
@@ -63,4 +65,16 @@ public function getFile($testGroup, $fileName, $type = 'output', $extension = '.
)
));
}
+
+ public function loadJsonFile(
+ $testGroup,
+ $fileName,
+ $type = '',
+ $assoc = true
+ ) {
+ return Json::loadfile(
+ $this->getFile($testGroup, $fileName, $type),
+ $assoc
+ );
+ }
}
diff --git a/tests/integration/lib/TestHarness/XdmodTestHelper.php b/tests/integration/lib/TestHarness/XdmodTestHelper.php
index abc74282d2..bd5802fd4e 100644
--- a/tests/integration/lib/TestHarness/XdmodTestHelper.php
+++ b/tests/integration/lib/TestHarness/XdmodTestHelper.php
@@ -22,33 +22,49 @@ public function __construct($config = array())
$this->headers = array();
$this->decodeTextAsJson = false;
- $this->curl = curl_init();
+ $this->cookiefile = tempnam(sys_get_temp_dir(), "xdmodtestcookies.");
+
+ if (isset($config['decodetextasjson'])) {
+ $this->decodeTextAsJson = true;
+ }
+ if (isset($config['verbose'])) {
+ $this->verbose = true;
+ }
+ $this->resetCurlSession();
+ }
+
+ /**
+ * Reset the cURL session.
+ *
+ * This function must be called after any use of CURLOPT_CUSTOMREQUEST to
+ * reset the request type.
+ */
+ private function resetCurlSession()
+ {
+ // Close existing session to write cookies to file.
+ if (isset($this->curl)) {
+ curl_close($this->curl);
+ }
+
+ $this->curl = curl_init();
curl_setopt($this->curl, CURLOPT_USERAGENT, "XDMoD REST Test harness");
curl_setopt($this->curl, CURLOPT_RETURNTRANSFER, true);
curl_setopt($this->curl, CURLOPT_FOLLOWLOCATION, true);
# Enable header information in the response data
- curl_setopt($this->curl, CURLOPT_RETURNTRANSFER, 1);
- curl_setopt($this->curl, CURLOPT_HEADERFUNCTION, array(&$this, 'processResponseHeader'));
+ curl_setopt($this->curl, CURLOPT_HEADERFUNCTION, array($this, 'processResponseHeader'));
# Disable ssl certificate checks (needed when using self-signed certificates).
curl_setopt($this->curl, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($this->curl, CURLOPT_SSL_VERIFYPEER, false);
- $this->cookiefile = tempnam(sys_get_temp_dir(), "xdmodtestcookies.");
curl_setopt($this->curl, CURLOPT_COOKIEFILE, $this->cookiefile);
+ curl_setopt($this->curl, CURLOPT_COOKIEJAR, $this->cookiefile);
if (isset($this->cookie)) {
curl_setopt($this->curl, CURLOPT_COOKIE, $this->cookie);
}
-
- if (isset($config['decodetextasjson'])) {
- $this->decodeTextAsJson = true;
- }
- if (isset($config['verbose'])) {
- $this->verbose = true;
- }
}
private function processResponseHeader($curl, $headerline)
@@ -273,7 +289,7 @@ private function docurl()
return array($content, $curlinfo, $this->responseHeaders);
}
- public function delete($path, $params = null)
+ public function delete($path, $params = null, $data = null)
{
$url = $this->siteurl . $path;
if ($params !== null) {
@@ -281,11 +297,21 @@ public function delete($path, $params = null)
}
curl_setopt($this->curl, CURLOPT_URL, $url);
- curl_setopt($this->curl, CURLOPT_POST, false);
+
+ if ($data === null) {
+ curl_setopt($this->curl, CURLOPT_POST, false);
+ } else {
+ curl_setopt($this->curl, CURLOPT_POST, true);
+ curl_setopt($this->curl, CURLOPT_POSTFIELDS, $data);
+ }
+
curl_setopt($this->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_setopt($this->curl, CURLOPT_HTTPHEADER, $this->getheaders());
- return $this->docurl();
+ $response = $this->docurl();
+ $this->resetCurlSession();
+
+ return $response;
}
public function get($path, $params = null, $isurl = false)
@@ -355,8 +381,8 @@ public function patch($path, $params = null, $data = null)
curl_setopt($this->curl, CURLOPT_CUSTOMREQUEST, 'PATCH');
curl_setopt($this->curl, CURLOPT_HTTPHEADER, $this->getheaders());
$response = $this->docurl();
+ $this->resetCurlSession();
- curl_setopt($this->curl, CURLOPT_CUSTOMREQUEST, null);
return $response;
}
public function getSiteurl(){