Add cloud user and system account group by

classes/DataWarehouse/Query/Cloud/GroupBys/GroupByPerson.php

@@ -128,29 +128,4 @@ public function pullQueryParameterDescriptions(&$request)
             "select long_name as field_label from modw.person  where id in (_filter_) order by order_id"
         );
     }
-
-    public function getPossibleValues($hint = null, $limit = null, $offset = null, array $parameters = array())
-    {
-        if ($this->_possible_values_query == null) {
-            return array();
-        }
-
-        $possible_values_query = $this->_possible_values_query;
-
-        foreach ($parameters as $pname => $pvalue) {
-            if ($pname == 'person') {
-                $possible_values_query = str_ireplace('where ', "where gt.id = $pvalue and ", $possible_values_query);
-            } elseif ($pname == 'provider') {//find the names all the people that have accounts on the resources at the provider.
-                $possible_values_query = str_ireplace('from ', "from modw.systemaccount sa,  modw.resourcefact rf, ", $possible_values_query);
-                $possible_values_query = str_ireplace('where ', "where rf.id = sa.resource_id and rf.organization_id = $pvalue and gt.id = sa.person_id  and ", $possible_values_query);
-            } elseif ($pname == 'institution') {
-                $possible_values_query = str_ireplace('where ', "where gt.organization_id = $pvalue  and ", $possible_values_query);
-            } elseif ($pname == 'pi') {
-                $possible_values_query = str_ireplace('from ', "from modw.peopleunderpi pup, ", $possible_values_query);
-                $possible_values_query = str_ireplace('where ', "where pup.principalinvestigator_person_id = $pvalue and gt.id = pup.person_id  and ", $possible_values_query);
-            }
-        }
-
-        return parent::getPossibleValues($hint, $limit, $offset, $parameters, $possible_values_query);
-    }
 }

classes/DataWarehouse/Query/Cloud/GroupBys/GroupByUsername.php

@@ -0,0 +1,131 @@
+<?php
+
+namespace DataWarehouse\Query\Cloud\GroupBys;
+
+/*
+* @author Greg Dean
+* @date 2019-01-27
+*
+* class for adding group by system username to a query
+*
+*/
+
+class GroupByUsername extends \DataWarehouse\Query\Cloud\GroupBy
+{
+    public static function getLabel()
+    {
+        return 'System Username';
+    }
+
+    public function getInfo()
+    {
+        return "The specific system username associated with a running session of a virtual machine.";
+    }
+    public function __construct()
+    {
+        parent::__construct(
+            'username',
+            array(),
+            "select distinct
+                gt.username as id,
+                gt.username as short_name,
+                gt.username as long_name
+                from systemaccount gt
+                where 1
+                order by gt.username",
+            array()
+        );
+        $this->_id_field_name = 'username';
+        $this->_short_name_field_name = 'username';
+        $this->_long_name_field_name = 'username';
+        $this->_order_id_field_name = 'username';
+        $this->modw_schema = new \DataWarehouse\Query\Model\Schema('modw');
+        $this->systemaccount_table = new \DataWarehouse\Query\Model\Table($this->modw_schema, 'systemaccount', 'sa');
+    }
+
+    public function applyTo(\DataWarehouse\Query\Query &$query, \DataWarehouse\Query\Model\Table $data_table, $multi_group = false)
+    {
+        $query->addTable($this->systemaccount_table);
+
+        $systemaccounttable_id_field = new \DataWarehouse\Query\Model\TableField($this->systemaccount_table, 'id');
+        $datatable_systemaccount_id_field = new \DataWarehouse\Query\Model\TableField($data_table, 'systemaccount_id');
+
+        $query->addWhereCondition(new \DataWarehouse\Query\Model\WhereCondition(
+            $systemaccounttable_id_field,
+            '=',
+            $datatable_systemaccount_id_field
+        ));
+
+        $id_field = new \DataWarehouse\Query\Model\TableField($this->systemaccount_table, $this->_id_field_name, $this->getIdColumnName($multi_group));
+        $name_field =  new \DataWarehouse\Query\Model\TableField($this->systemaccount_table, $this->_long_name_field_name, $this->getLongNameColumnName($multi_group));
+        $shortname_field =  new \DataWarehouse\Query\Model\TableField($this->systemaccount_table, $this->_short_name_field_name, $this->getShortNameColumnName($multi_group));
+        $order_id_field = new \DataWarehouse\Query\Model\TableField($this->systemaccount_table, $this->_order_id_field_name, $this->getOrderIdColumnName($multi_group));
+
+        $query->addField($order_id_field);
+        $query->addField($id_field);
+        $query->addField($name_field);
+        $query->addField($shortname_field);
+
+        $query->addGroup($id_field);
+
+        $this->addOrder($query, $multi_group);
+    }
+
+    public function addWhereJoin(
+        \DataWarehouse\Query\Query &$query,
+        \DataWarehouse\Query\Model\Table $data_table,
+        $multi_group,
+        $operation,
+        $whereConstraint
+    ) {
+        // construct the join between the main data_table and this group by table
+        $query->addTable($this->systemaccount_table);
+
+        $systemaccounttable_id_field = new \DataWarehouse\Query\Model\TableField($this->systemaccount_table, 'id');
+        $datatable_systemaccount_id_field = new \DataWarehouse\Query\Model\TableField($data_table, 'systemaccount_id');
+
+        // the where condition that specifies the join of the tables
+        $query->addWhereCondition(new \DataWarehouse\Query\Model\WhereCondition(
+            $systemaccounttable_id_field,
+            '=',
+            $datatable_systemaccount_id_field
+        ));
+
+        // the where condition that specifies the constraint on the joined table
+        // note that the where condition applies to strings
+        if (is_array($whereConstraint)) {
+            $whereConstraint="('". implode("','", $whereConstraint) ."')";
+        }
+
+        $query->addWhereCondition(
+            new \DataWarehouse\Query\Model\WhereCondition(
+                $systemaccounttable_id_field,
+                $operation,
+                $whereConstraint
+            )
+        );
+    } // addWhereJoin
+
+    public function addOrder(\DataWarehouse\Query\Query &$query, $multi_group = false, $dir = 'asc', $prepend = false)
+    {
+        $orderField = new \DataWarehouse\Query\Model\OrderBy(new \DataWarehouse\Query\Model\TableField($this->systemaccount_table, $this->_order_id_field_name), $dir, $this->getName());
+        if ($prepend === true) {
+            $query->prependOrder($orderField);
+        } else {
+            $query->addOrder($orderField);
+        }
+    }
+
+    public function pullQueryParameters(&$request)
+    {
+        return parent::pullQueryParameters2($request, 'select id from modw.systemaccount where username in (_filter_)', 'systemaccount_id');
+    }
+
+    public function pullQueryParameterDescriptions(&$request)
+    {
+        return parent::pullQueryParameterDescriptions2(
+            $request,
+            "select distinct username as field_label from modw.systemaccount  where username in (_filter_) order by username"
+        );
+    }
+}

classes/OpenXdmod/DataWarehouseInitializer.php

@@ -207,7 +207,7 @@ public function ingestCloudDataOpenStack()
         if( $this->isRealmEnabled('Cloud') ){
             try{
                 $this->logger->notice('Ingesting OpenStack event log data');
-                Utilities::runEtlPipeline(array('jobs-cloud-extract-openstack'), $this->logger);
+                Utilities::runEtlPipeline(array('jobs-cloud-import-users-openstack', 'jobs-cloud-extract-openstack'), $this->logger);
             }
             catch( Exception $e ){
                 if( $e->getCode() == 1146 ){
@@ -230,7 +230,7 @@ public function ingestCloudDataGeneric()
         if( $this->isRealmEnabled('Cloud') ){
             try{
                 $this->logger->notice('Ingesting generic cloud log files');
-                Utilities::runEtlPipeline(array('jobs-cloud-extract-generic'), $this->logger);
+                Utilities::runEtlPipeline(array('jobs-cloud-import-users-generic', 'jobs-cloud-extract-generic'), $this->logger);
             }
             catch( Exception $e ){
                 if( $e->getCode() == 1146 ){

classes/OpenXdmod/Migration/Version800To810/ConfigFilesMigration.php

@@ -0,0 +1,70 @@
+<?php
+/**
+ * @author Greg Dean <[email protected]>
+ */
+
+namespace OpenXdmod\Migration\Version800To810;
+
+use CCR\Json;
+use OpenXdmod\Migration\ConfigFilesMigration as AbstractConfigFilesMigration;
+use Exception;
+
+/**
+ * Update config files from version 8.0.0 To 8.0.1.
+ */
+class ConfigFilesMigration extends AbstractConfigFilesMigration
+{
+
+    private $cloudRolesFilePath;
+
+    public function __construct($currentVersion, $newVersion)
+    {
+         $this->cloudRolesFilePath = CONFIG_DIR."/roles.d/cloud.json";
+         parent::__construct($currentVersion, $newVersion);
+    }
+
+    /**
+     * Execute the migration.
+     */
+    public function execute()
+    {
+        if (file_exists($this->cloudRolesFilePath)) {
+            $this->addCloudRolesGroupBy();
+        }
+    }
+
+    /**
+     * Adds new group bys to roles.d/cloud.json
+     */
+    public function addCloudRolesGroupBy()
+    {
+        // Json::loadFile throws an exception if the file is completely empty or if there some other
+        // problem loading the file. If those exceptions are thrown catch them so the rest of the
+        // migration script can continue to run
+        try{
+            $cloudRolesFile = Json::loadFile($this->cloudRolesFilePath);
+        }
+        catch(Exception $e){
+            return false;
+        }
+
+        if (array_key_exists('+roles', $cloudRolesFile)) {
+            foreach($cloudRolesFile['+roles'] as $key => $value) {
+                $cloudRolesFile['+roles'][$key]['+query_descripters'][] = array('realm' => 'Cloud', 'group_by' => 'person');
+                $cloudRolesFile['+roles'][$key]['+query_descripters'][] = array('realm' => 'Cloud', 'group_by' => 'username');
+            }
+
+            // An exception can be thrown if there is a problem writing the file. Catch and log the issue
+            // while letting the rest of the migration script run
+            try{
+                JSON::saveFile($this->cloudRolesFilePath, $cloudRolesFile);
+            }
+            catch(Exception $e){
+                $this->logger->notice("Unable to write to roles.d/cloud.json config file. Continuing upgrade");
+                return false;
+            }
+        }
+
+        return true;
+    }
+}

configuration/datawarehouse.json

@@ -361,6 +361,14 @@
                 {
                    "name": "submission_venue",
                    "class": "GroupBySubmissionVenue"
+                },
+                {
+                   "name": "person",
+                   "class": "GroupByPerson"
+                },
+                {
+                   "name": "username",
+                   "class": "GroupByUsername"
                 }
             ],
             "statistics": [