Merge branch 'xdmod8.1' into sign-in-updates

ubccr · Nov 9, 2018 · a0af91d · a0af91d
2 parents 9512b46 + 4552ced
commit a0af91d
Show file tree

Hide file tree

Showing 605 changed files with 8,997 additions and 2,676 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,7 +1,41 @@
 Open XDMoD Change Log
 =====================
-## 2018-??-?? v8.0.0
+## 2018-10-30 v8.0.0
 
+- Features
+    - General
+        - Added a **beta** version of the Cloud realm to provide metrics relevant to cloud computing resources.
+        - Added a **beta** version of the Storage realm to provide metrics relevant to storage systems installed at a center.
+        - Federated XDMoD has been released for production. Federated XDMoD allows individual, locally managed, XDMoD instances to report all or a subset of their accounting data to a central Hub which provides a global view of the federation.
+        - All XDMoD user profiles are now associated with an organization. Previously, this was only required for Campus Champions.
+        - Added support for automatically detecting / assigning a new SSO User's organization.
+        - Added support for automatically detecting if a user's organization has changed and updating their accounts accordingly. This may include, but is not limited to, the removal of elevated privileges.
+        - Hardened the login and password reset process as a result of a security audit by University of Cambridge.
+        - Improved support for resource manager job arrays.
+        - Many improvements to the documentation.
+    - ETL
+        - Reorganized several ETL pipelines.
+        - Improved data sanitization for tighter checks present in MySQL 5.7.
+        - Refactored Jobs realm ingestion to utilize ETLv2.
+        - Standardize action names to follow the format module.pipeline.action. For example, xdmod.acls.manage-tables.
+        - Added character set and coalition to table definitions.
+        - Added support for foreign key constraints.
+        - Added support for the definition of ETL variables on the command line using -d variable=value.
+        - Add ingestion of node hostname data from SGE logs.
+        - Various ETL performance improvements.
+- Bug Fixes
+    - User Interface
+        - Deep linking when logged in using SSO has been restored.
+        - Update the logrotate configuration to use the su and create options.
+    - ETL
+        - Add primary keys to select ETL source queries.
+        - When modifying an existing table, preserve the order of the columns in the definition file.
+        - Ensure that file handles are flushed before inserting the final chunk of data.
+    - Misc
+        - Fixed several exceptions that were outside of a namespace.
+        - Fixed an issue where ACLs were not properly created on upgrade.
+        - Several minor bugfixes
+
 ## 2018-05-23 v7.5.1
 
 - Bug Fixes

diff --git a/Dockerfile b/Dockerfile
@@ -1 +1 @@
-FROM       tas-tools-ext-01.ccr.xdmod.org/xdmod-centos7:open7.5.1-v5
+FROM       tas-tools-ext-01.ccr.xdmod.org/xdmod-centos7:open8.0.0-v1
diff --git a/bin/xdmod-upgrade b/bin/xdmod-upgrade
@@ -26,8 +26,7 @@ ini_set('memory_limit', -1);
  * @var array
  */
 $supportedUpgrades = array(
-    '7.5.0' => '7.5.1',
-    '7.5.1' => '8.0.0',
+    '8.0.0' => '8.1.0',
 );
 
 /**

diff --git a/classes/Authentication/SAML/XDSamlAuthentication.php b/classes/Authentication/SAML/XDSamlAuthentication.php
@@ -31,21 +31,12 @@ class XDSamlAuthentication
      */
     protected $_isConfigured = false;
 
-    /**
-     * Whether or not we allow Single Sign On users local access. Defaults to true.
-     *
-     * @var boolean
-     */
-    protected $_allowLocalAccessViaSSO = true;
-
     const BASE_ADMIN_EMAIL = <<<EML
 
 Person Details -----------------------------------
 Name:              %s
 Username:          %s
 E-Mail:            %s
-Organization ID:   %s
-Organization Name: %s
 
 SAML Attributes ----------------------------------
 %s
@@ -72,10 +63,6 @@ public function __construct()
         );
 
         $this->_sources = \SimpleSAML_Auth_Source::getSources();
-        try {
-            $this->_allowLocalAccessViaSSO = strtolower(\xd_utilities\getConfiguration('authentication', 'allowLocalAccessViaFederation')) === "false" ? false : true;
-        } catch (Exception $e) {
-        }
         if ($this->isSamlConfigured()) {
             try {
                 $authSource = \xd_utilities\getConfiguration('authentication', 'source');
@@ -110,124 +97,87 @@ public function isSamlConfigured()
     public function getXdmodAccount()
     {
         $samlAttrs = $this->_as->getAttributes();
-        if (!isset($samlAttrs["username"])) {
-            $thisUserName = null;
-        } else {
-            $thisUserName = !empty($samlAttrs['username'][0]) ? $samlAttrs['username'][0] : null;
-        }
-        if (!isset($samlAttrs["system_username"])) {
-            $thisSystemUserName = $thisUserName;
-        } else {
-            $thisSystemUserName = !empty($samlAttrs['system_username'][0]) ? $samlAttrs['system_username'][0] : null;
-        }
-        if ($this->_as->isAuthenticated() && !empty($thisUserName)) {
-            $xdmodUserId = \XDUser::userExistsWithUsername($thisUserName);
+
+        if ($this->_as->isAuthenticated()) {
+            $userName = $samlAttrs['username'][0];
+
+            $xdmodUserId = \XDUser::userExistsWithUsername($userName);
+
             if ($xdmodUserId !== INVALID) {
-                return \XDUser::getUserByID($xdmodUserId);
-            } elseif ($this->_allowLocalAccessViaSSO && isset($samlAttrs['email_address'])) {
-                $xdmodUserId = \XDUser::userExistsWithEmailAddress($samlAttrs['email_address'][0]);
-                if ($xdmodUserId === AMBIGUOUS) {
-                    return "AMBIGUOUS";
-                }
-                if ($xdmodUserId !== INVALID) {
-                    return \XDUser::getUserByID($xdmodUserId);
-                }
+                $user = \XDUser::getUserByID($xdmodUserId);
+                $user->setSSOAttrs($samlAttrs);
+                return $user;
             }
-            $emailAddress = !empty($samlAttrs['email_address'][0]) ? $samlAttrs['email_address'][0] : NO_EMAIL_ADDRESS_SET;
-            $personId = \DataWarehouse::getPersonIdByUsername($thisSystemUserName);
 
+            // If we've gotten this far then we're creating a new user. Proceed with gathering the
+            // information we'll need to do so.
+            $emailAddress = isset($samlAttrs['email_address']) ? $samlAttrs['email_address'][0] : NO_EMAIL_ADDRESS_SET;
+            $systemUserName = isset($samlAttrs['system_username']) ? $samlAttrs['system_username'][0] : $userName;
+            $firstName = isset($samlAttrs['first_name']) ? $samlAttrs['first_name'][0] : 'UNKNOWN';
+            $middleName = isset($samlAttrs['middle_name']) ? $samlAttrs['middle_name'][0] : null;
+            $lastName = isset($samlAttrs['last_name']) ? $samlAttrs['last_name'][0] : null;
+            $personId = \DataWarehouse::getPersonIdFromPII($systemUserName, $samlAttrs['organization'][0]);
+
+            // Attempt to identify which organization this user should be associated with. Prefer
+            // using the personId if not unknown, then fall back to the saml attributes if the
+            // 'organization' property is present, and finally defaulting to the Unknown organization
+            // if none of the preceding conditions are met.
             $userOrganization = $this->getOrganizationId($samlAttrs, $personId);
 
-            if (!isset($samlAttrs["first_name"])) {
-                $samlAttrs["first_name"] = array("UNKNOWN");
-            }
-            if (!isset($samlAttrs["middle_name"])) {
-                $samlAttrs["middle_name"] = array(null);
-            }
-            if (!isset($samlAttrs["last_name"])) {
-                $samlAttrs["last_name"] = array("UNKNOWN");
-            }
             try {
                 $newUser = new \XDUser(
-                    $thisUserName,
+                    $userName,
                     null,
                     $emailAddress,
-                    $samlAttrs["first_name"][0],
-                    $samlAttrs["middle_name"][0],
-                    $samlAttrs["last_name"][0],
+                    $firstName,
+                    $middleName,
+                    $lastName,
                     array(ROLE_ID_USER),
                     ROLE_ID_USER,
                     $userOrganization,
-                    $personId
+                    $personId,
+                    $samlAttrs
                 );
             } catch (Exception $e) {
-                return "EXISTS";
+                throw new Exception('An account is currently configured with this information, please contact an administrator.');
             }
+
             $newUser->setUserType(SSO_USER_TYPE);
+
             try {
                 $newUser->saveUser();
             } catch (Exception $e) {
                 $this->logger->err('User creation failed: ' . $e->getMessage());
-                return false;
+                throw $e;
             }
 
             $this->handleNotifications($newUser, $samlAttrs, ($personId != UNKNOWN_USER_TYPE));
 
             return $newUser;
         }
-        return false;
+        throw new \DataWarehouse\Query\Exceptions\AccessDeniedException('Authentication failure.');
     }
 
     /**
-     * Retrieves the organization_id that this User should be associated with. There is one
-     * configuration property that affects the return value of this function,
-     * `force_default_organization`. It does so in the following ways:
+     * Retrieves the organization_id that this User should be associated with.
      *
-     * - If the `force_default_organization` property in `portal_settings.ini` === 'on'
-     *   - Then the users organization is determined by the run time constant
-     *     `ORGANIZATION_NAME` ( which is derived from the `organization.json` configuration
-     *     file. ) This value will be used to find a corresponding record in the
-     *     `modw.organization` table via the `name` column.
+     * - If we were able to identify which `person` this user should be associated with
+     *   - then look up which organization they are associated via the `modw.person.id` column.
      * - If SAML has been provided with an `organization` property.
      *   - Then the users organization will be determined by attempting to find a record in
      *     the `modw.organization` table that has a `name` column that matches the provided
-     *     value.
-     *   - If unable to identify an organization in the previous step and a personId has been
-     *     supplied, attempt to retrieve the organization_id for this person via the
-     *     `modw.person.id` column.
-     * - If we were able to identify which `person` this user should be associated with
-     *   - then look up which organization they are associated via the `modw.person.id` column.
+     *     value. If no records are found then the 'Unknown' organization ( -1 ) is returned.
      * - and finally, if none of the other conditions are satisfied, return the Unknown organization
      *   ( i.e. -1 )
      *
-     * The default setting for an OpenXDMoD installation is:
-     *   - `force_default_organization="on"`
      * @param array $samlAttrs the saml attributes returned for this user
      * @param int   $personId  the id property for the Person this user has been associated with.
      * @return int the id of the organization that a new SSO user should be associated with.
      * @throws Exception if there is a problem retrieving an organization_id
      */
     public function getOrganizationId($samlAttrs, $personId)
     {
-        $techSupportRecipient = \xd_utilities\getConfiguration('general', 'tech_support_recipient');
-
-        $forceDefaultOrganization = null;
-        try {
-            $forceDefaultOrganization = \xd_utilities\getConfiguration('sso', 'force_default_organization') === 'on';
-        } catch (Exception $e) {
-            $this->notify(
-                $techSupportRecipient,
-                $techSupportRecipient,
-                '',
-                $techSupportRecipient,
-                'Error retrieving XDMoD configuration property "force_default_organization" form portal_settings.ini',
-                $e->getMessage()
-            );
-        }
-
-        if ($forceDefaultOrganization) {
-            return Organizations::getIdByName(ORGANIZATION_NAME);
-        } elseif ($personId !== -1 ) {
+        if ($personId !== -1 ) {
             return Organizations::getOrganizationIdForPerson($personId);
         } elseif(!empty($samlAttrs['organization'])) {
             return Organizations::getIdByName($samlAttrs['organization'][0]);
@@ -292,71 +242,33 @@ public function getLoginLink()
      * @param array $samlAttributes the attributes that we received via SAML for this user.
      * @param boolean $linked whether or not we were able to link the SSO User with an XDMoD Person.
      * @throws Exception if there is a problem with notifying the user.
-     * @throws Exception if there is a problem retrieving the name for the users organization.
      */
     private function handleNotifications(XDUser $user, $samlAttributes, $linked)
     {
-        // We only notify admins iff we were unable to identify which organization the user should
-        // be associated with ( i.e. we had to assign them to the 'Unknown' organization ).
-        if ($user->getOrganizationID() === -1) {
-            $techSupportRecipient = \xd_utilities\getConfiguration('general', 'tech_support_recipient');
-            $senderEmail = \xd_utilities\getConfiguration('mailer', 'sender_email');
-
-            $userEmail = $user->getEmailAddress()
-                ? $user->getEmailAddress()
-                : $samlAttributes['email_address'][0];
-
-            $userOrganization = $user->getOrganizationID();
-
-            $emailBody = sprintf(
-                self::BASE_ADMIN_EMAIL,
-                $user->getFormalName(true),
-                $user->getUsername(),
-                $userEmail,
-                $userOrganization,
-                Organizations::getNameById($userOrganization),
-                json_encode($samlAttributes, JSON_PRETTY_PRINT)
-            );
+        if ($user->getOrganizationID() !== -1) {
+            // Only send email if we were unable to identify an organization for the user.
+            return;
+        }
 
-            $subject = sprintf(
-                'New %s Single Sign On User Created',
-                ($linked ? 'Linked' : 'Unlinked')
-            );
+        $subject = sprintf(
+            'New %s Single Sign On User Created',
+            ($linked ? 'Linked' : 'Unlinked')
+        );
 
-            $this->notify(
-                $techSupportRecipient,
-                $techSupportRecipient,
-                '',
-                $senderEmail,
-                $subject,
-                $emailBody
-            );
-        }
-    }
+        $body = sprintf(
+            self::BASE_ADMIN_EMAIL,
+            $user->getFormalName(true),
+            $user->getUsername(),
+            $user->getEmailAddress(),
+            json_encode($samlAttributes, JSON_PRETTY_PRINT)
+        );
 
-    /**
-     * Attempt to generate / send a notification email with the specified parameters.
-     *
-     * @param string $toAddress    the address that the notification will be sent to.
-     * @param string $fromAddress  the address that the notification will be sent from.
-     * @param string $fromName     A name to be used when identifying the `$fromAddress`
-     * @param string $replyAddress the address to be used when a user replies to the notification.
-     * @param string $subject      the subject of the notification.
-     * @param string $body         the body of the notification.
-     *
-     * @throws Exception if there is a problem sending the notification email.
-     */
-    public function notify($toAddress, $fromAddress, $fromName, $replyAddress, $subject, $body)
-    {
         try {
             MailWrapper::sendMail(
                 array(
                     'subject' => $subject,
                     'body' => $body,
-                    'toAddress' => $toAddress,
-                    'fromAddress' => $fromAddress,
-                    'fromName' => $fromName,
-                    'replyAddress' => $replyAddress
+                    'toAddress' => \xd_utilities\getConfiguration('general', 'tech_support_recipient')
                 )
             );
         } catch (Exception $e) {

diff --git a/classes/CCR/DB/MySQLHelper.php b/classes/CCR/DB/MySQLHelper.php
@@ -416,9 +416,15 @@ public static function grantAllPrivileges(
         $dbUsername,
         $dbPassword
     ) {
+        /**
+         *  The privileges are listed out instead of ALL so that the user
+         *  created cannot administrate users or database tasks.
+         */
         $stmt = "GRANT TRIGGER, DROP, INDEX, CREATE, INSERT,"
             . " SELECT, DELETE, UPDATE, CREATE VIEW, SHOW VIEW,"
             . " ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,"
+            . " CREATE ROUTINE, ALTER ROUTINE, EVENT, RELOAD, FILE,"
+            . " CREATE TABLESPACE, PROCESS, REFERENCES,"
             . " LOCK TABLES"
             . " ON *.* TO '$dbUsername'@'$localHost'"
             . " IDENTIFIED BY '$dbPassword';FLUSH PRIVILEGES;";
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		FROM tas-tools-ext-01.ccr.xdmod.org/xdmod-centos7:open7.5.1-v5
		FROM tas-tools-ext-01.ccr.xdmod.org/xdmod-centos7:open8.0.0-v1