VCP-19989: add S3 DF assume role support for customer or Kaltura ARN

infra/storage/RefreshableRole.class.php

@@ -20,9 +20,14 @@ class RefreshableRole
 	const ROLE_SESSION_NAME_PREFIX = "kaltura_s3_access_";
 	const ASSUME_ROLE_CREDENTIALS_EXPIRY_TIME = 43200;
 
-	public static function getCacheCredentialsProvider($roleArn, $s3Region = null)
+	public static function getCacheCredentialsProvider($roleArn, $s3Region = null, $dirnameSuffix = null)
 	{
 		$credentialsCacheDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 's3_creds_cache_v3';
+
+		if ($dirnameSuffix)
+		{
+			$credentialsCacheDir .= "_$dirnameSuffix";
+		}
 
 		$profile = new InstanceProfileProvider();
 		$cache = new DoctrineCacheAdapter(new FilesystemCache($credentialsCacheDir));
@@ -50,4 +55,4 @@ public static function getCacheCredentialsProvider($roleArn, $s3Region = null)
 
 		return CredentialProvider::cache($provider, $cache);
 	}
-}
+}

infra/storage/file_transfer_managers/s3Mgr.class.php

@@ -33,6 +33,7 @@ class s3Mgr extends kFileTransferMgr
 	protected $endPoint = null;
 	protected $storageClass = null;
 	protected $s3Arn = null;
+	protected $dirnameSuffix = null;
 
 	const SIZE = 'Size';
 	const LAST_MODIFICATION = 'LastModified';
@@ -79,14 +80,24 @@ protected function __construct(array $options = null)
 		{
 			$this->storageClass = $options['storageClass'];
 		}
-
-		if (class_exists('KBatchBase'))
+		
+		if($options && isset($options['s3Arn']))
 		{
-			$this->s3Arn = kBatchUtils::getKconfParam('arnRole', true);
+			$this->s3Arn = $options['s3Arn'];
+			$this->dirnameSuffix = 'dropFolderWatcherRemoteS3';
 		}
 		else
 		{
-			$this->s3Arn = kConf::get('s3Arn', 'cloud_storage', null);
+			if (class_exists('KBatchBase'))
+			{
+				$this->s3Arn = kBatchUtils::getKconfParam('arnRole', true);
+			}
+			else
+			{
+				$this->s3Arn = kConf::get('s3Arn', 'cloud_storage', null);
+			}
+
+			$this->dirnameSuffix = 's3Mgr';
 		}
 
 		// do nothing
@@ -109,7 +120,7 @@ protected function doConnect($sftp_server, &$sftp_port)
 	{
 		return 1;
 	}
-
+	
 	// login to an existing connection with given user/pass (ftp_passive_mode is irrelevant)
 	//
 	// S3 Signature is required to be V4 for SSE-KMS support. Newer S3 regions also require V4.
@@ -124,7 +135,8 @@ protected function doLogin($sftp_user, $sftp_pass)
 
 		if($this->s3Arn && (!isset($sftp_user) || !$sftp_user) && (!isset($sftp_pass) || !$sftp_pass))
 		{
-			KalturaLog::debug('Found env VAR from config- ' . $this->s3Arn);
+			KalturaLog::debug("Login using ARN [ {$this->s3Arn} ] and region [ {$this->s3Region} ]");
+
 			if(!class_exists('Aws\Sts\StsClient'))
 			{
 				KalturaLog::err('Class Aws\S3\StsClient was not found!!');
@@ -145,7 +157,7 @@ protected function doLogin($sftp_user, $sftp_pass)
 
 	private function generateCachedCredentialsS3Client()
 	{
-		$cacheProviderCredentials = RefreshableRole::getCacheCredentialsProvider($this->s3Arn);
+		$cacheProviderCredentials = RefreshableRole::getCacheCredentialsProvider($this->s3Arn, $this->s3Region, $this->dirnameSuffix);
 		$config = $this->getBaseClientConfig();
 		$config['credentials'] = $cacheProviderCredentials;
 		$this->s3 = S3Client::factory($config);

infra/storage/kFileTransferMgr.class.php

@@ -96,7 +96,7 @@ public function getResults()
 	 * @return the connection resource identifier
 	 */
 	abstract protected function doConnect($server, &$port);
-
+	
 	/**
 	 * Should login to a previous initiatied connection with the user / pass given.
 	 *
@@ -272,8 +272,7 @@ public function getConnection ()
 	{
 		return $this->connection_id;
 	}
-
-
+
 	/**
 	 * Connect & authenticate on the given server, using the given username & password.
 	 *

plugins/drop_folder/admin/forms/DropFolderConfigure.php

@@ -220,7 +220,13 @@ public function init()
 		    ));
 		    $extendTypeSubFormTitle = new Zend_Form_Element_Hidden(self::EXTENSION_SUBFORM_NAME.'_title');
     		$extendTypeSubFormTitle->setLabel($extendTypeSubForm->getTitle());
-    		$extendTypeSubFormTitle->setDecorators(array('ViewHelper', array('Label', array('placement' => 'append')), array('HtmlTag',  array('tag' => 'b'))));
+			$extendTypeSubFormTitle->setDescription($extendTypeSubForm->getDescription());
+    		$extendTypeSubFormTitle->setDecorators(array(
+				'ViewHelper',
+				array('Label',array('placement' => 'append')),
+				array('HtmlTag',  array('tag' => 'b')),
+				array('Description', array('escape' => false, 'tag' => 'p', 'class' => 'description')))
+			);
     		$this->addElement($extendTypeSubFormTitle);
     		$extendTypeSubForm->setDecorators(array(
     	        'FormElements',

plugins/drop_folder/admin/forms/DropFolderConfigureExtend.php

@@ -24,4 +24,9 @@ public function getTitle()
 	{
 	    return;
 	}
-}
+
+	public function getDescription()
+	{
+		return;
+	}
+}

plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php

@@ -280,7 +280,20 @@ public static function getFileTransferManager(KalturaDropFolder $dropFolder)
 		if($dropFolder instanceof KalturaS3DropFolder)
 		{
 			$engineOptions['s3Region'] = $dropFolder->s3Region;
+
+			if ($dropFolder->useS3Arn)
+			{
+				$engineOptions['s3Arn'] = $dropFolder->s3Arn;
+
+				if (empty($engineOptions['s3Arn']))
+				{
+					// throw Exception and not kFileTransferMgrException::otherError to catch msg at: plugins/drop_folder/batch/DropFolderWatcher/KAsyncDropFolderWatcher.class.php:72
+					$msg = "Drop Folder ID [{$dropFolder->id}] enabled 'Bucket Policy Allow Access' but 's3Arn' value under 's3_drop_folder' in 'runtime_config' map is missing";
+					throw new Exception($msg, kFileTransferMgrException::otherError);
+				}
+			}
 		}
+
 		$fileTransferMgr = kFileTransferMgr::getInstance(self::getFileTransferMgrType($dropFolder->type), $engineOptions);
 
 		$host =null; $username=null; $password=null; $port=null;

plugins/drop_folder/lib/api/types/KalturaDropFolder.php

@@ -291,6 +291,11 @@ public function toInsertableObject ( $object_to_fill = null , $props_to_skip = a
 		return $result;
 	}
 
+	public function validateForInsert($propertiesToSkip = array())
+	{
+		parent::validateForInsert($propertiesToSkip);
+	}
+
 	/**
 	 * @param int $type
 	 * @return KalturaDropFolder

plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php

@@ -9,6 +9,14 @@ public function getTitle()
 	{
 		return 'S3 settings';
 	}
+
+	public function getDescription()
+	{
+		return "Authentication precedence:<br>
+				1. User & Password (if passed will be used)<br>
+				2. Bucket Policy Allows Access<br>
+				Note: Bucket policy must allow 'runtime_config' map 's3_drop_folder' section 's3Arn' value role to operate it";
+	}
 
 	public function init()
 	{
@@ -31,6 +39,12 @@ public function init()
 			'label'			=> 'Password:',
 			'filters'		=> array('StringTrim'),
 		));
+
+		$this->addElement('checkbox', 'useS3Arn', array(
+			'label'      => 'Bucket Policy Allow Access',
+			'decorators' => array('ViewHelper', array('Label', array('placement' => 'append')), array('HtmlTag', array('tag' => 'div', 'class' => 'rememeber'))),
+			'uncheckedValue' => false,
+			'checkedValue'   => true,
+		));
 	}
-
 }

plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php

@@ -1,4 +1,3 @@
-
 <?php
 /**
  * @package plugins.S3DropFolder
@@ -25,6 +24,17 @@ class KalturaS3DropFolder extends KalturaDropFolder
 	 * @var string
 	 */
 	public $s3Password;
+
+	/**
+	 * @var bool
+	 */
+	public $useS3Arn;
+
+	/**
+	 * @var string
+	 * @readonly
+	 */
+	public $s3Arn;
 
 	/*
 	 * mapping between the field on this object (on the left) and the setter/getter on the entry object (on the right)
@@ -33,7 +43,9 @@ class KalturaS3DropFolder extends KalturaDropFolder
 		's3Host',
 		's3Region',
 		's3UserId',
-		's3Password'
+		's3Password',
+		'useS3Arn',
+		's3Arn'
 	);
 
 	public function getMapBetweenObjects()
@@ -51,4 +63,17 @@ public function toObject($dbObject = null, $skip = array())
 		$dbObject->setType(S3DropFolderPlugin::getDropFolderTypeCoreValue(S3DropFolderType::S3DROPFOLDER));
 		return parent::toObject($dbObject, $skip);
 	}
+
+	public function validateForInsert($propertiesToSkip = array())
+	{
+		if ($this->useS3Arn)
+		{
+			if (empty(kConf::getArrayValue('s3Arn', 's3_drop_folder', 'runtime_config', null)))
+			{
+				throw new KalturaAPIException(KalturaS3DropFolderErrors::MISSING_S3ARN_CONFIG);
+			}
+		}
+
+		parent::validateForInsert($propertiesToSkip);
+	}
 }

plugins/s3DropFolderPlugin/lib/api/errors/KalturaS3DropFolderErrors.php

@@ -0,0 +1,10 @@
+<?php
+
+/**
+ * @package plugins.vendor
+ * @subpackage api.errors
+ */
+class KalturaS3DropFolderErrors extends KalturaErrors
+{
+	const MISSING_S3ARN_CONFIG = "MISSING_S3_ARN;;'useS3Arn' ('Bucket Policy Allow Access') was set but 's3Arn' config is missing";
+}

plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php

@@ -12,6 +12,8 @@ class S3DropFolder extends RemoteDropFolder
 	const S3_USER_ID = 's3UserId';
 
 	const S3_PASSWORD = 's3Password';
+
+	const USE_S3_ARN = 'useS3Arn';
 
 	/**
 	 * @var string
@@ -32,6 +34,16 @@ class S3DropFolder extends RemoteDropFolder
 	 * @var string
 	 */
 	protected $s3Password;
+
+	/**
+	 * @var bool
+	 */
+	protected $useS3Arn;
+
+	/**
+	 * @var string
+	 */
+	protected $s3Arn;
 
 	/**
 	 * return string
@@ -52,6 +64,24 @@ public function getS3UserId (){ return $this->getFromCustomData(self::S3_USER_ID
 	 * return string
 	 */
 	public function getS3Password (){ return $this->getFromCustomData(self::S3_PASSWORD);}
+
+	/**
+	 * @return bool
+	 */
+	public function getUseS3Arn (){ return (bool) $this->getFromCustomData(self::USE_S3_ARN);}
+
+	/**
+	 * @return string
+	 */
+	public function getS3Arn ()
+	{
+		if (empty($this->getUseS3Arn()))
+		{
+			return null;
+		}
+
+		return kConf::getArrayValue('s3Arn', 's3_drop_folder', 'runtime_config', null);
+	}
 
 	/**
 	 * @param string $v
@@ -72,6 +102,11 @@ public function setS3UserId ($v){ $this->putInCustomData(self::S3_USER_ID, $v);}
 	 * @param string $v
 	 */
 	public function setS3Password ($v){ $this->putInCustomData(self::S3_PASSWORD, $v);}
+
+	/**
+	 * @param bool $v
+	 */
+	public function setUseS3Arn ($v){ $this->putInCustomData(self::USE_S3_ARN, (bool) $v);}
 
 
 	public function getImportJobData()
@@ -100,6 +135,9 @@ public function getDropFolderParams()
 			's3Host' => $this->getS3Host(),
 			's3UserId' => $this->getS3UserId(),
 			's3Password' => $this->getS3Password(),
-			's3Region' => $this->getS3Region());
+			's3Region' => $this->getS3Region(),
+			'useS3Arn' => $this->getUseS3Arn(),
+			's3Arn' => $this->getS3Arn()
+		);
 	}
 }

plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php

@@ -21,6 +21,14 @@ public function getFileUrl()
 	{
 		$dbDropFolder = DropFolderPeer::retrieveByPK($this->getDropFolderId());
 		$s3Options = $dbDropFolder->getDropFolderParams();
+
+		if ($s3Options['useS3Arn'] && empty($s3Options['s3Arn']))
+		{
+			$msg = "Drop Folder ID [{$dbDropFolder->getId()}] enabled 'Bucket Policy Allow Access' but 's3Arn' value under 's3_drop_folder' in 'runtime_config' map is missing";
+			throw new kFileTransferMgrException($msg, kFileTransferMgrException::otherError);
+		}
+
+
 		$s3TransferMgr = kFileTransferMgr::getInstance(kFileTransferMgrType::S3, $s3Options);
 		$s3TransferMgr->login($dbDropFolder->getS3Host(), $dbDropFolder->getS3UserId(), $dbDropFolder->getS3Password());
 		$expiry = time() + 5 * 86400;