From 6e9424ecd5edce1af516d6e1c8f0b235195ddb1c Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Sun, 2 Feb 2025 13:47:24 +0200 Subject: [PATCH 01/27] VCP-19989: add S3 DF assume role support for customer or Kaltura ARN --- infra/general/kString.class.php | 5 ++ infra/storage/RefreshableRole.class.php | 9 +++- .../file_transfer_managers/s3Mgr.class.php | 50 +++++++++++++++++-- infra/storage/kFileTransferMgr.class.php | 27 ++++++++++ .../Engine/KDropFolderFileTransferEngine.php | 9 ++++ .../forms/S3DropFolderConfigureExtend.php | 6 +++ .../lib/api/KalturaS3DropFolder.php | 9 +++- .../lib/model/S3DropFolder.php | 21 +++++++- .../lib/model/S3DropFolderFile.php | 15 +++++- 9 files changed, 142 insertions(+), 9 deletions(-) diff --git a/infra/general/kString.class.php b/infra/general/kString.class.php index a6692772b6e..83ce9462c4b 100644 --- a/infra/general/kString.class.php +++ b/infra/general/kString.class.php @@ -597,4 +597,9 @@ public static function kStripos(string $haystack, string $needle, int $offset = return stripos($haystack, $needle, $offset); } + + public static function isValidAwsArn($arn) + { + return preg_match('#^arn:aws[a-zA-Z-]*:[a-z0-9-]+:[a-z0-9-]*:\d{12}:[a-zA-Z0-9-_:/]+$#', $arn) === 1; + } } diff --git a/infra/storage/RefreshableRole.class.php b/infra/storage/RefreshableRole.class.php index 9d9769a485c..d313bdba27a 100644 --- a/infra/storage/RefreshableRole.class.php +++ b/infra/storage/RefreshableRole.class.php @@ -20,9 +20,14 @@ class RefreshableRole const ROLE_SESSION_NAME_PREFIX = "kaltura_s3_access_"; const ASSUME_ROLE_CREDENTIALS_EXPIRY_TIME = 43200; - public static function getCacheCredentialsProvider($roleArn, $s3Region = null) + public static function getCacheCredentialsProvider($roleArn, $s3Region = null, $dirnameSuffix = null) { $credentialsCacheDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 's3_creds_cache_v3'; + + if ($dirnameSuffix) + { + $credentialsCacheDir .= $dirnameSuffix; + } $profile = new InstanceProfileProvider(); $cache = new DoctrineCacheAdapter(new FilesystemCache($credentialsCacheDir)); @@ -50,4 +55,4 @@ public static function getCacheCredentialsProvider($roleArn, $s3Region = null) return CredentialProvider::cache($provider, $cache); } -} \ No newline at end of file +} diff --git a/infra/storage/file_transfer_managers/s3Mgr.class.php b/infra/storage/file_transfer_managers/s3Mgr.class.php index 05cc73c1a82..25d085a6c9c 100644 --- a/infra/storage/file_transfer_managers/s3Mgr.class.php +++ b/infra/storage/file_transfer_managers/s3Mgr.class.php @@ -83,10 +83,12 @@ protected function __construct(array $options = null) if (class_exists('KBatchBase')) { $this->s3Arn = kBatchUtils::getKconfParam('arnRole', true); + $this->s3Region = kBatchUtils::getKconfParam('s3Region', true); } else { $this->s3Arn = kConf::get('s3Arn', 'cloud_storage', null); + $this->s3Region = kConf::getArrayValue('s3Region', 'storage_options', 'cloud_storage', null); } // do nothing @@ -109,6 +111,46 @@ protected function doConnect($sftp_server, &$sftp_port) { return 1; } + + protected function doLoginIAMRole($s3IAMRole = null, $s3Region = null, $dirnameSuffix = null) + { + if(!class_exists('Aws\S3\S3Client')) + { + KalturaLog::err('Class Aws\S3\S3Client was not found!!'); + return false; + } + + // check if the given ARN is valid + KalturaLog::debug("Checking if [ $s3IAMRole ] is a valid IAM Role"); + $isValidIAMRole = kString::isValidAwsArn($s3IAMRole); + + // if IAM Role or region were not provided or IAM role invalid - try default s3Arn and s3Region from cloud_config + // this is to support a case were customer give our cloud_storage config 's3Arn' access to his bucket so no need to assume customer arn role + if (!$isValidIAMRole) + { + KalturaLog::debug("IAM Role [ $s3IAMRole ] is not valid - attempting access to S3 bucket using Kaltura s3Arn"); + return $this->generateCachedCredentialsS3Client(); + } + + // use the given ARN and Region + if ($s3Region) + { + $this->s3Arn = $s3IAMRole; + $this->s3Region = $s3Region; + KalturaLog::debug("Login using ARN [ $s3IAMRole ] and region [ $s3Region ]"); + + if(!class_exists('Aws\Sts\StsClient')) + { + KalturaLog::err('Class Aws\S3\StsClient was not found!!'); + return false; + } + + return $this->generateCachedCredentialsS3Client($dirnameSuffix); + } + + KalturaLog::err("Missing 's3Region' in Drop Folder config - cannot assume customer ARN [ $s3IAMRole ]"); + return false; + } // login to an existing connection with given user/pass (ftp_passive_mode is irrelevant) // @@ -124,7 +166,9 @@ protected function doLogin($sftp_user, $sftp_pass) if($this->s3Arn && (!isset($sftp_user) || !$sftp_user) && (!isset($sftp_pass) || !$sftp_pass)) { - KalturaLog::debug('Found env VAR from config- ' . $this->s3Arn); + $s3Arn = $this->s3Arn; + KalturaLog::debug("Login using ARN [ $s3Arn ]"); + if(!class_exists('Aws\Sts\StsClient')) { KalturaLog::err('Class Aws\S3\StsClient was not found!!'); @@ -143,9 +187,9 @@ protected function doLogin($sftp_user, $sftp_pass) return $this->generateStaticCredsClient($sftp_user, $sftp_pass); } - private function generateCachedCredentialsS3Client() + private function generateCachedCredentialsS3Client($dirnameSuffix = null) { - $cacheProviderCredentials = RefreshableRole::getCacheCredentialsProvider($this->s3Arn); + $cacheProviderCredentials = RefreshableRole::getCacheCredentialsProvider($this->s3Arn, $this->s3Region, $dirnameSuffix); $config = $this->getBaseClientConfig(); $config['credentials'] = $cacheProviderCredentials; $this->s3 = S3Client::factory($config); diff --git a/infra/storage/kFileTransferMgr.class.php b/infra/storage/kFileTransferMgr.class.php index ca498742a2c..ecff4c1f08e 100644 --- a/infra/storage/kFileTransferMgr.class.php +++ b/infra/storage/kFileTransferMgr.class.php @@ -37,6 +37,7 @@ class kFileTransferMgrException extends Exception const remoteFileExists = 6; // trying to putFile that already exists with $overwrite == false const extensionMissing = 7; // php extension is not installed const attributeMissing = 8; // option attribute is missing + const errorAssumeRole = 9; // error trying to assume role const otherError = 99; // other - exception's getMessage() will provide more details } @@ -96,6 +97,19 @@ public function getResults() * @return the connection resource identifier */ abstract protected function doConnect($server, &$port); + + /** + * @param $s3IAMRole string|null + * @param $s3Region string|null + * @param $dirnameSuffix string|null + * + * @throws kFileTransferMgrException + * + * @return bool + */ + + // TODO implement in all classes that extends kFileTransferMgr + abstract protected function doLoginIAMRole($s3IAMRole = null, $s3Region = null, $dirnameSuffix = null); /** * Should login to a previous initiatied connection with the user / pass given. @@ -272,6 +286,19 @@ public function getConnection () { return $this->connection_id; } + + public function loginIAMRole($s3IAMRole = null, $s3Region = null, $dirnameSuffix = null) + { + KalturaLog::debug("Attempting to assume role [ $s3IAMRole ] for S3 Region [ $s3Region ]"); + + if(!$this->doLoginIAMRole($s3IAMRole, $s3Region, $dirnameSuffix)) + { + $last_error = error_get_last(); + throw new kFileTransferMgrException ( "Error trying to assume role [ $s3IAMRole ] - " . $last_error['message'], kFileTransferMgrException::errorAssumeRole); + } + + KalturaLog::debug("Logged in using IAM Role successfully"); + } /** diff --git a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php index b0e3b136b23..f10d49b4e20 100644 --- a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php +++ b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php @@ -285,6 +285,8 @@ public static function getFileTransferManager(KalturaDropFolder $dropFolder) $host =null; $username=null; $password=null; $port=null; $privateKey = null; $publicKey = null; + $s3IAMRole = null; + $s3Region = null; if($dropFolder instanceof KalturaRemoteDropFolder) { @@ -304,6 +306,8 @@ public static function getFileTransferManager(KalturaDropFolder $dropFolder) $host = $dropFolder->s3Host; $username = $dropFolder->s3UserId; $password = $dropFolder->s3Password; + $s3IAMRole = $dropFolder->s3IAMRole; + $s3Region = $dropFolder->s3Region; } // login to server @@ -313,6 +317,11 @@ public static function getFileTransferManager(KalturaDropFolder $dropFolder) $publicKeyFile = $publicKey ? kFile::createTempFile($publicKey, 'publicKey'): null; $fileTransferMgr->loginPubKey($host, $username, $publicKeyFile, $privateKeyFile, $passPhrase, $port); } + elseif ($s3IAMRole && $s3Region) + { + $dirnameSuffix = '_drop_folder_id_' . $dropFolder->id; + $fileTransferMgr->loginIAMRole($s3IAMRole, $s3Region, $dirnameSuffix); + } else { $fileTransferMgr->login($host, $username, $password, $port); diff --git a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php index 4912c31f9fa..e8a833ce7f4 100644 --- a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php +++ b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php @@ -31,6 +31,12 @@ public function init() 'label' => 'Password:', 'filters' => array('StringTrim'), )); + + $this->addElement('text', 's3IAMRole', array( + 'label' => 'IAM Role:', + 'filters' => array('StringTrim'), + 'description' => 'If set to "true" - we use Kaltura IAM role (bucket must allow Kaltura EC2 role to operate on it)', + )); } } diff --git a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php index 89c2eb93caa..0af5fa67f2e 100644 --- a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php +++ b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php @@ -1,4 +1,3 @@ - getFromCustomData(self::S3_USER_ID * return string */ public function getS3Password (){ return $this->getFromCustomData(self::S3_PASSWORD);} + + /** + * @return string|null + */ + public function getS3IAMRole(){ return $this->getFromCustomData(self::S3_IAM_ROLE);} /** * @param string $v @@ -72,6 +84,11 @@ public function setS3UserId ($v){ $this->putInCustomData(self::S3_USER_ID, $v);} * @param string $v */ public function setS3Password ($v){ $this->putInCustomData(self::S3_PASSWORD, $v);} + + /** + * @param string $v + */ + public function setS3IAMRole ($v){ $this->putInCustomData(self::S3_IAM_ROLE, $v);} public function getImportJobData() @@ -100,6 +117,8 @@ public function getDropFolderParams() 's3Host' => $this->getS3Host(), 's3UserId' => $this->getS3UserId(), 's3Password' => $this->getS3Password(), - 's3Region' => $this->getS3Region()); + 's3Region' => $this->getS3Region(), + 's3IAMRole' => $this->getS3IAMRole(), + ); } } diff --git a/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php b/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php index 45322efba29..46b8a3ce79b 100644 --- a/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php +++ b/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php @@ -19,10 +19,23 @@ public function getNameForParsing () public function getFileUrl() { + KalturaLog::debug("Generating drop folder file URL"); + $dbDropFolder = DropFolderPeer::retrieveByPK($this->getDropFolderId()); $s3Options = $dbDropFolder->getDropFolderParams(); $s3TransferMgr = kFileTransferMgr::getInstance(kFileTransferMgrType::S3, $s3Options); - $s3TransferMgr->login($dbDropFolder->getS3Host(), $dbDropFolder->getS3UserId(), $dbDropFolder->getS3Password()); + + // login + if ($dbDropFolder->getS3IAMRole()) + { + $dirnameSuffix = '_drop_folder_id_' . $dbDropFolder->getId(); + $s3TransferMgr->loginIAMRole($dbDropFolder->getS3IAMRole(), $dbDropFolder->getS3Region(), $dirnameSuffix); + } + else + { + $s3TransferMgr->login($dbDropFolder->getS3Host(), $dbDropFolder->getS3UserId(), $dbDropFolder->getS3Password()); + } + $expiry = time() + 5 * 86400; $fullName = $dbDropFolder->getPath() . '/' . $this->getFileName(); $fileUrl = $s3TransferMgr->getFileUrl($fullName, $expiry); From 67e8d96bf24ddca9904d058204295b53e1796ea8 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Sun, 2 Feb 2025 14:00:01 +0200 Subject: [PATCH 02/27] verify required classes exists + tweaks --- .../file_transfer_managers/s3Mgr.class.php | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/infra/storage/file_transfer_managers/s3Mgr.class.php b/infra/storage/file_transfer_managers/s3Mgr.class.php index 25d085a6c9c..2b72b9ffd70 100644 --- a/infra/storage/file_transfer_managers/s3Mgr.class.php +++ b/infra/storage/file_transfer_managers/s3Mgr.class.php @@ -114,9 +114,9 @@ protected function doConnect($sftp_server, &$sftp_port) protected function doLoginIAMRole($s3IAMRole = null, $s3Region = null, $dirnameSuffix = null) { - if(!class_exists('Aws\S3\S3Client')) + // verify required class exists + if (!$this->verifyClassExists('Aws\S3\S3Client') || !$this->verifyClassExists('Aws\Sts\StsClient')) { - KalturaLog::err('Class Aws\S3\S3Client was not found!!'); return false; } @@ -124,7 +124,7 @@ protected function doLoginIAMRole($s3IAMRole = null, $s3Region = null, $dirnameS KalturaLog::debug("Checking if [ $s3IAMRole ] is a valid IAM Role"); $isValidIAMRole = kString::isValidAwsArn($s3IAMRole); - // if IAM Role or region were not provided or IAM role invalid - try default s3Arn and s3Region from cloud_config + // if ARN or region were not provided or ARN invalid - try default s3Arn and s3Region from cloud_config // this is to support a case were customer give our cloud_storage config 's3Arn' access to his bucket so no need to assume customer arn role if (!$isValidIAMRole) { @@ -135,15 +135,9 @@ protected function doLoginIAMRole($s3IAMRole = null, $s3Region = null, $dirnameS // use the given ARN and Region if ($s3Region) { + KalturaLog::debug("Login using ARN [ $s3IAMRole ] and region [ $s3Region ]"); $this->s3Arn = $s3IAMRole; $this->s3Region = $s3Region; - KalturaLog::debug("Login using ARN [ $s3IAMRole ] and region [ $s3Region ]"); - - if(!class_exists('Aws\Sts\StsClient')) - { - KalturaLog::err('Class Aws\S3\StsClient was not found!!'); - return false; - } return $this->generateCachedCredentialsS3Client($dirnameSuffix); } @@ -605,4 +599,15 @@ public function getFileUrl($remote_file, $expires = null) return $this->getPreSignedUrl($remote_file, $expires); } + + private function verifyClassExists($className) + { + if (!class_exists($className)) + { + KalturaLog::err('Class Aws\S3\S3Client was not found!!'); + return false; + } + + return true; + } } From 43e0befed7ba0c1eeb9ac5f6d7d9ee8dbee9e3a9 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Sun, 2 Feb 2025 14:28:26 +0200 Subject: [PATCH 03/27] do not override $this->s3Region until needed --- .../storage/file_transfer_managers/s3Mgr.class.php | 14 ++++++++++++-- .../admin/forms/S3DropFolderConfigureExtend.php | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/infra/storage/file_transfer_managers/s3Mgr.class.php b/infra/storage/file_transfer_managers/s3Mgr.class.php index 2b72b9ffd70..a5cea0c30a6 100644 --- a/infra/storage/file_transfer_managers/s3Mgr.class.php +++ b/infra/storage/file_transfer_managers/s3Mgr.class.php @@ -83,12 +83,10 @@ protected function __construct(array $options = null) if (class_exists('KBatchBase')) { $this->s3Arn = kBatchUtils::getKconfParam('arnRole', true); - $this->s3Region = kBatchUtils::getKconfParam('s3Region', true); } else { $this->s3Arn = kConf::get('s3Arn', 'cloud_storage', null); - $this->s3Region = kConf::getArrayValue('s3Region', 'storage_options', 'cloud_storage', null); } // do nothing @@ -129,6 +127,8 @@ protected function doLoginIAMRole($s3IAMRole = null, $s3Region = null, $dirnameS if (!$isValidIAMRole) { KalturaLog::debug("IAM Role [ $s3IAMRole ] is not valid - attempting access to S3 bucket using Kaltura s3Arn"); + $this->s3Region = $this->getInternalS3Region(); + return $this->generateCachedCredentialsS3Client(); } @@ -610,4 +610,14 @@ private function verifyClassExists($className) return true; } + + private function getInternalS3Region() + { + if (class_exists('KBatchBase')) + { + return kBatchUtils::getKconfParam('s3Region', true); + } + + return kConf::getArrayValue('s3Region', 'storage_options', 'cloud_storage', null); + } } diff --git a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php index e8a833ce7f4..16eb1d81ed6 100644 --- a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php +++ b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php @@ -35,7 +35,7 @@ public function init() $this->addElement('text', 's3IAMRole', array( 'label' => 'IAM Role:', 'filters' => array('StringTrim'), - 'description' => 'If set to "true" - we use Kaltura IAM role (bucket must allow Kaltura EC2 role to operate on it)', + 'description' => 'If set to "true" - we use Kaltura ARN (bucket must allow Kaltura EC2 role to operate on it)', )); } From b5d24ebaaf193f48fda2fba6dd3f1a681a311f26 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Sun, 2 Feb 2025 14:57:14 +0200 Subject: [PATCH 04/27] simplify + fix static classname --- .../file_transfer_managers/s3Mgr.class.php | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/infra/storage/file_transfer_managers/s3Mgr.class.php b/infra/storage/file_transfer_managers/s3Mgr.class.php index a5cea0c30a6..1f0c25b6f98 100644 --- a/infra/storage/file_transfer_managers/s3Mgr.class.php +++ b/infra/storage/file_transfer_managers/s3Mgr.class.php @@ -132,18 +132,10 @@ protected function doLoginIAMRole($s3IAMRole = null, $s3Region = null, $dirnameS return $this->generateCachedCredentialsS3Client(); } - // use the given ARN and Region - if ($s3Region) - { - KalturaLog::debug("Login using ARN [ $s3IAMRole ] and region [ $s3Region ]"); - $this->s3Arn = $s3IAMRole; - $this->s3Region = $s3Region; - - return $this->generateCachedCredentialsS3Client($dirnameSuffix); - } + $this->s3Arn = $s3IAMRole; + KalturaLog::debug("Login using ARN [ $s3IAMRole ] and region [ $s3Region ]"); - KalturaLog::err("Missing 's3Region' in Drop Folder config - cannot assume customer ARN [ $s3IAMRole ]"); - return false; + return $this->generateCachedCredentialsS3Client($dirnameSuffix); } // login to an existing connection with given user/pass (ftp_passive_mode is irrelevant) @@ -604,7 +596,7 @@ private function verifyClassExists($className) { if (!class_exists($className)) { - KalturaLog::err('Class Aws\S3\S3Client was not found!!'); + KalturaLog::err("Class [ $className ] was not found!!"); return false; } From 1e1a451f4d2024e8c498c4b11479eba2c06a2b27 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Mon, 3 Feb 2025 10:06:19 +0200 Subject: [PATCH 05/27] if region provided, take it --- infra/storage/file_transfer_managers/s3Mgr.class.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/infra/storage/file_transfer_managers/s3Mgr.class.php b/infra/storage/file_transfer_managers/s3Mgr.class.php index 1f0c25b6f98..2d326ff6339 100644 --- a/infra/storage/file_transfer_managers/s3Mgr.class.php +++ b/infra/storage/file_transfer_managers/s3Mgr.class.php @@ -112,17 +112,17 @@ protected function doConnect($sftp_server, &$sftp_port) protected function doLoginIAMRole($s3IAMRole = null, $s3Region = null, $dirnameSuffix = null) { - // verify required class exists + // verify required classes exists if (!$this->verifyClassExists('Aws\S3\S3Client') || !$this->verifyClassExists('Aws\Sts\StsClient')) { return false; } // check if the given ARN is valid - KalturaLog::debug("Checking if [ $s3IAMRole ] is a valid IAM Role"); + KalturaLog::debug("Checking if [ $s3IAMRole ] is a valid ARN"); $isValidIAMRole = kString::isValidAwsArn($s3IAMRole); - // if ARN or region were not provided or ARN invalid - try default s3Arn and s3Region from cloud_config + // if ARN was not provided or invalid - try default s3Arn and s3Region from cloud_config // this is to support a case were customer give our cloud_storage config 's3Arn' access to his bucket so no need to assume customer arn role if (!$isValidIAMRole) { @@ -133,6 +133,8 @@ protected function doLoginIAMRole($s3IAMRole = null, $s3Region = null, $dirnameS } $this->s3Arn = $s3IAMRole; + $this->s3Region = $s3Region ?? $this->s3Region; + KalturaLog::debug("Login using ARN [ $s3IAMRole ] and region [ $s3Region ]"); return $this->generateCachedCredentialsS3Client($dirnameSuffix); From 5fead8271d2c815ddbdb7791950b804ea3985c13 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Mon, 3 Feb 2025 16:11:10 +0200 Subject: [PATCH 06/27] refactor flow --- infra/general/kString.class.php | 5 -- infra/storage/RefreshableRole.class.php | 2 +- .../file_transfer_managers/s3Mgr.class.php | 79 +++++-------------- infra/storage/kFileTransferMgr.class.php | 28 +------ .../Engine/KDropFolderFileTransferEngine.php | 10 +-- .../lib/model/S3DropFolderFile.php | 14 +--- 6 files changed, 25 insertions(+), 113 deletions(-) diff --git a/infra/general/kString.class.php b/infra/general/kString.class.php index 83ce9462c4b..a6692772b6e 100644 --- a/infra/general/kString.class.php +++ b/infra/general/kString.class.php @@ -597,9 +597,4 @@ public static function kStripos(string $haystack, string $needle, int $offset = return stripos($haystack, $needle, $offset); } - - public static function isValidAwsArn($arn) - { - return preg_match('#^arn:aws[a-zA-Z-]*:[a-z0-9-]+:[a-z0-9-]*:\d{12}:[a-zA-Z0-9-_:/]+$#', $arn) === 1; - } } diff --git a/infra/storage/RefreshableRole.class.php b/infra/storage/RefreshableRole.class.php index d313bdba27a..86b4e13b4cb 100644 --- a/infra/storage/RefreshableRole.class.php +++ b/infra/storage/RefreshableRole.class.php @@ -26,7 +26,7 @@ public static function getCacheCredentialsProvider($roleArn, $s3Region = null, $ if ($dirnameSuffix) { - $credentialsCacheDir .= $dirnameSuffix; + $credentialsCacheDir .= "_$dirnameSuffix"; } $profile = new InstanceProfileProvider(); diff --git a/infra/storage/file_transfer_managers/s3Mgr.class.php b/infra/storage/file_transfer_managers/s3Mgr.class.php index 2d326ff6339..bbd1f2bbea3 100644 --- a/infra/storage/file_transfer_managers/s3Mgr.class.php +++ b/infra/storage/file_transfer_managers/s3Mgr.class.php @@ -33,6 +33,7 @@ class s3Mgr extends kFileTransferMgr protected $endPoint = null; protected $storageClass = null; protected $s3Arn = null; + protected $hash = false; const SIZE = 'Size'; const LAST_MODIFICATION = 'LastModified'; @@ -79,14 +80,24 @@ protected function __construct(array $options = null) { $this->storageClass = $options['storageClass']; } - - if (class_exists('KBatchBase')) + + if($options && isset($options['s3Arn'])) { - $this->s3Arn = kBatchUtils::getKconfParam('arnRole', true); + $this->s3Arn = $options['s3Arn']; + $this->hash = true; } else { - $this->s3Arn = kConf::get('s3Arn', 'cloud_storage', null); + if (class_exists('KBatchBase')) + { + $this->s3Arn = kBatchUtils::getKconfParam('arnRole', true); + $this->s3Region = kBatchUtils::getKconfParam('s3Region', true); + } + else + { + $this->s3Arn = kConf::get('s3Arn', 'cloud_storage', null); + $this->s3Region = kConf::getArrayValue('s3Region', 'storage_options', 'cloud_storage', null); + } } // do nothing @@ -110,36 +121,6 @@ protected function doConnect($sftp_server, &$sftp_port) return 1; } - protected function doLoginIAMRole($s3IAMRole = null, $s3Region = null, $dirnameSuffix = null) - { - // verify required classes exists - if (!$this->verifyClassExists('Aws\S3\S3Client') || !$this->verifyClassExists('Aws\Sts\StsClient')) - { - return false; - } - - // check if the given ARN is valid - KalturaLog::debug("Checking if [ $s3IAMRole ] is a valid ARN"); - $isValidIAMRole = kString::isValidAwsArn($s3IAMRole); - - // if ARN was not provided or invalid - try default s3Arn and s3Region from cloud_config - // this is to support a case were customer give our cloud_storage config 's3Arn' access to his bucket so no need to assume customer arn role - if (!$isValidIAMRole) - { - KalturaLog::debug("IAM Role [ $s3IAMRole ] is not valid - attempting access to S3 bucket using Kaltura s3Arn"); - $this->s3Region = $this->getInternalS3Region(); - - return $this->generateCachedCredentialsS3Client(); - } - - $this->s3Arn = $s3IAMRole; - $this->s3Region = $s3Region ?? $this->s3Region; - - KalturaLog::debug("Login using ARN [ $s3IAMRole ] and region [ $s3Region ]"); - - return $this->generateCachedCredentialsS3Client($dirnameSuffix); - } - // login to an existing connection with given user/pass (ftp_passive_mode is irrelevant) // // S3 Signature is required to be V4 for SSE-KMS support. Newer S3 regions also require V4. @@ -152,10 +133,11 @@ protected function doLogin($sftp_user, $sftp_pass) return false; } - if($this->s3Arn && (!isset($sftp_user) || !$sftp_user) && (!isset($sftp_pass) || !$sftp_pass)) + if($this->s3Arn && $this->s3Region && (!isset($sftp_user) || !$sftp_user) && (!isset($sftp_pass) || !$sftp_pass)) { $s3Arn = $this->s3Arn; - KalturaLog::debug("Login using ARN [ $s3Arn ]"); + $s3Region = $this->s3Region; + KalturaLog::debug("Login using ARN [ $s3Arn ] and region [ $s3Region ]"); if(!class_exists('Aws\Sts\StsClient')) { @@ -175,8 +157,10 @@ protected function doLogin($sftp_user, $sftp_pass) return $this->generateStaticCredsClient($sftp_user, $sftp_pass); } - private function generateCachedCredentialsS3Client($dirnameSuffix = null) + private function generateCachedCredentialsS3Client() { + $dirnameSuffix = $this->hash ? hash('md5', $this->s3Arn) : null; + $cacheProviderCredentials = RefreshableRole::getCacheCredentialsProvider($this->s3Arn, $this->s3Region, $dirnameSuffix); $config = $this->getBaseClientConfig(); $config['credentials'] = $cacheProviderCredentials; @@ -593,25 +577,4 @@ public function getFileUrl($remote_file, $expires = null) return $this->getPreSignedUrl($remote_file, $expires); } - - private function verifyClassExists($className) - { - if (!class_exists($className)) - { - KalturaLog::err("Class [ $className ] was not found!!"); - return false; - } - - return true; - } - - private function getInternalS3Region() - { - if (class_exists('KBatchBase')) - { - return kBatchUtils::getKconfParam('s3Region', true); - } - - return kConf::getArrayValue('s3Region', 'storage_options', 'cloud_storage', null); - } } diff --git a/infra/storage/kFileTransferMgr.class.php b/infra/storage/kFileTransferMgr.class.php index ecff4c1f08e..65c0a1ab1bc 100644 --- a/infra/storage/kFileTransferMgr.class.php +++ b/infra/storage/kFileTransferMgr.class.php @@ -37,7 +37,6 @@ class kFileTransferMgrException extends Exception const remoteFileExists = 6; // trying to putFile that already exists with $overwrite == false const extensionMissing = 7; // php extension is not installed const attributeMissing = 8; // option attribute is missing - const errorAssumeRole = 9; // error trying to assume role const otherError = 99; // other - exception's getMessage() will provide more details } @@ -98,19 +97,6 @@ public function getResults() */ abstract protected function doConnect($server, &$port); - /** - * @param $s3IAMRole string|null - * @param $s3Region string|null - * @param $dirnameSuffix string|null - * - * @throws kFileTransferMgrException - * - * @return bool - */ - - // TODO implement in all classes that extends kFileTransferMgr - abstract protected function doLoginIAMRole($s3IAMRole = null, $s3Region = null, $dirnameSuffix = null); - /** * Should login to a previous initiatied connection with the user / pass given. * @@ -287,19 +273,6 @@ public function getConnection () return $this->connection_id; } - public function loginIAMRole($s3IAMRole = null, $s3Region = null, $dirnameSuffix = null) - { - KalturaLog::debug("Attempting to assume role [ $s3IAMRole ] for S3 Region [ $s3Region ]"); - - if(!$this->doLoginIAMRole($s3IAMRole, $s3Region, $dirnameSuffix)) - { - $last_error = error_get_last(); - throw new kFileTransferMgrException ( "Error trying to assume role [ $s3IAMRole ] - " . $last_error['message'], kFileTransferMgrException::errorAssumeRole); - } - - KalturaLog::debug("Logged in using IAM Role successfully"); - } - /** * Connect & authenticate on the given server, using the given username & password. @@ -464,6 +437,7 @@ public function putFile ($remote_file, $local_file, $overwrite = false, $overwri * Download a file from the server * * @param $remote_file Remote file name + * @param $remote_file Remote file name * @param $local_file Local file name * * @throws kFileTransferMgrException diff --git a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php index f10d49b4e20..5d7b678ef5a 100644 --- a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php +++ b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php @@ -280,13 +280,12 @@ public static function getFileTransferManager(KalturaDropFolder $dropFolder) if($dropFolder instanceof KalturaS3DropFolder) { $engineOptions['s3Region'] = $dropFolder->s3Region; + $engineOptions['s3Arn'] = !empty($dropFolder->s3IAMRole) ? $dropFolder->s3IAMRole : null; } $fileTransferMgr = kFileTransferMgr::getInstance(self::getFileTransferMgrType($dropFolder->type), $engineOptions); $host =null; $username=null; $password=null; $port=null; $privateKey = null; $publicKey = null; - $s3IAMRole = null; - $s3Region = null; if($dropFolder instanceof KalturaRemoteDropFolder) { @@ -306,8 +305,6 @@ public static function getFileTransferManager(KalturaDropFolder $dropFolder) $host = $dropFolder->s3Host; $username = $dropFolder->s3UserId; $password = $dropFolder->s3Password; - $s3IAMRole = $dropFolder->s3IAMRole; - $s3Region = $dropFolder->s3Region; } // login to server @@ -317,11 +314,6 @@ public static function getFileTransferManager(KalturaDropFolder $dropFolder) $publicKeyFile = $publicKey ? kFile::createTempFile($publicKey, 'publicKey'): null; $fileTransferMgr->loginPubKey($host, $username, $publicKeyFile, $privateKeyFile, $passPhrase, $port); } - elseif ($s3IAMRole && $s3Region) - { - $dirnameSuffix = '_drop_folder_id_' . $dropFolder->id; - $fileTransferMgr->loginIAMRole($s3IAMRole, $s3Region, $dirnameSuffix); - } else { $fileTransferMgr->login($host, $username, $password, $port); diff --git a/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php b/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php index 46b8a3ce79b..e22cf767ebe 100644 --- a/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php +++ b/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php @@ -20,22 +20,10 @@ public function getNameForParsing () public function getFileUrl() { KalturaLog::debug("Generating drop folder file URL"); - $dbDropFolder = DropFolderPeer::retrieveByPK($this->getDropFolderId()); $s3Options = $dbDropFolder->getDropFolderParams(); $s3TransferMgr = kFileTransferMgr::getInstance(kFileTransferMgrType::S3, $s3Options); - - // login - if ($dbDropFolder->getS3IAMRole()) - { - $dirnameSuffix = '_drop_folder_id_' . $dbDropFolder->getId(); - $s3TransferMgr->loginIAMRole($dbDropFolder->getS3IAMRole(), $dbDropFolder->getS3Region(), $dirnameSuffix); - } - else - { - $s3TransferMgr->login($dbDropFolder->getS3Host(), $dbDropFolder->getS3UserId(), $dbDropFolder->getS3Password()); - } - + $s3TransferMgr->login($dbDropFolder->getS3Host(), $dbDropFolder->getS3UserId(), $dbDropFolder->getS3Password()); $expiry = time() + 5 * 86400; $fullName = $dbDropFolder->getPath() . '/' . $this->getFileName(); $fileUrl = $s3TransferMgr->getFileUrl($fullName, $expiry); From c3aa8298013845d2623b936ad804574677374e68 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Mon, 3 Feb 2025 16:13:36 +0200 Subject: [PATCH 07/27] remove --- infra/storage/kFileTransferMgr.class.php | 1 - 1 file changed, 1 deletion(-) diff --git a/infra/storage/kFileTransferMgr.class.php b/infra/storage/kFileTransferMgr.class.php index 65c0a1ab1bc..08a8c8afdf0 100644 --- a/infra/storage/kFileTransferMgr.class.php +++ b/infra/storage/kFileTransferMgr.class.php @@ -437,7 +437,6 @@ public function putFile ($remote_file, $local_file, $overwrite = false, $overwri * Download a file from the server * * @param $remote_file Remote file name - * @param $remote_file Remote file name * @param $local_file Local file name * * @throws kFileTransferMgrException From c124cbd5e59cdd88be859447590dbc2bab6df622 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Mon, 3 Feb 2025 16:15:27 +0200 Subject: [PATCH 08/27] remove --- infra/storage/kFileTransferMgr.class.php | 1 - 1 file changed, 1 deletion(-) diff --git a/infra/storage/kFileTransferMgr.class.php b/infra/storage/kFileTransferMgr.class.php index 08a8c8afdf0..63353354377 100644 --- a/infra/storage/kFileTransferMgr.class.php +++ b/infra/storage/kFileTransferMgr.class.php @@ -273,7 +273,6 @@ public function getConnection () return $this->connection_id; } - /** * Connect & authenticate on the given server, using the given username & password. * From c98f2554b0a8466ec07fbb7666e3a3506bd0a12e Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Mon, 3 Feb 2025 16:16:09 +0200 Subject: [PATCH 09/27] remove description for s3 df configure --- .../admin/forms/S3DropFolderConfigureExtend.php | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php index 16eb1d81ed6..c2435811e76 100644 --- a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php +++ b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php @@ -35,7 +35,6 @@ public function init() $this->addElement('text', 's3IAMRole', array( 'label' => 'IAM Role:', 'filters' => array('StringTrim'), - 'description' => 'If set to "true" - we use Kaltura ARN (bucket must allow Kaltura EC2 role to operate on it)', )); } From 2934735f8165ab5011738f59f0ecca2f20107b5c Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Tue, 4 Feb 2025 07:39:57 +0200 Subject: [PATCH 10/27] keep s3 DF s3Region even if using cloud_storage s3Arn --- infra/storage/file_transfer_managers/s3Mgr.class.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/infra/storage/file_transfer_managers/s3Mgr.class.php b/infra/storage/file_transfer_managers/s3Mgr.class.php index bbd1f2bbea3..6a275a7137e 100644 --- a/infra/storage/file_transfer_managers/s3Mgr.class.php +++ b/infra/storage/file_transfer_managers/s3Mgr.class.php @@ -91,12 +91,10 @@ protected function __construct(array $options = null) if (class_exists('KBatchBase')) { $this->s3Arn = kBatchUtils::getKconfParam('arnRole', true); - $this->s3Region = kBatchUtils::getKconfParam('s3Region', true); } else { $this->s3Arn = kConf::get('s3Arn', 'cloud_storage', null); - $this->s3Region = kConf::getArrayValue('s3Region', 'storage_options', 'cloud_storage', null); } } From 9d6a6ad89aa69ef3952f7cd91641cd74173eff6d Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Tue, 4 Feb 2025 12:26:17 +0200 Subject: [PATCH 11/27] add description to s3 drop folder S3 Settings --- .../drop_folder/admin/forms/DropFolderConfigure.php | 8 +++++++- .../admin/forms/DropFolderConfigureExtend.php | 7 ++++++- .../admin/forms/S3DropFolderConfigureExtend.php | 11 +++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/plugins/drop_folder/admin/forms/DropFolderConfigure.php b/plugins/drop_folder/admin/forms/DropFolderConfigure.php index 26ab8be3985..a686ea5fb1f 100644 --- a/plugins/drop_folder/admin/forms/DropFolderConfigure.php +++ b/plugins/drop_folder/admin/forms/DropFolderConfigure.php @@ -220,7 +220,13 @@ public function init() )); $extendTypeSubFormTitle = new Zend_Form_Element_Hidden(self::EXTENSION_SUBFORM_NAME.'_title'); $extendTypeSubFormTitle->setLabel($extendTypeSubForm->getTitle()); - $extendTypeSubFormTitle->setDecorators(array('ViewHelper', array('Label', array('placement' => 'append')), array('HtmlTag', array('tag' => 'b')))); + $extendTypeSubFormTitle->setDescription($extendTypeSubForm->getDescription()); + $extendTypeSubFormTitle->setDecorators(array( + 'ViewHelper', + array('Label',array('placement' => 'append')), + array('HtmlTag', array('tag' => 'b')), + array('Description', array('escape' => false, 'tag' => 'p', 'class' => 'description'))) + ); $this->addElement($extendTypeSubFormTitle); $extendTypeSubForm->setDecorators(array( 'FormElements', diff --git a/plugins/drop_folder/admin/forms/DropFolderConfigureExtend.php b/plugins/drop_folder/admin/forms/DropFolderConfigureExtend.php index 2bfc39515d4..b83df0562e2 100644 --- a/plugins/drop_folder/admin/forms/DropFolderConfigureExtend.php +++ b/plugins/drop_folder/admin/forms/DropFolderConfigureExtend.php @@ -24,4 +24,9 @@ public function getTitle() { return; } -} \ No newline at end of file + + public function getDescription() + { + return; + } +} diff --git a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php index c2435811e76..dd419bb2418 100644 --- a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php +++ b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php @@ -9,6 +9,17 @@ public function getTitle() { return 'S3 settings'; } + + public function getDescription() + { + return "Authentication precedence:
+ 1. User & Password (leave empty to use 'IAM Role')

+ 2. IAM Role:
+ 2.1. Set Role ARN - will be used to assume that role
+ Note: role 'Trust Policy' must allow 'cloud_storage' s3Arn to assume it

+ 2.2. Leave empty - will use 'cloud_storage' s3Arn to access bucket
+ Note: Bucket Policy must allow s3Arn to operate it"; + } public function init() { From db063828901376fe30fb6154f4a527145d1ab5b2 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Tue, 4 Feb 2025 16:07:04 +0200 Subject: [PATCH 12/27] rename S3IAMRole to s3Arn since S3DropFolderFile send $s3Options which are not recognized by s3Mgr constructor --- .../batch/Engine/KDropFolderFileTransferEngine.php | 2 +- .../admin/forms/S3DropFolderConfigureExtend.php | 8 ++++---- .../s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php | 4 ++-- plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php | 10 +++++----- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php index 5d7b678ef5a..67d98dac9e4 100644 --- a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php +++ b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php @@ -280,7 +280,7 @@ public static function getFileTransferManager(KalturaDropFolder $dropFolder) if($dropFolder instanceof KalturaS3DropFolder) { $engineOptions['s3Region'] = $dropFolder->s3Region; - $engineOptions['s3Arn'] = !empty($dropFolder->s3IAMRole) ? $dropFolder->s3IAMRole : null; + $engineOptions['s3Arn'] = !empty($dropFolder->s3Arn) ? $dropFolder->s3Arn : null; } $fileTransferMgr = kFileTransferMgr::getInstance(self::getFileTransferMgrType($dropFolder->type), $engineOptions); diff --git a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php index dd419bb2418..fd93f645444 100644 --- a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php +++ b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php @@ -13,8 +13,8 @@ public function getTitle() public function getDescription() { return "Authentication precedence:
- 1. User & Password (leave empty to use 'IAM Role')

- 2. IAM Role:
+ 1. User & Password (leave empty to use 'S3 ARN')

+ 2. S3 ARN:
2.1. Set Role ARN - will be used to assume that role
Note: role 'Trust Policy' must allow 'cloud_storage' s3Arn to assume it

2.2. Leave empty - will use 'cloud_storage' s3Arn to access bucket
@@ -43,8 +43,8 @@ public function init() 'filters' => array('StringTrim'), )); - $this->addElement('text', 's3IAMRole', array( - 'label' => 'IAM Role:', + $this->addElement('text', 's3Arn', array( + 'label' => 'S3 ARN:', 'filters' => array('StringTrim'), )); } diff --git a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php index 0af5fa67f2e..07462142dc0 100644 --- a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php +++ b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php @@ -28,7 +28,7 @@ class KalturaS3DropFolder extends KalturaDropFolder /** * @var string */ - public $s3IAMRole; + public $s3Arn; /* * mapping between the field on this object (on the left) and the setter/getter on the entry object (on the right) @@ -38,7 +38,7 @@ class KalturaS3DropFolder extends KalturaDropFolder 's3Region', 's3UserId', 's3Password', - 's3IAMRole' + 's3Arn' ); public function getMapBetweenObjects() diff --git a/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php b/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php index 1d32ee56cfa..e97938a7cd7 100644 --- a/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php +++ b/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php @@ -13,7 +13,7 @@ class S3DropFolder extends RemoteDropFolder const S3_PASSWORD = 's3Password'; - const S3_IAM_ROLE = 's3IAMRole'; + const S3_ARN = 's3Arn'; /** * @var string @@ -38,7 +38,7 @@ class S3DropFolder extends RemoteDropFolder /** * @var string */ - protected $s3IAMRole; + protected $s3Arn; /** * return string @@ -63,7 +63,7 @@ public function getS3Password (){ return $this->getFromCustomData(self::S3_PASSW /** * @return string|null */ - public function getS3IAMRole(){ return $this->getFromCustomData(self::S3_IAM_ROLE);} + public function getS3Arn (){ return $this->getFromCustomData(self::S3_ARN);} /** * @param string $v @@ -88,7 +88,7 @@ public function setS3Password ($v){ $this->putInCustomData(self::S3_PASSWORD, $v /** * @param string $v */ - public function setS3IAMRole ($v){ $this->putInCustomData(self::S3_IAM_ROLE, $v);} + public function setS3Arn ($v){ $this->putInCustomData(self::S3_ARN, $v);} public function getImportJobData() @@ -118,7 +118,7 @@ public function getDropFolderParams() 's3UserId' => $this->getS3UserId(), 's3Password' => $this->getS3Password(), 's3Region' => $this->getS3Region(), - 's3IAMRole' => $this->getS3IAMRole(), + 's3Arn' => $this->getS3Arn(), ); } } From b2fad942241f59658bb5e637043f81be83801ef3 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Wed, 5 Feb 2025 12:52:25 +0200 Subject: [PATCH 13/27] fix papi feedback --- .../file_transfer_managers/s3Mgr.class.php | 17 ++++++++--------- .../lib/model/S3DropFolderFile.php | 1 - 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/infra/storage/file_transfer_managers/s3Mgr.class.php b/infra/storage/file_transfer_managers/s3Mgr.class.php index 6a275a7137e..615129336b5 100644 --- a/infra/storage/file_transfer_managers/s3Mgr.class.php +++ b/infra/storage/file_transfer_managers/s3Mgr.class.php @@ -33,7 +33,7 @@ class s3Mgr extends kFileTransferMgr protected $endPoint = null; protected $storageClass = null; protected $s3Arn = null; - protected $hash = false; + protected $dirnameSuffix = null; const SIZE = 'Size'; const LAST_MODIFICATION = 'LastModified'; @@ -84,7 +84,7 @@ protected function __construct(array $options = null) if($options && isset($options['s3Arn'])) { $this->s3Arn = $options['s3Arn']; - $this->hash = true; + $this->dirnameSuffix = hash('md5', $this->s3Arn); } else { @@ -96,7 +96,10 @@ protected function __construct(array $options = null) { $this->s3Arn = kConf::get('s3Arn', 'cloud_storage', null); } + + $this->dirnameSuffix = 's3Mgr'; } + // do nothing $this->connection_id = 1; //SIMULATING! @@ -131,11 +134,9 @@ protected function doLogin($sftp_user, $sftp_pass) return false; } - if($this->s3Arn && $this->s3Region && (!isset($sftp_user) || !$sftp_user) && (!isset($sftp_pass) || !$sftp_pass)) + if($this->s3Arn && (!isset($sftp_user) || !$sftp_user) && (!isset($sftp_pass) || !$sftp_pass)) { - $s3Arn = $this->s3Arn; - $s3Region = $this->s3Region; - KalturaLog::debug("Login using ARN [ $s3Arn ] and region [ $s3Region ]"); + KalturaLog::debug("Login using ARN [ {$this->s3Arn} ] and region [ {$this->s3Region} ]"); if(!class_exists('Aws\Sts\StsClient')) { @@ -157,9 +158,7 @@ protected function doLogin($sftp_user, $sftp_pass) private function generateCachedCredentialsS3Client() { - $dirnameSuffix = $this->hash ? hash('md5', $this->s3Arn) : null; - - $cacheProviderCredentials = RefreshableRole::getCacheCredentialsProvider($this->s3Arn, $this->s3Region, $dirnameSuffix); + $cacheProviderCredentials = RefreshableRole::getCacheCredentialsProvider($this->s3Arn, $this->s3Region, $this->dirnameSuffix); $config = $this->getBaseClientConfig(); $config['credentials'] = $cacheProviderCredentials; $this->s3 = S3Client::factory($config); diff --git a/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php b/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php index e22cf767ebe..45322efba29 100644 --- a/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php +++ b/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php @@ -19,7 +19,6 @@ public function getNameForParsing () public function getFileUrl() { - KalturaLog::debug("Generating drop folder file URL"); $dbDropFolder = DropFolderPeer::retrieveByPK($this->getDropFolderId()); $s3Options = $dbDropFolder->getDropFolderParams(); $s3TransferMgr = kFileTransferMgr::getInstance(kFileTransferMgrType::S3, $s3Options); From 74221e7fe5c62b312c7e633a9ecf698ae0d94d53 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Wed, 5 Feb 2025 19:19:58 +0200 Subject: [PATCH 14/27] change logic get s3Arn from drop folder worker config --- .../file_transfer_managers/s3Mgr.class.php | 3 +-- .../Engine/KDropFolderFileTransferEngine.php | 2 +- .../forms/S3DropFolderConfigureExtend.php | 20 +++++++++---------- .../lib/api/KalturaS3DropFolder.php | 2 +- .../lib/model/S3DropFolder.php | 12 +++++------ 5 files changed, 18 insertions(+), 21 deletions(-) diff --git a/infra/storage/file_transfer_managers/s3Mgr.class.php b/infra/storage/file_transfer_managers/s3Mgr.class.php index 615129336b5..a1c58bece30 100644 --- a/infra/storage/file_transfer_managers/s3Mgr.class.php +++ b/infra/storage/file_transfer_managers/s3Mgr.class.php @@ -84,7 +84,7 @@ protected function __construct(array $options = null) if($options && isset($options['s3Arn'])) { $this->s3Arn = $options['s3Arn']; - $this->dirnameSuffix = hash('md5', $this->s3Arn); + $this->dirnameSuffix = 'remote_s3_drop_folder'; } else { @@ -99,7 +99,6 @@ protected function __construct(array $options = null) $this->dirnameSuffix = 's3Mgr'; } - // do nothing $this->connection_id = 1; //SIMULATING! diff --git a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php index 67d98dac9e4..30be7da8539 100644 --- a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php +++ b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php @@ -280,7 +280,7 @@ public static function getFileTransferManager(KalturaDropFolder $dropFolder) if($dropFolder instanceof KalturaS3DropFolder) { $engineOptions['s3Region'] = $dropFolder->s3Region; - $engineOptions['s3Arn'] = !empty($dropFolder->s3Arn) ? $dropFolder->s3Arn : null; + $engineOptions['s3Arn'] = $dropFolder->s3Arn ? kBatchBase::$taskConfig->params->s3Arn : null; } $fileTransferMgr = kFileTransferMgr::getInstance(self::getFileTransferMgrType($dropFolder->type), $engineOptions); diff --git a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php index fd93f645444..947477956cb 100644 --- a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php +++ b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php @@ -13,12 +13,9 @@ public function getTitle() public function getDescription() { return "Authentication precedence:
- 1. User & Password (leave empty to use 'S3 ARN')

- 2. S3 ARN:
- 2.1. Set Role ARN - will be used to assume that role
- Note: role 'Trust Policy' must allow 'cloud_storage' s3Arn to assume it

- 2.2. Leave empty - will use 'cloud_storage' s3Arn to access bucket
- Note: Bucket Policy must allow s3Arn to operate it"; + 1. User & Password (if passed will be used)
+ 2. Bucket Policy Allow Access:
+ Note: Bucket policy must allow 'DropFolderWatcherRemoteS3' s3Arn role"; } public function init() @@ -42,11 +39,12 @@ public function init() 'label' => 'Password:', 'filters' => array('StringTrim'), )); - - $this->addElement('text', 's3Arn', array( - 'label' => 'S3 ARN:', - 'filters' => array('StringTrim'), + + $this->addElement('checkbox', 's3Arn', array( + 'label' => 'Bucket Policy Allow Access', + 'decorators' => array('ViewHelper', array('Label', array('placement' => 'append')), array('HtmlTag', array('tag' => 'dt'))), + 'uncheckedValue' => false, + 'checkedValue' => true, )); } - } diff --git a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php index 07462142dc0..c95a5eb55e4 100644 --- a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php +++ b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php @@ -26,7 +26,7 @@ class KalturaS3DropFolder extends KalturaDropFolder public $s3Password; /** - * @var string + * @var bool */ public $s3Arn; diff --git a/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php b/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php index e97938a7cd7..0c5cda54855 100644 --- a/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php +++ b/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php @@ -36,7 +36,7 @@ class S3DropFolder extends RemoteDropFolder protected $s3Password; /** - * @var string + * @var bool */ protected $s3Arn; @@ -61,9 +61,9 @@ public function getS3UserId (){ return $this->getFromCustomData(self::S3_USER_ID public function getS3Password (){ return $this->getFromCustomData(self::S3_PASSWORD);} /** - * @return string|null + * @return bool */ - public function getS3Arn (){ return $this->getFromCustomData(self::S3_ARN);} + public function getS3Arn (){ return (bool) $this->getFromCustomData(self::S3_ARN);} /** * @param string $v @@ -86,9 +86,9 @@ public function setS3UserId ($v){ $this->putInCustomData(self::S3_USER_ID, $v);} public function setS3Password ($v){ $this->putInCustomData(self::S3_PASSWORD, $v);} /** - * @param string $v + * @param bool $v */ - public function setS3Arn ($v){ $this->putInCustomData(self::S3_ARN, $v);} + public function setS3Arn ($v){ $this->putInCustomData(self::S3_ARN, (bool) $v);} public function getImportJobData() @@ -118,7 +118,7 @@ public function getDropFolderParams() 's3UserId' => $this->getS3UserId(), 's3Password' => $this->getS3Password(), 's3Region' => $this->getS3Region(), - 's3Arn' => $this->getS3Arn(), + 's3Arn' => (boolean) $this->getS3Arn(), ); } } From aead7f8b3ca73b34b80541d707d4e43cd79cfbcf Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Wed, 5 Feb 2025 20:04:07 +0200 Subject: [PATCH 15/27] change s3Arn to useS3Arn --- .../batch/Engine/KDropFolderFileTransferEngine.php | 2 +- .../admin/forms/S3DropFolderConfigureExtend.php | 2 +- .../s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php | 4 ++-- plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php | 10 +++++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php index 30be7da8539..383cffdc745 100644 --- a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php +++ b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php @@ -280,7 +280,7 @@ public static function getFileTransferManager(KalturaDropFolder $dropFolder) if($dropFolder instanceof KalturaS3DropFolder) { $engineOptions['s3Region'] = $dropFolder->s3Region; - $engineOptions['s3Arn'] = $dropFolder->s3Arn ? kBatchBase::$taskConfig->params->s3Arn : null; + $engineOptions['s3Arn'] = $dropFolder->useS3Arn ? kBatchBase::$taskConfig->params->s3Arn : null; } $fileTransferMgr = kFileTransferMgr::getInstance(self::getFileTransferMgrType($dropFolder->type), $engineOptions); diff --git a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php index 947477956cb..de76530d5ae 100644 --- a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php +++ b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php @@ -40,7 +40,7 @@ public function init() 'filters' => array('StringTrim'), )); - $this->addElement('checkbox', 's3Arn', array( + $this->addElement('checkbox', 'useS3Arn', array( 'label' => 'Bucket Policy Allow Access', 'decorators' => array('ViewHelper', array('Label', array('placement' => 'append')), array('HtmlTag', array('tag' => 'dt'))), 'uncheckedValue' => false, diff --git a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php index c95a5eb55e4..3438014e5ae 100644 --- a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php +++ b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php @@ -28,7 +28,7 @@ class KalturaS3DropFolder extends KalturaDropFolder /** * @var bool */ - public $s3Arn; + public $useS3Arn; /* * mapping between the field on this object (on the left) and the setter/getter on the entry object (on the right) @@ -38,7 +38,7 @@ class KalturaS3DropFolder extends KalturaDropFolder 's3Region', 's3UserId', 's3Password', - 's3Arn' + 'useS3Arn' ); public function getMapBetweenObjects() diff --git a/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php b/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php index 0c5cda54855..18101e2a019 100644 --- a/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php +++ b/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php @@ -13,7 +13,7 @@ class S3DropFolder extends RemoteDropFolder const S3_PASSWORD = 's3Password'; - const S3_ARN = 's3Arn'; + const USE_S3_ARN = 'useS3Arn'; /** * @var string @@ -38,7 +38,7 @@ class S3DropFolder extends RemoteDropFolder /** * @var bool */ - protected $s3Arn; + protected $useS3Arn; /** * return string @@ -63,7 +63,7 @@ public function getS3Password (){ return $this->getFromCustomData(self::S3_PASSW /** * @return bool */ - public function getS3Arn (){ return (bool) $this->getFromCustomData(self::S3_ARN);} + public function getUseS3Arn (){ return (bool) $this->getFromCustomData(self::USE_S3_ARN);} /** * @param string $v @@ -88,7 +88,7 @@ public function setS3Password ($v){ $this->putInCustomData(self::S3_PASSWORD, $v /** * @param bool $v */ - public function setS3Arn ($v){ $this->putInCustomData(self::S3_ARN, (bool) $v);} + public function setUseS3Arn ($v){ $this->putInCustomData(self::USE_S3_ARN, (bool) $v);} public function getImportJobData() @@ -118,7 +118,7 @@ public function getDropFolderParams() 's3UserId' => $this->getS3UserId(), 's3Password' => $this->getS3Password(), 's3Region' => $this->getS3Region(), - 's3Arn' => (boolean) $this->getS3Arn(), + 's3Arn' => $this->getUseS3Arn(), ); } } From 212132bc3c6543443e218476b3aa9e62cefbee6b Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Thu, 6 Feb 2025 14:41:17 +0200 Subject: [PATCH 16/27] add validation --- .../file_transfer_managers/s3Mgr.class.php | 2 +- .../Engine/KDropFolderFileTransferEngine.php | 15 ++++++++++++++- .../s3DropFolderPlugin/lib/model/S3DropFolder.php | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/infra/storage/file_transfer_managers/s3Mgr.class.php b/infra/storage/file_transfer_managers/s3Mgr.class.php index a1c58bece30..ded741b1b4f 100644 --- a/infra/storage/file_transfer_managers/s3Mgr.class.php +++ b/infra/storage/file_transfer_managers/s3Mgr.class.php @@ -84,7 +84,7 @@ protected function __construct(array $options = null) if($options && isset($options['s3Arn'])) { $this->s3Arn = $options['s3Arn']; - $this->dirnameSuffix = 'remote_s3_drop_folder'; + $this->dirnameSuffix = 'dropFolderWatcherRemoteS3'; } else { diff --git a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php index 383cffdc745..67a03eb3f76 100644 --- a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php +++ b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php @@ -280,8 +280,21 @@ public static function getFileTransferManager(KalturaDropFolder $dropFolder) if($dropFolder instanceof KalturaS3DropFolder) { $engineOptions['s3Region'] = $dropFolder->s3Region; - $engineOptions['s3Arn'] = $dropFolder->useS3Arn ? kBatchBase::$taskConfig->params->s3Arn : null; + + if ($dropFolder->useS3Arn) + { + if (!empty(kBatchBase::$taskConfig->params->s3Arn)) + { + $engineOptions['s3Arn'] = kBatchBase::$taskConfig->params->s3Arn; + } + else + { + $msg = "Drop Folder ID [{$dropFolder->id}] enabled 'Bucket Policy Allow Access' but 's3Arn' value in 'DropFolderWatcherRemoteS3' is missing"; + throw new kFileTransferMgrException($msg, kFileTransferMgrException::otherError); + } + } } + $fileTransferMgr = kFileTransferMgr::getInstance(self::getFileTransferMgrType($dropFolder->type), $engineOptions); $host =null; $username=null; $password=null; $port=null; diff --git a/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php b/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php index 18101e2a019..8adec6431ca 100644 --- a/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php +++ b/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php @@ -118,7 +118,7 @@ public function getDropFolderParams() 's3UserId' => $this->getS3UserId(), 's3Password' => $this->getS3Password(), 's3Region' => $this->getS3Region(), - 's3Arn' => $this->getUseS3Arn(), + 'useS3Arn' => $this->getUseS3Arn(), ); } } From 2f2d831584079feb27dcf27095eb7717c16a16a9 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Thu, 6 Feb 2025 14:44:49 +0200 Subject: [PATCH 17/27] modify df description --- .../admin/forms/S3DropFolderConfigureExtend.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php index de76530d5ae..519ed916675 100644 --- a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php +++ b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php @@ -14,8 +14,8 @@ public function getDescription() { return "Authentication precedence:
1. User & Password (if passed will be used)
- 2. Bucket Policy Allow Access:
- Note: Bucket policy must allow 'DropFolderWatcherRemoteS3' s3Arn role"; + 2. Bucket Policy Allows Access
+ Note: Bucket policy must allow 'DropFolderWatcherRemoteS3' s3Arn role to operate it"; } public function init() From c5288d0d0c0a1082705d3dd7172cbcddaf2298a5 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Sun, 9 Feb 2025 11:14:09 +0200 Subject: [PATCH 18/27] set "remember" to new DF checkbox to show db value --- .../admin/forms/S3DropFolderConfigureExtend.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php index 519ed916675..dc3fa4f0485 100644 --- a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php +++ b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php @@ -42,7 +42,7 @@ public function init() $this->addElement('checkbox', 'useS3Arn', array( 'label' => 'Bucket Policy Allow Access', - 'decorators' => array('ViewHelper', array('Label', array('placement' => 'append')), array('HtmlTag', array('tag' => 'dt'))), + 'decorators' => array('ViewHelper', array('Label', array('placement' => 'append')), array('HtmlTag', array('tag' => 'dt', 'class' => 'rememeber'))), 'uncheckedValue' => false, 'checkedValue' => true, )); From d2c683051e21ff145fa77524981b3fc2e2a4062b Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Sun, 9 Feb 2025 11:20:53 +0200 Subject: [PATCH 19/27] set "tag => div" over "dt" since there is no need for a list --- .../admin/forms/S3DropFolderConfigureExtend.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php index dc3fa4f0485..365cddbf286 100644 --- a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php +++ b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php @@ -42,7 +42,7 @@ public function init() $this->addElement('checkbox', 'useS3Arn', array( 'label' => 'Bucket Policy Allow Access', - 'decorators' => array('ViewHelper', array('Label', array('placement' => 'append')), array('HtmlTag', array('tag' => 'dt', 'class' => 'rememeber'))), + 'decorators' => array('ViewHelper', array('Label', array('placement' => 'append')), array('HtmlTag', array('tag' => 'div', 'class' => 'rememeber'))), 'uncheckedValue' => false, 'checkedValue' => true, )); From dbe8fa63f86a5b75489c1b4352e763c3f1c1332d Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Sun, 16 Feb 2025 14:36:35 +0200 Subject: [PATCH 20/27] use new drop_folder map to fetch s3Arn rather then taking from worker params --- .../batch/Engine/KDropFolderFileTransferEngine.php | 10 ++++------ .../lib/model/S3DropFolderFile.php | 13 +++++++++++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php index 67a03eb3f76..11306b5c5fe 100644 --- a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php +++ b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php @@ -283,13 +283,11 @@ public static function getFileTransferManager(KalturaDropFolder $dropFolder) if ($dropFolder->useS3Arn) { - if (!empty(kBatchBase::$taskConfig->params->s3Arn)) - { - $engineOptions['s3Arn'] = kBatchBase::$taskConfig->params->s3Arn; - } - else + $engineOptions['s3Arn'] = kConf::getArrayValue('s3Arn', 's3_drop_folder', 'drop_folder', null); + + if (empty($engineOptions['s3Arn'])) { - $msg = "Drop Folder ID [{$dropFolder->id}] enabled 'Bucket Policy Allow Access' but 's3Arn' value in 'DropFolderWatcherRemoteS3' is missing"; + $msg = "Drop Folder ID [{$dropFolder->id}] enabled 'Bucket Policy Allow Access' but 's3Arn' value under 's3_drop_folder' in 'drop_folder' map is missing"; throw new kFileTransferMgrException($msg, kFileTransferMgrException::otherError); } } diff --git a/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php b/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php index 45322efba29..f253cb13bbd 100644 --- a/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php +++ b/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php @@ -21,6 +21,19 @@ public function getFileUrl() { $dbDropFolder = DropFolderPeer::retrieveByPK($this->getDropFolderId()); $s3Options = $dbDropFolder->getDropFolderParams(); + + if ($s3Options['useS3Arn']) + { + $s3Options['s3Arn'] = kConf::getArrayValue('s3Arn', 's3_drop_folder', 'drop_folder', null); + + if (empty($s3Options['s3Arn'])) + { + $msg = "Drop Folder ID [{$dbDropFolder->getId()}] enabled 'Bucket Policy Allow Access' but 's3Arn' value under 's3_drop_folder' in 'drop_folder' map is missing"; + throw new kFileTransferMgrException($msg, kFileTransferMgrException::otherError); + } + } + + $s3TransferMgr = kFileTransferMgr::getInstance(kFileTransferMgrType::S3, $s3Options); $s3TransferMgr->login($dbDropFolder->getS3Host(), $dbDropFolder->getS3UserId(), $dbDropFolder->getS3Password()); $expiry = time() + 5 * 86400; From 7250cf6bc40edd98d762cbd71019e6765eda5cef Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Mon, 17 Feb 2025 14:55:50 +0200 Subject: [PATCH 21/27] load s3Arn in runtime from runtime_config --- .../Engine/KDropFolderFileTransferEngine.php | 4 ++-- .../lib/api/KalturaS3DropFolder.php | 9 ++++++++- .../lib/model/S3DropFolder.php | 19 +++++++++++++++++++ .../lib/model/S3DropFolderFile.php | 11 +++-------- 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php index 11306b5c5fe..39768f6afaf 100644 --- a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php +++ b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php @@ -283,11 +283,11 @@ public static function getFileTransferManager(KalturaDropFolder $dropFolder) if ($dropFolder->useS3Arn) { - $engineOptions['s3Arn'] = kConf::getArrayValue('s3Arn', 's3_drop_folder', 'drop_folder', null); + $engineOptions['s3Arn'] = $dropFolder->s3Arn; if (empty($engineOptions['s3Arn'])) { - $msg = "Drop Folder ID [{$dropFolder->id}] enabled 'Bucket Policy Allow Access' but 's3Arn' value under 's3_drop_folder' in 'drop_folder' map is missing"; + $msg = "Drop Folder ID [{$dropFolder->id}] enabled 'Bucket Policy Allow Access' but 's3Arn' value under 's3_drop_folder' in 'runtime_config' map is missing"; throw new kFileTransferMgrException($msg, kFileTransferMgrException::otherError); } } diff --git a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php index 3438014e5ae..2fc9697ff34 100644 --- a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php +++ b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php @@ -29,6 +29,12 @@ class KalturaS3DropFolder extends KalturaDropFolder * @var bool */ public $useS3Arn; + + /** + * @var string + * @readonly + */ + public $s3Arn; /* * mapping between the field on this object (on the left) and the setter/getter on the entry object (on the right) @@ -38,7 +44,8 @@ class KalturaS3DropFolder extends KalturaDropFolder 's3Region', 's3UserId', 's3Password', - 'useS3Arn' + 'useS3Arn', + 's3Arn' ); public function getMapBetweenObjects() diff --git a/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php b/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php index 8adec6431ca..145719f4159 100644 --- a/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php +++ b/plugins/s3DropFolderPlugin/lib/model/S3DropFolder.php @@ -39,6 +39,11 @@ class S3DropFolder extends RemoteDropFolder * @var bool */ protected $useS3Arn; + + /** + * @var string + */ + protected $s3Arn; /** * return string @@ -64,6 +69,19 @@ public function getS3Password (){ return $this->getFromCustomData(self::S3_PASSW * @return bool */ public function getUseS3Arn (){ return (bool) $this->getFromCustomData(self::USE_S3_ARN);} + + /** + * @return string + */ + public function getS3Arn () + { + if (empty($this->getUseS3Arn())) + { + return null; + } + + return kConf::getArrayValue('s3Arn', 's3_drop_folder', 'runtime_config', null); + } /** * @param string $v @@ -119,6 +137,7 @@ public function getDropFolderParams() 's3Password' => $this->getS3Password(), 's3Region' => $this->getS3Region(), 'useS3Arn' => $this->getUseS3Arn(), + 's3Arn' => $this->getS3Arn() ); } } diff --git a/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php b/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php index f253cb13bbd..d553c63293a 100644 --- a/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php +++ b/plugins/s3DropFolderPlugin/lib/model/S3DropFolderFile.php @@ -22,15 +22,10 @@ public function getFileUrl() $dbDropFolder = DropFolderPeer::retrieveByPK($this->getDropFolderId()); $s3Options = $dbDropFolder->getDropFolderParams(); - if ($s3Options['useS3Arn']) + if ($s3Options['useS3Arn'] && empty($s3Options['s3Arn'])) { - $s3Options['s3Arn'] = kConf::getArrayValue('s3Arn', 's3_drop_folder', 'drop_folder', null); - - if (empty($s3Options['s3Arn'])) - { - $msg = "Drop Folder ID [{$dbDropFolder->getId()}] enabled 'Bucket Policy Allow Access' but 's3Arn' value under 's3_drop_folder' in 'drop_folder' map is missing"; - throw new kFileTransferMgrException($msg, kFileTransferMgrException::otherError); - } + $msg = "Drop Folder ID [{$dbDropFolder->getId()}] enabled 'Bucket Policy Allow Access' but 's3Arn' value under 's3_drop_folder' in 'runtime_config' map is missing"; + throw new kFileTransferMgrException($msg, kFileTransferMgrException::otherError); } From ff85c8adb99d7dad40d51df84c887241277fe963 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Mon, 17 Feb 2025 14:59:19 +0200 Subject: [PATCH 22/27] fix admin add df description --- .../admin/forms/S3DropFolderConfigureExtend.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php index 365cddbf286..a9692f8b023 100644 --- a/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php +++ b/plugins/s3DropFolderPlugin/admin/forms/S3DropFolderConfigureExtend.php @@ -15,7 +15,7 @@ public function getDescription() return "Authentication precedence:
1. User & Password (if passed will be used)
2. Bucket Policy Allows Access
- Note: Bucket policy must allow 'DropFolderWatcherRemoteS3' s3Arn role to operate it"; + Note: Bucket policy must allow 'runtime_config' map 's3_drop_folder' section 's3Arn' value role to operate it"; } public function init() From 9a6e4edeb32bd2ce9df073e2614b5a64494db739 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Wed, 19 Feb 2025 15:42:35 +0200 Subject: [PATCH 23/27] throw Exception and not kFileTransferMgrException to show error msg in DF Error Message --- .../drop_folder/batch/Engine/KDropFolderFileTransferEngine.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php index 39768f6afaf..e836632f05e 100644 --- a/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php +++ b/plugins/drop_folder/batch/Engine/KDropFolderFileTransferEngine.php @@ -287,8 +287,9 @@ public static function getFileTransferManager(KalturaDropFolder $dropFolder) if (empty($engineOptions['s3Arn'])) { + // throw Exception and not kFileTransferMgrException::otherError to catch msg at: plugins/drop_folder/batch/DropFolderWatcher/KAsyncDropFolderWatcher.class.php:72 $msg = "Drop Folder ID [{$dropFolder->id}] enabled 'Bucket Policy Allow Access' but 's3Arn' value under 's3_drop_folder' in 'runtime_config' map is missing"; - throw new kFileTransferMgrException($msg, kFileTransferMgrException::otherError); + throw new Exception($msg, kFileTransferMgrException::otherError); } } } From f30bd509671b2911fdc5e455bf2be882e1b9409e Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Wed, 19 Feb 2025 16:59:53 +0200 Subject: [PATCH 24/27] add insert validation when adding df with useS3Arn "Bucket Policy Allow Access" true --- .../drop_folder/lib/api/types/KalturaDropFolder.php | 5 +++++ .../lib/api/KalturaS3DropFolder.php | 13 +++++++++++++ .../lib/api/errors/KalturaS3DropFolderErrors.php | 10 ++++++++++ 3 files changed, 28 insertions(+) create mode 100644 plugins/s3DropFolderPlugin/lib/api/errors/KalturaS3DropFolderErrors.php diff --git a/plugins/drop_folder/lib/api/types/KalturaDropFolder.php b/plugins/drop_folder/lib/api/types/KalturaDropFolder.php index af5fa2ad2f0..bd3fb58e840 100644 --- a/plugins/drop_folder/lib/api/types/KalturaDropFolder.php +++ b/plugins/drop_folder/lib/api/types/KalturaDropFolder.php @@ -291,6 +291,11 @@ public function toInsertableObject ( $object_to_fill = null , $props_to_skip = a return $result; } + public function validateForInsert($propertiesToSkip = array()) + { + parent::validateForInsert($propertiesToSkip); + } + /** * @param int $type * @return KalturaDropFolder diff --git a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php index 2fc9697ff34..1113aab9639 100644 --- a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php +++ b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php @@ -63,4 +63,17 @@ public function toObject($dbObject = null, $skip = array()) $dbObject->setType(S3DropFolderPlugin::getDropFolderTypeCoreValue(S3DropFolderType::S3DROPFOLDER)); return parent::toObject($dbObject, $skip); } + + public function validateForInsert($propertiesToSkip = array()) + { + if ($this->useS3Arn) + { + if (empty(kConf::getArrayValue('s3Arnnnnn', 's3_drop_folder', 'runtime_config', null))) + { + throw new KalturaAPIException(KalturaS3DropFolderErrors::MISSING_S3ARN_CONFIG); + } + } + + parent::validateForInsert($propertiesToSkip); + } } diff --git a/plugins/s3DropFolderPlugin/lib/api/errors/KalturaS3DropFolderErrors.php b/plugins/s3DropFolderPlugin/lib/api/errors/KalturaS3DropFolderErrors.php new file mode 100644 index 00000000000..6ed7d855f14 --- /dev/null +++ b/plugins/s3DropFolderPlugin/lib/api/errors/KalturaS3DropFolderErrors.php @@ -0,0 +1,10 @@ + Date: Wed, 19 Feb 2025 17:02:39 +0200 Subject: [PATCH 25/27] fix test typo --- plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php index 1113aab9639..f08b62ffb39 100644 --- a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php +++ b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php @@ -68,7 +68,7 @@ public function validateForInsert($propertiesToSkip = array()) { if ($this->useS3Arn) { - if (empty(kConf::getArrayValue('s3Arnnnnn', 's3_drop_folder', 'runtime_config', null))) + if (empty(kConf::getArrayValue('s3Arn', 's3_drop_folder', 'runtime_config', null))) { throw new KalturaAPIException(KalturaS3DropFolderErrors::MISSING_S3ARN_CONFIG); } From 8222c8ab5450f637fde3b2dbc546d34ec8d994b1 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Wed, 19 Feb 2025 17:10:28 +0200 Subject: [PATCH 26/27] change exception error --- plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php | 2 +- .../lib/api/errors/KalturaS3DropFolderErrors.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php index f08b62ffb39..b79f9519ff0 100644 --- a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php +++ b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php @@ -68,7 +68,7 @@ public function validateForInsert($propertiesToSkip = array()) { if ($this->useS3Arn) { - if (empty(kConf::getArrayValue('s3Arn', 's3_drop_folder', 'runtime_config', null))) + if (empty(kConf::getArrayValue('s3Arnn', 's3_drop_folder', 'runtime_config', null))) { throw new KalturaAPIException(KalturaS3DropFolderErrors::MISSING_S3ARN_CONFIG); } diff --git a/plugins/s3DropFolderPlugin/lib/api/errors/KalturaS3DropFolderErrors.php b/plugins/s3DropFolderPlugin/lib/api/errors/KalturaS3DropFolderErrors.php index 6ed7d855f14..11e68222314 100644 --- a/plugins/s3DropFolderPlugin/lib/api/errors/KalturaS3DropFolderErrors.php +++ b/plugins/s3DropFolderPlugin/lib/api/errors/KalturaS3DropFolderErrors.php @@ -6,5 +6,5 @@ */ class KalturaS3DropFolderErrors extends KalturaErrors { - const MISSING_S3ARN_CONFIG = "MISSING_S3_ARN;;'useS3Arn' ('Bucket Policy Allow Access') was set but 's3Arn' value under 's3_drop_folder' in 'runtime_config' map is missing"; + const MISSING_S3ARN_CONFIG = "MISSING_S3_ARN;;'useS3Arn' ('Bucket Policy Allow Access') was set but 's3Arn' config is missing"; } From f0f47063dbd79f4a937432f5d26cbcc3f80cd281 Mon Sep 17 00:00:00 2001 From: Amir Asaelov Date: Wed, 19 Feb 2025 17:13:43 +0200 Subject: [PATCH 27/27] change test intentinal typo --- plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php index b79f9519ff0..f08b62ffb39 100644 --- a/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php +++ b/plugins/s3DropFolderPlugin/lib/api/KalturaS3DropFolder.php @@ -68,7 +68,7 @@ public function validateForInsert($propertiesToSkip = array()) { if ($this->useS3Arn) { - if (empty(kConf::getArrayValue('s3Arnn', 's3_drop_folder', 'runtime_config', null))) + if (empty(kConf::getArrayValue('s3Arn', 's3_drop_folder', 'runtime_config', null))) { throw new KalturaAPIException(KalturaS3DropFolderErrors::MISSING_S3ARN_CONFIG); }